一个支持断点续传的大文件分片上传的小模块

2021 年 12 月 23 日
本文字数：5265 字
阅读完需：约 17 分钟

为什么要分片

当前几乎所有的 cms 系统都会有上传文件的需求，而要上传的文件除了文档，图片等小型文件外，偶尔也会遇到上传大的视频文件，压缩包等等，这时候就要用到分片上传的功能。如果是内网环境下，我们可以考虑使用 ftp 来上传大文件，但大多数情况我们都是在外网环境下进行上传的，当然也可以配置 ftp 来上传，但少不了一番折腾，而且还会造成很大的安全隐患。

分片上传是走的 http 协议的，优点就是安全，方便；缺点就是速度慢，而且 http 上传大文件会有网络限制等问题，所以就需要进行分片上传。

服务端

基本思路就是

1.接收分块文件基本信息，md5 值，大小，分片数量等

2.根据 md5 值判定是否完整上传过该文件，完整上传过的话，返回结果是否需要覆盖，非完整上传过则根据上传日志开始续传

3.开始接收分块文件，并记录上传日志

4.合并文件，记录日志，上传结束

这里有两种接收形式，一种是接收文件客户端发来的分片文件，还有一种是直接通过 web 网页上传

代码着实乱了一点，我就不贴太多了，最后放上 Gitee 的地址

/// <summary>/// 外部系统上传文件/// </summary>/// <returns></returns>[HttpPost]public IActionResult UploadApi([FromService] FIleLib fi){  var data = Request.Form.Files[0];  string md5 = Request.Form["md5"].ToString();  var total = Request.Form["total"];  var fileName = Request.Form["fileName"];  var index = Request.Form["index"];  var is_rewrite = Request.Form["rewrite"];  string temporary = Path.Combine(_baseUploadDir, md5);//临时保存分块的目录  if (redis.HashExists("upload_list", md5))  {    resp.code = 3;    resp.msg = "已上传";    if (string.IsNullOrEmpty(is_rewrite) || is_rewrite.ToString() == "n")    {      return Json(resp);    }    else if (!string.IsNullOrEmpty(is_rewrite) && is_rewrite.ToString() == "y")    {      if (Directory.Exists(temporary))        Directory.Delete(temporary, true);      redis.HashDelete("upload_list", md5);    }  }  try  {    bool mergeOk = false;    if (!Directory.Exists(temporary))      Directory.CreateDirectory(temporary);
    string filePath = Path.Combine(temporary, index.ToString());    //如果该文件上传过，且没有上传完成，执行如下续传逻辑    //如果只有一个文件,直接删掉，如果大于一个文件，为了保证文件完整性，把最后一个文件删掉，再继续上传；    if (index.ToString() == "0" && System.IO.File.Exists(filePath) && is_rewrite.ToString() != "y")    {      resp.code = 2;      resp.msg = $"{fileName}已上传，开始续传";      var files = Directory.GetFiles(temporary);      if (files.Length > 1)      {        //把标号最大的文件删掉；        string last_file = files.OrderByDescending(u => u.Length).ThenByDescending(u => u).First();        System.IO.File.Delete(last_file);        resp.data = new { index = files.Length - 2, total, mergeOk, fileName };      }      else        System.IO.File.Delete(filePath);      return Json(resp);    }    if (!Convert.IsDBNull(data))    {      using (FileStream fs = new FileStream(filePath, FileMode.Create))      {        data.CopyTo(fs);      }    }    if (total == index)    {      mergeOk = fi.MergeFileForClient(md5, fileName,_baseUploadDir);      if (mergeOk)        redis.HashSet("upload_list", md5, "success");    }    string url = $"/upfile/{DateTime.Now.ToString("yyyyMM")}/{md5}/{fileName}";    resp.code = 1;    resp.msg = $"{index}/{total}上传完成";    resp.data = new { index, total, mergeOk, fileName, url };    return Json(resp);
  }  catch (Exception ex)  {    Directory.Delete(temporary);//删除文件夹    throw ex;  }}

复制代码

验证文件逻辑

/// <summary>/// 验证文件上传情况/// </summary>/// <returns></returns>[HttpPost]public IActionResult CheckFile(){  //前端传来的md5值  var file_name = Request.Form["file_name"];  var md5 = Request.Form["file_md5"];  var maxChunk = Convert.ToInt32(Request.Form["file_total"]);  string logPath = _baseUploadDir + md5 + ".txt";              var logs = LogicLog.ReadLogicLog(logPath);  if (logs != null)  {    int shards = logs.Count();                    if (shards >= Convert.ToInt32(maxChunk))    {      resp.code = 2;      resp.msg = "该文件已上传";      resp.data = new { url = $"upfile/{DateTime.Now.ToString("yyyyMM")}/{md5}/{file_name}" };    }    else    {      resp.code = 0;      resp.msg = $"该文件已上传步长{shards}/{maxChunk}，继续上传";      resp.data = new { file_index = shards - 1, percent = Convert.ToDouble(shards) / Convert.ToDouble(maxChunk) * 100 };    }    //存在日志，文件却找不着了，说明被删了,注意排除掉临时文件夹的情况      string file_path = $"{_baseUploadDir}/{DateTime.Now.ToString("yyyyMM")}/{md5}/{file_name}";    string tmp_path = $"{_baseUploadDir}/{md5}";    if (!System.IO.Directory.Exists(tmp_path) && !System.IO.File.Exists(file_path) && System.IO.File.Exists(logPath))    {      System.IO.File.Delete(logPath);      resp.code = 0;      resp.msg = "该文件已被删除";      return Json(resp);    }  }  return Json(resp);}

复制代码

合并文件的逻辑，分为客户端 api 和 web 两种方式

/// <summary>/// 合并文件/// </summary>/// <param name="md5"></param>/// <param name="fileName"></param>/// <param name="basdir"></param>/// <returns></returns>public bool MergeFileForClient(string md5, string fileName, string basdir){  bool ok = false;  try  {    //临时文件夹    var temporary = Path.Combine(basdir, md5);    //获得下面的所有分片文件    var files = Directory.GetFiles(temporary);    //指定最终存储路径(注意兼容Linux路径写法)    //string finalPath = $"{basdir}\\{DateTime.Now.ToString("yyyyMM")}\\{md5}";    string finalPath = $"{basdir}/{DateTime.Now.ToString("yyyyMM")}/{md5}";    if (!Directory.Exists(finalPath))      Directory.CreateDirectory(finalPath);    //finalPath += $"\\{fileName}";    finalPath += $"/{fileName}";
    using (var fs = new FileStream(finalPath, FileMode.Create))    {      foreach (var part in files.OrderBy(x => x.Length).ThenBy(x => x))//排一下序，保证从0-N Write      {        var bytes = File.ReadAllBytes(part);        fs.Write(bytes, 0, bytes.Length);        bytes = null;        File.Delete(part);//删除分块      }      fs.Flush();      fs.Close();    }    //删除临时文件夹    Directory.Delete(temporary);                ok = true;  }  catch (Exception ex)  {    Console.WriteLine($"文件合并出错，{ex.Message},{ex.StackTrace}");    return false;  }  return ok;}

复制代码

客户端

客户端的核心逻辑就是获取文件 md5 值，构造并提交接收模型，分片提交文件

/// <summary>/// 切片上传/// </summary>/// <param name="filePath"></param>/// <param name="displayProgress"></param>/// <returns></returns>public static async Task<bool> SliceUploadAsync(string filePath, Func<int, int> displayProgress = null){  try  {    string filename = Path.GetFileName(filePath);    string md5 = GetMD5HashFromFile(filePath);    var client = new RestClient(upload_url);    client.Timeout = -1;    var request = new RestRequest(Method.POST);    Console.ForegroundColor = ConsoleColor.Yellow;    Console.WriteLine($"开始上传文件[{filename}]...{DateTime.Now.ToString("yy/MM/dd HH:mm:ss")}");    Console.WriteLine();    using (FileStream fsRead = new FileStream(filePath, FileMode.Open, FileAccess.Read))    {      //文件总长度      long left = fsRead.Length;      //存储读取结果        byte[] bytes = new byte[1024 * 1024];      //每次读取长度        int maxLength = bytes.Length;      int total = Convert.ToInt32(Math.Ceiling(Convert.ToDouble(left / 1024 / 1024)));      int index = 0;      long start = 0;      int num = 0;      request.AddParameter("md5", md5);      request.AddParameter("total", total);      request.AddParameter("fileName", filename);
      while (left > 0)      {        fsRead.Position = start;        num = 0;        if (left < maxLength)          num = await fsRead.ReadAsync(bytes, 0, Convert.ToInt32(left));        else          num = await fsRead.ReadAsync(bytes, 0, maxLength);        if (num == 0)          break;        if (request.Files.Count > 0)          request.Files.RemoveAt(0);        if (request.Parameters.Count(u => u.Name == "index") > 0)          request.Parameters.Remove(request.Parameters.Where(u => u.Name == "index").FirstOrDefault());
        request.AddFile("data", bytes, filename);        request.AddParameter("index", index);        IRestResponse response = await client.ExecuteAsync(request);        if (index == 0)        {          //Console.WriteLine(response.Content);          Response resp = JsonHelper.JsonDeserialize<Response>(response.Content);          if (resp.code == 2)          {            Console.ForegroundColor = ConsoleColor.Cyan;            Console.WriteLine($"\n{resp.message}\n");            index = Convert.ToInt32(resp.data.index) - 1;            if (index < 0)              index = 0;            start = index * maxLength;            left = left - start;            //num = await fsRead.ReadAsync(bytes, 0, maxLength);          }          else if (resp.code == 3)          {            Console.WriteLine($"\n {filename}已上传，确定要覆盖吗? y-覆盖，n-跳过, y/n");            is_rewrite:            string is_rewrite = Console.ReadLine();            if (is_rewrite == "y")            {              //一切从头来              num = 0;              start = 0;              left = fsRead.Length;              index = -1;              if (request.Files.Count > 0)                request.Files.RemoveAt(0);              if (request.Parameters.Count(u => u.Name == "index") > 0)                request.Parameters.Remove(request.Parameters.Where(u => u.Name == "index").FirstOrDefault());              request.AddParameter("rewrite", is_rewrite);            }            else if (is_rewrite == "n")            {              request.AddParameter("rewrite", is_rewrite);              Console.WriteLine("已跳过");              return true;            }            else            {              Console.ForegroundColor = ConsoleColor.Red;              Console.WriteLine("无效参数，请重新输入");              Console.ForegroundColor = ConsoleColor.Yellow;              goto is_rewrite;              //return false;            }          }        }        //增加大于0的条件是，当重写的时候，index会出现-1的情况；        if (displayProgress != null && index > 0) displayProgress(Convert.ToInt32(index / (total * 1.0) * 100));        start += num;        left -= num;        index++;      }    }                
  }  catch(Exception ex)  {    Console.ForegroundColor = ConsoleColor.Red;    Console.WriteLine($"上传失败:{ex.Message},{ex.StackTrace}");    return false;  }  return true;}