From 9ff6e2d690c2e30b3ff54c6fcf836dd2c7d32863 Mon Sep 17 00:00:00 2001 From: xlc Date: Thu, 16 May 2024 13:33:53 +0800 Subject: [PATCH] fix multipart upload merge file --- .../internal/library/storager/upload_local.go | 8 ++- .../library/storager/upload_local_test.go | 61 +++++++++++++++++++ 2 files changed, 68 insertions(+), 1 deletion(-) create mode 100644 server/internal/library/storager/upload_local_test.go diff --git a/server/internal/library/storager/upload_local.go b/server/internal/library/storager/upload_local.go index 97bca90..3f7fa93 100644 --- a/server/internal/library/storager/upload_local.go +++ b/server/internal/library/storager/upload_local.go @@ -15,6 +15,8 @@ import ( "github.com/gogf/gf/v2/util/gconv" "os" "path/filepath" + "sort" + "strconv" "strings" ) @@ -139,7 +141,11 @@ func MergePartFile(srcPath, dstPath string) (err error) { if err != nil { return err } - + sort.Slice(dir, func(i, j int) bool { + fiIndex, _ := strconv.Atoi(dir[i].Name()) + fjIndex, _ := strconv.Atoi(dir[j].Name()) + return fiIndex < fjIndex + }) for _, file := range dir { filePath := filepath.Join(srcPath, file.Name()) if err = gfile.PutBytesAppend(dstPath, gfile.GetBytes(filePath)); err != nil { diff --git a/server/internal/library/storager/upload_local_test.go b/server/internal/library/storager/upload_local_test.go new file mode 100644 index 0000000..8dea7bf --- /dev/null +++ b/server/internal/library/storager/upload_local_test.go @@ -0,0 +1,61 @@ +package storager + +import ( + "crypto/md5" + "errors" + "fmt" + "io" + "os" + "testing" +) + +func TestMergePartFile(t *testing.T) { + srcFile, _ := os.Open("1.zip") + defer srcFile.Close() + fileInfo, err := srcFile.Stat() + if err != nil { + t.Error(err) + } + hash := md5.New() + _, err = io.Copy(hash, srcFile) + sum := hash.Sum(nil) + fileHash := fmt.Sprintf("%x", sum) + + var singleSize = int64(1024 * 1024 * 100) + chunks := int(fileInfo.Size() / singleSize) + if fileInfo.Size()%singleSize != 0 { + chunks += 1 + } + srcFile.Seek(0, io.SeekStart) + for j := 0; j < chunks; j++ { + partSize := singleSize + if j == chunks-1 { + partSize = fileInfo.Size() - int64(j)*singleSize + } + partData := make([]byte, partSize) + _, err = io.ReadFull(srcFile, partData) + pf, _ := os.Create(fmt.Sprintf("tmp/%d", j+1)) + _, err = pf.Write(partData) + if err != nil { + t.Error(err) + return + } + pf.Close() + } + + err = MergePartFile("tmp/", "2.zip") + if err != nil { + t.Error(err) + return + } + + f2, _ := os.Open("2.zip") + hash2 := md5.New() + _, err = io.Copy(hash2, f2) + sum2 := hash.Sum(nil) + fileHash2 := fmt.Sprintf("%x", sum2) + if fileHash != fileHash2 { + t.Error(errors.New("hash mismatch")) + } + +}