golang中处理多个文件同时上传以及文件上传速度优化

首先给大家推荐一本书Go in Practice，通过一个个超级小巧而又非常实战的案例提升golang内功。
在TECHNIQUE 48 Incrementally saving a file中，向读者讲述了如何处理多个文件同时提交的处理方法
以下代码和文中略有不同：

func uploadBigFile(w http.ResponseWriter, r *http.Request) {
    mr, err := r.MultipartReader()
    if err != nil {
        fmt.Sprintln(err)
        fmt.Fprintln(w, err)

        return
    }

    values := make(map[string][]string, 0)
    maxValueBytes := int64(10 << 20)
    for {
        part, err := mr.NextPart()
        if err == io.EOF {
            break
        }

        name := part.FormName()
        if name == "" {
            continue
        }

        fileName := part.FileName()

        var b bytes.Buffer

        if fileName == "" {
            n, err := io.CopyN(&b, part, maxValueBytes)
            if err != nil && err != io.EOF {
                fmt.Sprintln(err)
                fmt.Fprintln(w, err)

                return
            }

            maxValueBytes -= n
            if maxValueBytes <= 0 {
                msg := "multipart message too large"
                fmt.Fprint(w, msg)
                return
            }

            values[name] = append(values[name], b.String())
        }

        dst, err := os.Create("/tmp/upload/" + fileName)
        defer dst.Close()

        for {
            buffer := make([]byte, 100000)
            cBytes, err := part.Read(buffer)
            if err == io.EOF {
                break
            }
            dst.Write(buffer[0:cBytes])
        }

    }
}

在进行benchmark测试的时候，发现上述方式在处理单个文件上传的时候要比普通的处理方法要快
普通的文件上传处理接口

func upload(w http.ResponseWriter, r *http.Request) {
    file, head, err := r.FormFile("my_file")
    if err != nil {
        fmt.Sprintln(err)
        fmt.Fprintln(w, err)

        return
    }

    localFileDir := "/tmp/upload/"
    err = os.MkdirAll(localFileDir, 0777)
    if err != nil {
        fmt.Sprintln(err)
        fmt.Fprintln(w, err)

        return
    }

    localFilePath := localFileDir + head.Filename

    localFile, err := os.Create(localFilePath)
    if err != nil {
        fmt.Sprintln(err)
        fmt.Fprintln(w, err)

        return
    }
    defer localFile.Close()

    io.Copy(localFile, file)
    fmt.Fprintln(w, localFilePath)

}

基准测试代码

func BenchmarkUpload(b *testing.B) {

    for i := 0; i < b.N; i++ {
        path := "/home/kes/code_1.13.1-1497464373_amd64.deb"
        file, err := os.Open(path)
        if err != nil {
            b.Error(err)
        }

        defer file.Close()
        body := &bytes.Buffer{}
        writer := multipart.NewWriter(body)
        part, err := writer.CreateFormFile("my_file", filepath.Base(path))
        if err != nil {
            b.Error(err)
        }
        io.Copy(part, file)
        writer.Close()

        req := httptest.NewRequest("POST", "/upload", body)
        req.Header.Set("Content-Type", writer.FormDataContentType())
        res := httptest.NewRecorder()

        upload(res, req)

        if res.Code != http.StatusOK {
            b.Error("not 200")
        }
    }

    // t.Log(res.Body.String())
    // t.Log(io.read)

}

func BenchmarkUploadBig(b *testing.B) {

    for i := 0; i < b.N; i++ {
        path := "/home/kes/code_1.13.1-1497464373_amd64.deb"
        file, err := os.Open(path)
        if err != nil {
            b.Error(err)
        }

        defer file.Close()
        body := &bytes.Buffer{}
        writer := multipart.NewWriter(body)
        part, err := writer.CreateFormFile("my_file", filepath.Base(path))
        if err != nil {
            b.Error(err)
        }
        io.Copy(part, file)
        writer.Close()

        req := httptest.NewRequest("POST", "/upload", body)
        req.Header.Set("Content-Type", writer.FormDataContentType())
        res := httptest.NewRecorder()

        uploadBigFile(res, req)

        if res.Code != http.StatusOK {
            b.Error("not 200")
        }
    }

    // t.Log(res.Body.String())
    // t.Log(io.read)

}

测试的文件大小大概44M
测试结果

$ go test -bench="." -v
BenchmarkUpload-4              1        1074491528 ns/op
BenchmarkUploadBig-4           2         517001745 ns/op
PASS
ok      _/home/kes/test    2.966s

$ go test -bench="." -v
BenchmarkUpload-4              1        1295672144 ns/op
BenchmarkUploadBig-4           2         537212487 ns/op
PASS
ok      _/home/kes/test    3.218s