golang 获取文件摘要代码优化 | Go 技术论坛

ioutil.ReadAllio.Copybufio.NewReader

golang 读取文件性能对比

ioutil.ReadAll

func ReadAll(path string) (fileMD5 string, err error) {
    f, err := os.Open(path)
    if err != nil {
        return fileMD5, err
    }
    defer f.Close()

    body, err := ioutil.ReadAll(f)
    if err != nil {
        return fileMD5, err
    }
    hash := sha1.New()
    hash.Write(body)
    fileMD5 = hex.EncodeToString(hash.Sum(nil))
    return fileMD5, nil
}

io.copy

func Copy(path string) (fileMD5 string, err error) {
    f, err := os.Open(path)
    if err != nil {
        return fileMD5, err
    }
    defer f.Close()

    md5hash := sha1.New()
    if _, err := io.Copy(md5hash, f); err != nil {
        return fileMD5, err
    }

    fileMD5 = hex.EncodeToString(md5hash.Sum(nil))
    return fileMD5, nil
}

bufio.NewReader

func ReadBuf(path string) (fileMD5 string, err error) {
    f, err := os.Open(path)
    if err != nil {
        return fileMD5, err
    }
    defer f.Close()

    buf := make([]byte, 1024)
    reader := bufio.NewReader(f)
    md5hash := sha1.New()
    for {
        n, err := reader.Read(buf)
        if err != nil { // 遇到任何错误立即返回，并忽略 EOF 错误信息
            if err == io.EOF {
                goto stop
            }
            return fileMD5, err
        }
        md5hash.Write(buf[:n])
    }
stop:
    fileMD5 = hex.EncodeToString(md5hash.Sum(nil))
    return fileMD5, nil
}

单元测试

package file_test

import (
    "testing"

    "filestore/test/file"
)

var (
    minPath = "~/6d827d1edddea7c73fb7d6efbb467167839ff2f6.jpg"
    maxPath = "/Users/zhanggaoyuan/学习/2004.mkv"
)

func BenchmarkReadAll(b *testing.B) {
    for i := 0; i < b.N; i++ {
        _, err := file.ReadAll(minPath)
        if err != nil {
            b.Error(err)
            return
        }
    }
}

func BenchmarkCopy(b *testing.B) {
    for i := 0; i < b.N; i++ {
        _, err := file.Copy(minPath)
        if err != nil {
            b.Error(err)
            return
        }
    }
}

func BenchmarkReadBuf(b *testing.B) {
    for i := 0; i < b.N; i++ {
        _, err := file.ReadBuf(minPath)
        if err != nil {
            b.Error(err)
            return
        }
    }
}

文件大小耗时情况

10MB

准备 6d827d1edddea7c73fb7d6efbb467167839ff2f6.jpg 9.7M 的文件，

go test -bench=. -benchmem

goos: darwin
goarch: amd64
pkg: filestore/test/file
BenchmarkReadAll-8             64      17963784 ns/op    33552812 B/op          22 allocs/op
BenchmarkCopy-8                80      12910267 ns/op       33192 B/op           8 allocs/op
BenchmarkReadBuf-8             66      15641888 ns/op        5544 B/op           9 allocs/op
PASS
ok      filestore/test/file    3.994s

BenchmarkCopy

40MB

准备文件 niushop_b2c_mf2.3.zip 37M

go test -bench=. -benchmem

goos: darwin
goarch: amd64
pkg: filestore/test/file
BenchmarkReadAll-8             16      63298080 ns/op    134216088 B/op          24 allocs/op
BenchmarkCopy-8                24      48085580 ns/op       33176 B/op           8 allocs/op
BenchmarkReadBuf-8             19      58120592 ns/op        5528 B/op           9 allocs/op
PASS
ok      filestore/test/file    4.206s

BenchmarkCopy

1G6G

ioutil.ReadAllio.copy

goos: darwin
goarch: amd64
pkg: filestore/test/file
BenchmarkCopy-8                 1    8612443012 ns/op       33176 B/op          10 allocs/op
BenchmarkReadBuf-8              1    8924705447 ns/op        5512 B/op           9 allocs/op
PASS
ok      filestore/test/file    17.554s

BenchmarkCopyio.copy

md5sha1

我们还是用不用的文件做比较，因为有时候很多函数的时间和空间复杂度和大小，数量有关系

10MB

sha1

goos: darwin
goarch: amd64
pkg: filestore/test/file
BenchmarkReadAll-8             64      17963784 ns/op    33552812 B/op          22 allocs/op
BenchmarkCopy-8                80      12910267 ns/op       33192 B/op           8 allocs/op
BenchmarkReadBuf-8             66      15641888 ns/op        5544 B/op           9 allocs/op
PASS
ok      filestore/test/file    3.994s

md5

goos: darwin
goarch: amd64
pkg: filestore/test/file
BenchmarkReadAll-8             56      20431688 ns/op    33552750 B/op          22 allocs/op
BenchmarkCopy-8                70      16554352 ns/op       33128 B/op           8 allocs/op
BenchmarkReadBuf-8             57      19220965 ns/op        5480 B/op           9 allocs/op
PASS
ok      filestore/test/file    4.202s

40MB

sha1

goos: darwin
goarch: amd64
pkg: filestore/test/file
BenchmarkReadAll-8             16      63298080 ns/op    134216088 B/op          24 allocs/op
BenchmarkCopy-8                24      48085580 ns/op       33176 B/op           8 allocs/op
BenchmarkReadBuf-8             19      58120592 ns/op        5528 B/op           9 allocs/op
PASS
ok      filestore/test/file    4.206s

md5

goos: darwin
goarch: amd64
pkg: filestore/test/file
BenchmarkReadAll-8             14      75987790 ns/op    134216024 B/op          24 allocs/op
BenchmarkCopy-8                19      61600369 ns/op       33112 B/op           8 allocs/op
BenchmarkReadBuf-8             15      72338837 ns/op        5464 B/op           9 allocs/op
PASS
ok      filestore/test/file    4.339s

1G6G

sha1

goos: darwin
goarch: amd64
pkg: filestore/test/file
BenchmarkCopy-8                 1    8612443012 ns/op       33176 B/op          10 allocs/op
BenchmarkReadBuf-8              1    8924705447 ns/op        5512 B/op           9 allocs/op
PASS
ok      filestore/test/file    17.554s

md5

goos: darwin
goarch: amd64
pkg: filestore/test/file
BenchmarkCopy-8                 1    8714197367 ns/op       33112 B/op          10 allocs/op
BenchmarkReadBuf-8              1    10077543682 ns/op        5448 B/op           9 allocs/op
PASS
ok      filestore/test/file    18.804s

sha1md5sha1io.copysha1q

原文标题：[golang 获取文件摘要算法优化]

本站使用「 署名-非商业性使用 4.0 国际 (CC BY-NC 4.0)」创作共享协议，转载或使用请署名并注明出处。

本作品采用《CC 协议》，转载必须注明作者和本文链接