golang csv导出多个文件,然后最终合并一个csv文件
package export
import (
"bytes"
"encoding/csv"
"fmt"
"log"
"os"
"os/exec"
"strconv"
"strings"
"testing"
"time"
)
// https://bbs.huaweicloud.com/blogs/286321
var tFmt = "2006-01-02 15:04:05"
// 第二步模拟获取数据，然后批量导出多个文件，然后合并一个文件
func Test_export(t *testing.T) {
content := readFile()
fmt.Printf("%s - read file end......%d\n", time.Now().Format(tFmt), len(content))
exportFile(content)
time.Sleep(time.Millisecond * 200)
}
// 第一步测试数据
func Test_write(t *testing.T) {
f, err := os.Create("test.csv")
if err != nil {
panic(err)
}
defer f.Close()
f.WriteString("\xEF\xBB\xBF") // 写入UTF-8 BOM
w := csv.NewWriter(f)
str := []string{"xxx", "ewewe", "werwe", ""}
for i := 1; i <= 100*10000; i++ {
str[3] = strconv.Itoa(i)
w.Write(str)
}
w.Flush()
}
func readFile() [][]string {
//准备读取文件
fileName := "test.csv"
fmt.Printf("%s - read file......\n", time.Now().Format(tFmt))
fs1, _ := os.Open(fileName)
r1 := csv.NewReader(fs1)
content, err := r1.ReadAll()
if err != nil {
log.Fatalf("can not readall, err is %+v", err)
}
return content
}
func exportFile(content [][]string) {
fmt.Printf("%s - start export multiple file......\n", time.Now().Format(tFmt))
length := len(content)
// 2. 并发循环导出，一次200,000 (20万数据) 到 1个文件，最终输出n个10万数据的文件
count := 1
pos := 0
fileNameArr := make([]string, 0)
for pos < length {
maxPos := pos + 20*10000
if maxPos > length {
maxPos = length
}
temp := content[pos:maxPos]
go wiriteTempFile(pos, "temp"+strconv.Itoa(count)+".csv", temp)
fileNameArr = append(fileNameArr, "temp"+strconv.Itoa(count)+".csv")
count++
pos += 20 * 10000
}
// 3. 合并n个文件到1个文件
var outBytes bytes.Buffer
fileNameStr := strings.Join(fileNameArr, " ")
cmdStr := fmt.Sprintf("cat %s > testall.csv", fileNameStr)
cmd := exec.Command("sh", "-c", cmdStr)
cmd.Stdout = &outBytes
err := cmd.Run()
if err != nil {
log.Println(err)
return
}
resStr := outBytes.String()
log.Println(resStr, cmdStr)
fmt.Printf("%s - export file success......\n", time.Now().Format(tFmt))
}
func wiriteTempFile(pos int, file string, data [][]string) {
fmt.Printf("len === %d , pos=%d \n", len(data), pos)
f, err := os.Create(file)
if err != nil {
panic(err)
}
defer f.Close()
f.WriteString("\xEF\xBB\xBF") // 写入UTF-8 BOM
w := csv.NewWriter(f)
w.WriteAll(data)
w.Flush()
}
// 创建文件夹
func MakeDir(dir string) error {
if !FileIsExisted(dir) {
if err := os.MkdirAll(dir, 0777); err != nil { //os.ModePerm
fmt.Println("MakeDir failed:", err)
return err
}
}
return nil
}
// 检测文件是否存在
func FileIsExisted(filename string) bool {
existed := true
if _, err := os.Stat(filename); os.IsNotExist(err) {
existed = false
}
return existed
}