GoLang 读取CSV文件

使用GoLang读取CSV文件

CSV文件

  • 注意 CSV文件前4行只有第3行为字段,其余行要跳过(具体根据项目来选择)
"商品ID","商品名称","商品支付购买类型","商品价格","商品简介","获得的物品ID","获得物品数量"
"int","string","int","int","string","int","int"
"Id","Name","Type","Price","Info","GetItemID","GetItemCount"
"商品ID","商品名称","商品支付购买类型,0金币,1钻石,2充值","商品价格","商品简介","获得的物品ID(若值为负数说明为非物品)","获得的物品数量"
3001,"金币商品1",0,1000,"这是xxx商品",400001,1
3002,"金币商品2",0,2000,"这是xxx商品",400002,1
3003,"金币商品3",0,3000,"这是xxx商品",400003,1
3004,"金币商品4",0,4000,"这是xxx商品",400004,1
3005,"金币商品5",0,5000,"这是xxx商品",400005,1
3006,"金币商品6",0,6000,"这是xxx商品",400006,1
3007,"金币商品7",0,7000,"这是xxx商品",400007,1
3008,"金币商品8",0,8000,"这是xxx商品",400008,1
3009,"金币商品9",0,9000,"这是xxx商品",400009,1
3010,"金币商品10",0,10000,"这是xxx商品",400010,1
3011,"金币商品11",0,11000,"这是xxx商品",400011,1
3012,"金币商品12",0,12000,"这是xxx商品",400012,1
3013,"金币商品13",0,13000,"这是xxx商品",400013,1
3014,"金币商品14",0,14000,"这是xxx商品",400014,1
3015,"金币商品15",0,15000,"这是xxx商品",400015,1
3016,"一小袋金币",1,10,"这是xxx商品",-1,10000
3017,"一大袋金币",1,20,"这是xxx商品",-1,20000
3018,"一小框金币",1,30,"这是xxx商品",-1,30000
3019,"一大筐金币",1,40,"这是xxx商品",-1,40000
3020,"一小桶金币",1,50,"这是xxx商品",-1,50000
3021,"一大桶金币",1,60,"这是xxx商品",-1,60000
3022,"一小箱金币",1,70,"这是xxx商品",-1,70000
3023,"一大箱金币",1,80,"这是xxx商品",-1,80000
3024,"大经验药水*10",1,100,"这是xxx商品",400016,10
3025,"大经验药水*100",1,900,"这是xxx商品",400016,100
3026,"一颗钻石",2,1,"这是xxx商品",-1,1
3027,"一小把钻石",2,10,"这是xxx商品",-1,100
3028,"一小筐钻石",2,100,"这是xxx商品",-1,1000
3029,"一大桶钻石",2,328,"这是xxx商品",-1,3280
3030,"一大箱钻石",2,648,"这是xxx商品",-1,6480

GOLang读取

package util

import (
	"bufio"
	"encoding/csv"
	"fmt"
	"io"
	"os"
	"reflect"
	"strconv"
	"strings"
	"unicode"
)

type CsvUtilMgr struct {
}

var csvUtilMgr *CsvUtilMgr = nil

func GetCsvUtilMgr() *CsvUtilMgr {
	if csvUtilMgr == nil {
		csvUtilMgr = new(CsvUtilMgr)
	}
	return csvUtilMgr
}

func (self *CsvUtilMgr) getFieldMap(config interface{}) map[string][]string {
	t := reflect.TypeOf(config)
	fieldMap := make(map[string][]string, t.NumField())
	for i := 0; i < t.NumField(); i++ {
		field := t.Field(i)
		tag := field.Tag.Get("json")
		data := make([]string, 2, 2)
		data[0] = field.Name
		data[1] = fmt.Sprintf("%v", field.Type)
		fieldMap[tag] = data
	}
	return fieldMap
}

func (self *CsvUtilMgr) readCsv(fileName string) [][]string {
	csvReadFile, err := os.Open(fileName)
	defer csvReadFile.Close()
	if err != nil {
		fmt.Println(err.Error())
		return [][]string{}
	}

	r := csv.NewReader(bufio.NewReader(csvReadFile))
	var records [][]string
	var recordIndex int
	var myIndex int
	for {
		record, err := r.Read()
		if err == io.EOF {
			break
		}
		if myIndex == 2 || myIndex > 3 {
			// remove bom head
			if recordIndex == 0 {
				for index := range record {
					if index == 0 && strings.Contains(record[index], "\ufeff") {
						record[index] = strings.Replace(record[index], "\ufeff", "", 1)
						break
					}
				}
				recordIndex += 1
			}
			records = append(records, record)
		}
		myIndex++
	}
	return records
}

func (self *CsvUtilMgr) getTagMap(fields []string) map[int]string {
	tagMap := make(map[int]string, len(fields))
	for index, v := range fields {
		tagMap[index] = v
	}

	return tagMap
}

func (self *CsvUtilMgr) getValueType(SlicePtr interface{}) reflect.Type {
	value := reflect.ValueOf(SlicePtr)
	if value.Kind() == reflect.Ptr {
		// 返回指针指向的值,这里是slice
		value = value.Elem()
	}

	// 返回当前值对应的类型
	outType := value.Type()
	// 返回这些类型包含的一个值
	outInnerType := outType.Elem()

	return outInnerType
}

// 指针只能取interface{}, 值得取Elem,否则会蹦
// tagMap: csv第一行
// filedMap: key: tag
func (self *CsvUtilMgr) genConfig(dataPtr interface{}, csvData [][]string, tagMap map[int]string,
	fieldMap map[string][]string, trimFlag map[int]bool, fileName string, keyTag string) (err error) {
	dataVal := reflect.Indirect(reflect.ValueOf(dataPtr))
	outInnerType := self.getValueType(dataPtr)

	//if fileName == "expspeedup" {
	//litter.Dump(tagMap)
	//litter.Dump(fieldMap)
	//}

	for r := 1; r < len(csvData); r++ {
		data := reflect.New(outInnerType.Elem())

		key := 0
		for c := 0; c < len(csvData[r]); c++ {
			tag := tagMap[c]
			if _, ok := trimFlag[c]; ok {
				tag = self.trimNumber(tag)
			}

			fieldInfo, ok := fieldMap[tag]
			if !ok {
				continue
			}

			if len(fieldInfo) != 2 {
				continue
			}
			fieldName := fieldInfo[0]
			filedType := fieldInfo[1]
			cellValue := csvData[r][c]
			switch filedType {
			case "int":
				v, err := strconv.Atoi(cellValue)
				if err != nil {
					fmt.Println(err.Error(), ", fileName:"+fileName, ", fieldName:", fieldName, ", index:", r)
					break
				}
				reflect.Indirect(data).FieldByName(fieldName).SetInt(int64(v))
				if tag == keyTag && key == 0 {
					key = v
					//fmt.Println("fileName:", fileName, ", tag:", tag)
				}
			case "int64":
				v, err := strconv.ParseInt(cellValue, 10, 64)
				if err != nil {
					fmt.Println(err.Error(), ", fileName:"+fileName, ", fieldName:", fieldName)
					break
				}
				reflect.Indirect(data).FieldByName(fieldName).SetInt(v)
			case "string":
				reflect.Indirect(data).FieldByName(fieldName).SetString(cellValue)
			case "[]int":
				v, err := strconv.Atoi(cellValue)
				if err != nil {
					fv, err := strconv.ParseFloat(cellValue, 64)
					if err != nil {
						fmt.Println(err.Error(), ", fileName:"+fileName, ", fieldName:", fieldName, ", row:", r, ", col:", c, ", ", strings.Join(csvData[r], ","))
					} else {
						c := reflect.Indirect(data).FieldByName(fieldName)
						newSlice := reflect.Append(c, reflect.ValueOf(int(fv*10)))
						reflect.Indirect(data).FieldByName(fieldName).Set(newSlice)
					}
					break
				} else {
					c := reflect.Indirect(data).FieldByName(fieldName)
					newSlice := reflect.Append(c, reflect.ValueOf(v))
					reflect.Indirect(data).FieldByName(fieldName).Set(newSlice)
				}
			case "[]int64":
				v, err := strconv.ParseInt(cellValue, 10, 0)
				if err != nil {
					fmt.Println(err.Error(), ", fileName:"+fileName)
					break
				}
				c := reflect.Indirect(data).FieldByName(fieldName)
				newSlice := reflect.Append(c, reflect.ValueOf(v))
				reflect.Indirect(data).FieldByName(fieldName).Set(newSlice)
			case "[]string":
				c := reflect.Indirect(data).FieldByName(fieldName)
				newSlice := reflect.Append(c, reflect.ValueOf(cellValue))
				reflect.Indirect(data).FieldByName(fieldName).Set(newSlice)
			case "float32":
				fv, err := strconv.ParseFloat(cellValue, 32)
				if err != nil {
					fmt.Println(err.Error(), ", fileName:"+fileName, ", fieldName:", fieldName, ", row:", r, ", col:", c, ", ", strings.Join(csvData[r], ","))
				} else {
					reflect.Indirect(data).FieldByName(fieldName).SetFloat(fv)
				}
			case "float64":
				fv, err := strconv.ParseFloat(cellValue, 64)
				if err != nil {
					fmt.Println(err.Error(), ", fileName:"+fileName, ", fieldName:", fieldName, ", row:", r, ", col:", c, ", ", strings.Join(csvData[r], ","))
				} else {
					reflect.Indirect(data).FieldByName(fieldName).SetFloat(fv)
				}
			case "[]float32":
				fv, err := strconv.ParseFloat(cellValue, 32)
				if err != nil {
					fmt.Println(err.Error(), ", fileName:"+fileName, ", fieldName:", fieldName, ", row:", r, ", col:", c, ", ", strings.Join(csvData[r], ","))
				} else {
					c := reflect.Indirect(data).FieldByName(fieldName)
					newSlice := reflect.Append(c, reflect.ValueOf(float32(fv)))
					reflect.Indirect(data).FieldByName(fieldName).Set(newSlice)
				}
			case "[]float64":
				fv, err := strconv.ParseFloat(cellValue, 64)
				if err != nil {
					fmt.Println(err.Error(), ", fileName:"+fileName, ", fieldName:", fieldName, ", row:", r, ", col:", c, ", ", strings.Join(csvData[r], ","))
				} else {
					c := reflect.Indirect(data).FieldByName(fieldName)
					newSlice := reflect.Append(c, reflect.ValueOf(fv))
					reflect.Indirect(data).FieldByName(fieldName).Set(newSlice)
				}
			}
		}

		kind := reflect.TypeOf(dataVal.Interface()).Kind()
		if kind == reflect.Slice {
			dataVal.Set(reflect.Append(dataVal, data))
		} else if kind == reflect.Map {
			dataVal.SetMapIndex(reflect.ValueOf(key), data)
		}
	}

	return nil
}

func (self *CsvUtilMgr) trimNumber(s string) string {
	subString := strings.TrimFunc(s, func(r rune) bool {
		return unicode.IsNumber(r)
	})

	return subString

}

func (self *CsvUtilMgr) LoadCsv(fileName string, SlicePtr interface{}) {
	csvFile := "csv/tableCsv/" + fileName + ".csv"
	csvData := self.readCsv(csvFile)
	if len(csvData) <= 1 {
		fmt.Println("len(csvData) <= 1, filename:", fileName)
		os.Exit(1)
		return
	}

	//LogDebug("Read File:", fileName, len(csvData))
	err := self.ParseDataSimple(csvData, SlicePtr, fileName)
	if err != nil {
		fmt.Println(err.Error())
		return
	}
}

func (self *CsvUtilMgr) LoadEventsCsv(fileName string, SlicePtr interface{}) {
	csvFile := "csv/ChapterMap/" + fileName + ".csv"
	csvData := self.readCsv(csvFile)
	if len(csvData) <= 1 {
		fmt.Println("len(csvData/ChapterMap) <= 1, fileName:", fileName)
		os.Exit(1)
		return
	}

	err := self.ParseDataSimple(csvData, SlicePtr, fileName)
	if err != nil {
		return
	}
}

func (self *CsvUtilMgr) getFieldMapSimple(config interface{}, tagMap map[int]string) (map[string][]string, map[int]bool, string) {
	t := reflect.TypeOf(config)
	fieldMap := make(map[string][]string, t.NumField())
	trimFlag := make(map[int]bool)
	keyTag := ""
	for i := 0; i < t.NumField(); i++ {
		field := t.Field(i)
		tag := field.Tag.Get("json")
		trim := field.Tag.Get("trim")

		if trim == "1" || trim == "" {
			tag = self.trimNumber(tag)
		}

		for key, v := range tagMap {
			if (trim == "" || trim == "1") && tag == self.trimNumber(v) {
				trimFlag[key] = true
			}
		}

		// 设置表数据结构
		for key, v := range tagMap {
			if (trim == "" || trim == "1") && tag == self.trimNumber(v) {
				tagMap[key] = tag
				break
			}
		}

		data := make([]string, 2, 2)
		data[0] = field.Name
		data[1] = fmt.Sprintf("%v", field.Type)
		fieldMap[tag] = data
		if i == 0 {
			keyTag = tag
		}
	}
	return fieldMap, trimFlag, keyTag
}

func (self *CsvUtilMgr) ParseDataSimple(csvData [][]string, dataPtr interface{}, fileName string) (err error) {
	outInnerType := self.getValueType(dataPtr)
	data := reflect.New(outInnerType.Elem())

	value := reflect.Indirect(data)
	tagMap := self.getTagMap(csvData[0])
	fieldMap, trimFlag, keyTag := self.getFieldMapSimple(value.Interface(), tagMap)
	err = self.genConfig(dataPtr, csvData, tagMap, fieldMap, trimFlag, fileName, keyTag)
	return
}

测试

  • 注意:使用Json标记需要和CSV的字段相同
package csv

import (
	"MyGameServer/logger"
	"MyGameServer/util"
)

type ConfigShop struct {
	ID           int `json:"Id"`
	Type         int `json:"Type"`
	Price        int `json:"Price"`
	GetItemID    int `json:"GetItemID"`
	GetItemCount int `json:"GetItemCount"`
}

var configShopMapByShopID map[int]*ConfigShop

func initConfigShop() {
	configShopMapByShopID = make(map[int]*ConfigShop)
	util.GetCsvUtilMgr().LoadCsv("ShopTable", &configShopMapByShopID)
	logger.PopDebug("csv_shop初始化!")
}

func GetShopConfigByShopID(shopID int) *ConfigShop {
	return configShopMapByShopID[shopID]
}