由于自己的服务器配置比较差,上传一篇较长博客后,进存kill,显示内存溢出。每次重启后不久,都会进存kill掉的现象出现。 问题 上传了一篇比较长的文章后,goblog后台直接被killed掉,查看日志,内存溢出:
2019/11/15 15:38:04.729 [I] [router.go:266] /root/Project/src/goblog/src/controller/admin no changed
2019/11/15 15:38:04.729 [I] [router.go:266] /root/Project/src/goblog/src/controller no changed
[goblog] 2019/11/15 15:38:04.733401 user_mapper.go:57 [INFO] call GetByCondition rows:1
2019/11/15 15:38:05.109 [I] [asm_amd64.s:1333] http server Running on http://:9090
2019/11/15 15:38:05.127 [I] [asm_amd64.s:1333] Admin server Running on 127.0.0.1:8088
2019/11/15 15:38:08 Gse dictionary loaded finished.
fatal error: runtime: out of memory
runtime stack:
runtime.throw(0xbeb588, 0x16)
/root/go/src/runtime/panic.go:608 +0x72
runtime.sysMap(0xc050000000, 0x4000000, 0x138fb58)
/root/go/src/runtime/mem_linux.go:156 +0xc7
runtime.(*mheap).sysAlloc(0x1376320, 0x4000000, 0x1376338, 0x7f4c067ddb48)
/root/go/src/runtime/malloc.go:619 +0x1c7
runtime.(*mheap).grow(0x1376320, 0x2, 0x0)
/root/go/src/runtime/mheap.go:920 +0x42
runtime.(*mheap).allocSpanLocked(0x1376320, 0x2, 0x138fb68, 0x800)
/root/go/src/runtime/mheap.go:848 +0x337
runtime.(*mheap).alloc_m(0x1376320, 0x2, 0x75, 0x7f4c069e5fff)
/root/go/src/runtime/mheap.go:692 +0x119
runtime.(*mheap).alloc.func1()
/root/go/src/runtime/mheap.go:759 +0x4c
runtime.(*mheap).alloc(0x1376320, 0x2, 0x7f4c06010075, 0x7f4c067ddab0)
/root/go/src/runtime/mheap.go:758 +0x8a
runtime.(*mcentral).grow(0x1379398, 0x0)
/root/go/src/runtime/mcentral.go:232 +0x94
runtime.(*mcentral).cacheSpan(0x1379398, 0x7f4c067ddab0)
/root/go/src/runtime/mcentral.go:106 +0x2f8
runtime.(*mcache).refill(0x7f4c1358d000, 0x7f4c0cbd1875)
/root/go/src/runtime/mcache.go:122 +0x95
runtime.(*mcache).nextFree.func1()
/root/go/src/runtime/malloc.go:749 +0x32
runtime.systemstack(0x45a029)
/root/go/src/runtime/asm_amd64.s:351 +0x66
runtime.mstart()
/root/go/src/runtime/proc.go:1229
goroutine 65 [running]:
runtime.systemstack_switch()
/root/go/src/runtime/asm_amd64.s:311 fp=0xc01af6b400 sp=0xc01af6b3f8 pc=0x45a120
runtime.(*mcache).nextFree(0x7f4c1358d000, 0x203075, 0xc04fffa000, 0x7f4c067ddab0, 0xc01af6b401)
/root/go/src/runtime/malloc.go:748 +0xb6 fp=0xc01af6b458 sp=0xc01af6b400 pc=0x40d506
runtime.mallocgc(0x4000, 0x0, 0x7f4c0ced8b00, 0xc04fffa000)
/root/go/src/runtime/malloc.go:903 +0x793 fp=0xc01af6b4f8 sp=0xc01af6b458 pc=0x40de53
runtime.rawstring(0x3fca, 0x0, 0x0, 0x0, 0x0, 0x0)
/root/go/src/runtime/string.go:258 +0x4f fp=0xc01af6b528 sp=0xc01af6b4f8 pc=0x4491df
runtime.rawstringtmp(0x0, 0x3fca, 0xc005a5b4f0, 0x0, 0x1370e00, 0x3fc9, 0x7f4c1358d000)
/root/go/src/runtime/string.go:123 +0x72 fp=0xc01af6b568 sp=0xc01af6b528 pc=0x448bb2
runtime.concatstrings(0x0, 0xc01af6b648, 0x2, 0x2, 0x0, 0xc04ab3a830)
/root/go/src/runtime/string.go:49 +0xfd fp=0xc01af6b600 sp=0xc01af6b568 pc=0x44868d
runtime.concatstring2(0x0, 0xc04fffa000, 0x3fc9, 0xc045f674c9, 0x1, 0xc005a5b400, 0x0)
/root/go/src/runtime/string.go:58 +0x47 fp=0xc01af6b640 sp=0xc01af6b600 pc=0x4488b7
github.com/go-ego/riot.(*Engine).ForSplitData(0x1370740, 0xc048a70000, 0x52cf, 0x52cf, 0x52cf, 0x52cf, 0x52cf)
/root/Project/pkg/mod/github.com/go-ego/riot@v0.0.0-20181222142112-75a2cab10779/segment.go:53 +0x141 fp=0xc01af6b710 sp=0xc01af6b640 pc=0x9e4671
github.com/go-ego/riot.(*Engine).splitData(0x1370740, 0x12cf0d9, 0x1, 0x2ab8b572, 0xc045f48a80, 0x6321, 0x0, 0x0, 0x0, 0x0, ...)
/root/Project/pkg/mod/github.com/go-ego/riot@v0.0.0-20181222142112-75a2cab10779/segment.go:108 +0x32d fp=0xc01af6b8e8 sp=0xc01af6b710 pc=0x9e4e0d
github.com/go-ego/riot.(*Engine).segmenterData(0x1370740, 0x12cf0d9, 0x1, 0x2ab8b572, 0xc045f48a80, 0x6321, 0x0, 0x0, 0x0, 0x0, ...)
/root/Project/pkg/mod/github.com/go-ego/riot@v0.0.0-20181222142112-75a2cab10779/segment.go:186 +0x2d2 fp=0xc01af6bb00 sp=0xc01af6b8e8 pc=0x9e54d2
github.com/go-ego/riot.(*Engine).makeTokensMap(0x1370740, 0x12cf0d9, 0x1, 0x2ab8b572, 0xc045f48a80, 0x6321, 0x0, 0x0, 0x0, 0x0, ...)
/root/Project/pkg/mod/github.com/go-ego/riot@v0.0.0-20181222142112-75a2cab10779/segment.go:214 +0x4a4 fp=0xc01af6bc70 sp=0xc01af6bb00 pc=0x9e6284
github.com/go-ego/riot.(*Engine).segmenterWorker(0x1370740)
/root/Project/pkg/mod/github.com/go-ego/riot@v0.0.0-20181222142112-75a2cab10779/segment.go:261 +0x1b5 fp=0xc01af6bfd8 sp=0xc01af6bc70 pc=0x9e6465
runtime.goexit()
/root/go/src/runtime/asm_amd64.s:1333 +0x1 fp=0xc01af6bfe0 sp=0xc01af6bfd8 pc=0x45c201
created by github.com/go-ego/riot.(*Engine).Init
/root/Project/pkg/mod/github.com/go-ego/riot@v0.0.0-20181222142112-75a2cab10779/engine.go:331 +0x52c
goroutine 1 [chan receive]:
github.com/astaxie/beego.(*App).Run(0xc000118130, 0x0, 0x0, 0x0)
/root/Project/pkg/mod/github.com/astaxie/beego@v1.9.2/app.go:221 +0x262
github.com/astaxie/beego.Run(0x0, 0x0, 0x0)
/root/Project/pkg/mod/github.com/astaxie/beego@v1.9.2/beego.go:67 +0x62
main.main()
/root/Project/src/goblog/main.go:12 +0x32
goroutine 5 [syscall]:
os/signal.signal_recv(0x0)
/root/go/src/runtime/sigqueue.go:139 +0x9c
os/signal.loop()
/root/go/src/os/signal/signal_unix.go:23 +0x22
created by os/signal.init.0
/root/go/src/os/signal/signal_unix.go:29 +0x41
goroutine 6 [select]:
goblog/src/logs.async()
/root/Project/src/goblog/src/logs/logs.go:224 +0x108
created by goblog/src/logs.InitLogs
/root/Project/src/goblog/src/logs/logs.go:89 +0x761
goroutine 7 [chan receive]:
goblog/src/logs.(*logs).logCut(0xc000088550, 0xc00000e688)
/root/Project/src/goblog/src/logs/logs.go:111 +0x2a7
created by goblog/src/logs.InitLogs
/root/Project/src/goblog/src/logs/logs.go:91 +0x724
goroutine 8 [select]:
database/sql.(*DB).connectionOpener(0xc0000f00c0, 0xcb23e0, 0xc000061d40)
/root/go/src/database/sql/sql.go:1001 +0xe8
created by database/sql.OpenDB
/root/go/src/database/sql/sql.go:671 +0x15d
goroutine 9 [select]:
database/sql.(*DB).connectionResetter(0xc0000f00c0, 0xcb23e0, 0xc000061d40)
/root/go/src/database/sql/sql.go:1014 +0xfb
created by database/sql.OpenDB
/root/go/src/database/sql/sql.go:672 +0x193
goroutine 13 [select]:
github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher.func1(0xc000062d20, 0xc0000f0180, 0xc00002a5a0)
/root/Project/pkg/mod/github.com/go-sql-driver/mysql@v1.4.0/connection_go18.go:179 +0xbf
created by github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher
/root/Project/pkg/mod/github.com/go-sql-driver/mysql@v1.4.0/connection_go18.go:176 +0xbe
goroutine 14 [select]:
database/sql.(*DB).connectionCleaner(0xc0000f00c0, 0x1a3185c5000)
/root/go/src/database/sql/sql.go:899 +0x37d
created by database/sql.(*DB).startCleanerLocked
/root/go/src/database/sql/sql.go:886 +0xa7
goroutine 19 [chan receive]:
github.com/go-ego/riot.(*Engine).Store(0x1370740)
/root/Project/pkg/mod/github.com/go-ego/riot@v0.0.0-20181222142112-75a2cab10779/engine.go:212 +0x48e
github.com/go-ego/riot.(*Engine).Init(0x1370740, 0x0, 0x3, 0xbc5db6, 0x2, 0x0, 0x0, 0x0, 0x0, 0x1, ...)
/root/Project/pkg/mod/github.com/go-ego/riot@v0.0.0-20181222142112-75a2cab10779/engine.go:351 +0x720
goblog/src/service.init.2.func1()
/root/Project/src/goblog/src/service/search_service.go:55 +0xba
created by goblog/src/component.GoRoutine
/root/Project/src/goblog/src/component/routine_component.go:9 +0x33
goroutine 63 [IO wait]:
internal/poll.runtime_pollWait(0x7f4c13531d60, 0x72, 0x0)
/root/go/src/runtime/netpoll.go:173 +0x66
internal/poll.(*pollDesc).wait(0xc0000ef998, 0x72, 0xc000060000, 0x0, 0x0)
/root/go/src/internal/poll/fd_poll_runtime.go:85 +0x9a
internal/poll.(*pollDesc).waitRead(0xc0000ef998, 0xffffffffffffff00, 0x0, 0x0)
/root/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Accept(0xc0000ef980, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/root/go/src/internal/poll/fd_unix.go:384 +0x1a0
net.(*netFD).accept(0xc0000ef980, 0x1370e00, 0x30, 0x30)
/root/go/src/net/fd_unix.go:238 +0x42
net.(*TCPListener).accept(0xc0001c84b0, 0xc000052d18, 0x7f4c1358d000, 0xc000153500)
/root/go/src/net/tcpsock_posix.go:139 +0x2e
net.(*TCPListener).AcceptTCP(0xc0001c84b0, 0x40e238, 0x30, 0xb67640)
/root/go/src/net/tcpsock.go:247 +0x47
net/http.tcpKeepAliveListener.Accept(0xc0001c84b0, 0xb67640, 0xc000df7da0, 0xaedd40, 0x135c100)
/root/go/src/net/http/server.go:3232 +0x2f
net/http.(*Server).Serve(0xc0000ff860, 0xcb21e0, 0xc0001c84b0, 0x0, 0x0)
/root/go/src/net/http/server.go:2826 +0x22f
net/http.(*Server).ListenAndServe(0xc0000ff860, 0xc0000ff860, 0x26)
/root/go/src/net/http/server.go:2764 +0xb6
net/http.ListenAndServe(0xc000e00450, 0xe, 0x0, 0x0, 0x1, 0xc000e00450)
/root/go/src/net/http/server.go:3004 +0x74
github.com/astaxie/beego.(*adminApp).Run(0xc00000e648)
/root/Project/pkg/mod/github.com/astaxie/beego@v1.9.2/admin.go:399 +0x358
created by github.com/astaxie/beego.registerAdmin
/root/Project/pkg/mod/github.com/astaxie/beego@v1.9.2/hooks.go:89 +0x67
...
分析
猜测是线程数配置的问题,查看engine.go的Init方法中各参数的作用,如下:
func (engine *Engine) Init(options types.EngineOpts) {
// 将线程数设置为CPU数
// runtime.GOMAXPROCS(runtime.NumCPU())
// runtime.GOMAXPROCS(128)
// 初始化初始参数
if engine.initialized {
log.Fatal("Do not re-initialize the engine.")
}
options = engine.initDef(options)
options.Init()
engine.initOptions = options
engine.initialized = true
if !options.NotUseGse {
if !engine.loaded {
// 载入分词器词典
engine.segmenter.LoadDict(options.GseDict)
engine.loaded = true
}
// 初始化停用词
engine.stopTokens.Init(options.StopTokenFile)
}
// 初始化索引器和排序器
for shard := 0; shard < options.NumShards; shard++ {
engine.indexers = append(engine.indexers, core.Indexer{})
engine.indexers[shard].Init(*options.IndexerOpts)
engine.rankers = append(engine.rankers, core.Ranker{})
engine.rankers[shard].Init(options.IDOnly)
}
// 初始化分词器通道
engine.segmenterChan = make(
chan segmenterReq, options.NumGseThreads)
// 初始化索引器通道
engine.Indexer(options)
// 初始化排序器通道
engine.Ranker(options)
// engine.CheckMem(engine.initOptions.UseStore)
engine.CheckMem()
// 初始化持久化存储通道
if engine.initOptions.UseStore {
engine.InitStore()
}
// 启动分词器
for iThread := 0; iThread < options.NumGseThreads; iThread++ {
go engine.segmenterWorker()
}
// 启动索引器和排序器
for shard := 0; shard < options.NumShards; shard++ {
go engine.indexerAddDoc(shard)
go engine.indexerRemoveDoc(shard)
go engine.rankerAddDoc(shard)
go engine.rankerRemoveDoc(shard)
for i := 0; i < options.NumIndexerThreads; i++ {
go engine.indexerLookup(shard)
}
for i := 0; i < options.NumRankerThreads; i++ {
go engine.rankerRank(shard)
}
}
// 启动持久化存储工作协程
if engine.initOptions.UseStore {
engine.Store()
}
atomic.AddUint64(&engine.numDocsStored, engine.numIndexingReqs)
}
它的第331行代码为:
// 启动分词器
for iThread := 0; iThread < options.NumGseThreads; iThread++ {
go engine.segmenterWorker()
}
推测是riot组件的设置问题,于是查看search_service.go文件的init方法:
func init() {
SearchBiz = searchService{}
engineType = make(map[string]SearchEngine)
//标签搜索
engineType[TAG] = TagSearchEngine{}
//栏目搜索
engineType[CATEGORY] = CategorySearchEngine{}
//博文搜索
engineType[ARTICLES_FULL_TEXT] = ArticlesSearchEngine{}
//归档搜索
engineType[PLACE_OF_FILE] = PlaceOfFileSearchEngine{}
runtime.GOMAXPROCS(runtime.NumCPU())
fmt.Println("start init search====================")
component.GoRoutine(func() {
gob.Register(ArticlesScoringFields{})
opts := types.EngineOpts{
Using: 3,
GseDict: "zh",
UseStore: true,
StoreFolder: "./indexer",
StoreShards: 4,
StoreEngine: "",
}
fullTextSearcher.Init(opts)
//data,_ := json.Marshal(opts)
//fmt.Println(data)
fullTextSearcher.Flush()
})
fmt.Println("end init search====================")
}
于是查看EngineOpts方件:
type EngineOpts struct {
// 是否使用分词器
// 默认使用,否则在启动阶段跳过 GseDict 和 StopTokenFile 设置
// 如果你不需要在引擎内分词,可以将这个选项设为 true
// 注意,如果你不用分词器,那么在调用 IndexDoc 时,
// DocIndexData 中的 Content 会被忽略
// Not use the gse segment
NotUseGse bool `toml:"not_use_gse"`
// new, 分词规则
Using int `toml:"using"`
// 半角逗号 "," 分隔的字典文件,具体用法见
// gse.Segmenter.LoadDict 函数的注释
GseDict string `toml:"gse_dict"`
PinYin bool `toml:"pin_yin"`
// 停用词文件
StopTokenFile string `toml:"stop_file"`
// Gse search mode
GseMode bool `toml:"gse_mode"`
Hmm bool `toml:"hmm"`
Model string `toml:"model"`
// 分词器线程数
// NumSegmenterThreads int
NumGseThreads int
// 索引器和排序器的 shard 数目
// 被检索/排序的文档会被均匀分配到各个 shard 中
NumShards int
// 索引器的信道缓冲长度
IndexerBufLen int
// 索引器每个shard分配的线程数
NumIndexerThreads int
// 排序器的信道缓冲长度
RankerBufLen int
// 排序器每个 shard 分配的线程数
NumRankerThreads int
// 索引器初始化选项
IndexerOpts *IndexerOpts
// 默认的搜索选项
DefRankOpts *RankOpts
// 是否使用持久数据库,以及数据库文件保存的目录和裂分数目
StoreOnly bool `toml:"store_only"`
UseStore bool `toml:"use_store"`
StoreFolder string `toml:"store_folder"`
StoreShards int `toml:"store_shards"`
StoreEngine string `toml:"store_engine"`
IDOnly bool `toml:"id_only"`
}
修改成:
component.GoRoutine(func() {
gob.Register(ArticlesScoringFields{})
opts := types.EngineOpts{
NumGseThreads:2,
NumIndexerThreads:2,
NumRankerThreads:2,
Using: 3,
GseDict: "zh",
UseStore: true,
StoreFolder: "./indexer",
StoreShards: 4,
StoreEngine: "",
}
fullTextSearcher.Init(opts)
//data,_ := json.Marshal(opts)
//fmt.Println(data)
fullTextSearcher.Flush()
})
重启服务,问题解决。
总结将相同的代码和配置放到一台性能较好的机器上不会复现上述问题,应该是riot组件默认设置对机器要求比较高,改低一些,问题解决。
另外还可以通过net/http/pprof查看golang内存情况,详见:https://segmentfault.com/a/1190000019929993