概述
go中方便的协程写法, 让并发变得很简单, 让开发者很多时候不需要关心协程(线程)管理的问题, 不过也是有值得注意的地方.
下面以一个http请求的例子, 请求的服务是进制数据-手机号码归属地查询.(找了很久才找到方便学demo的免费API, 想跑demo的朋友注册一下在代码中填上自己的appkey即可)
并发方式
平时常用到3种方式做并发请求.
- 简单粗暴, 一次请求开一个线程, 线程中完成请求与响应处理.
- 优雅一点, 限制线程池, 以master-worker的方式处理并发, 最后又把响应统一处理.
- 优雅地限制线程池(2020.7.15更新)
1. 简单并发
代码
package main
import (
"fmt"
"io/ioutil"
"net/http"
"sync"
"time"
)
func main() {
var numberTasks = [5]string{"13456755448", " 13419385751", "13419317885", " 13434343439","13438522395"}
client = &http.Client{}
beg := time.Now()
wg := &sync.WaitGroup{}
for _, keyword := range numberTasks{
wg.Add(1)
go func(keyword string, group *sync.WaitGroup) {
body, err := NumberQueryRequest(keyword)
if err != nil{
fmt.Printf("error occurred in query keyword: %s, error: %s\n",keyword, err.Error() )
}else{
fmt.Printf("search %s success, data size is %d\n, body is %s\n",keyword, len(body), string(body))
}
group.Done()
}(keyword, wg)
}
wg.Wait() //waitGroup在主线程等待所有任务完成
fmt.Printf("time consumed: %fs", time.Now().Sub(beg).Seconds())
}
var client *http.Client
//GET请求的通用写法, 注意复用client即可.
func NumberQueryRequest(keyword string)(body []byte, err error){
url := fmt.Sprintf("https://api.binstd.com/shouji/query?appkey=df2720f76a0991fa&shouji=%s", keyword)
req, err := http.NewRequest("GET", url, nil)
if err != nil{
return nil, err
}
resp, err := client.Get(url)
if err != nil{
return nil, err
}
if resp.StatusCode != http.StatusOK{
data, _ := ioutil.ReadAll(resp.Body)
return nil, fmt.Errorf("response status code is not OK, response code is %d, body:%s", resp.StatusCode, string(data))
}
if resp != nil && resp.Body != nil{
defer resp.Body.Close()
}
body, err = ioutil.ReadAll(resp.Body)
if err != nil{
return nil, err
}
return body, nil
}
解析
- 在for循环中有多少个任务就go出去多少个协程,没有限制,在查询量固定或者不大的时候没什么问题.
- 没有用到 channel的特性, 仅利用了多核调度.
2. master-worker形式
代码
package main
import (
"fmt"
"io/ioutil"
"net/http"
"sync"
"time"
)
const (
routineCountTotal = 5 //限制线程数
)
func main() {
var numberTasks = [5]string{"13456755448", " 13419385751", "13419317885", " 13434343439", "13438522395"}
client = &http.Client{}
beg := time.Now()
wg := &sync.WaitGroup{}
tasks := make(chan string)
results := make(chan string)
//receiver接受响应并处理的函数块, 也可以单独写在一个函数
go func() {
for result := range results{
if result == ""{
close(results)
}else{
fmt.Println("result:", result)
}
}
}()
for i:= 0; i < routineCountTotal; i++{
wg.Add(1)
go worker(wg, tasks, results)
}
//分发任务
for _, task := range numberTasks{
tasks <- task
}
tasks <- "" //worker结束标志
wg.Wait() //同步结束
results <- "" // result结束标志
fmt.Printf("time consumed: %fs", time.Now().Sub(beg).Seconds())
}
func worker(group *sync.WaitGroup, tasks chan string, result chan string){
for task := range tasks{
if task == ""{
close(tasks)
}else{
respBody, err := NumberQueryRequest(task)
if err != nil{
fmt.Printf("error occurred in NumberQueryRequest: %s\n", task)
result <- err.Error()
}else{
result <- string(respBody)
}
}
}
group.Done()
}
var client *http.Client
func NumberQueryRequest(keyword string) (body []byte, err error) {
url := fmt.Sprintf("https://api.binstd.com/shouji/query?appkey=df2720f76a0991fa&shouji=%s", keyword)
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, err
}
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36")
resp, err := client.Get(url)
if err != nil {
return nil, err
}
if resp.StatusCode != http.StatusOK {
data, _ := ioutil.ReadAll(resp.Body)
return nil, fmt.Errorf("response status code is not OK, response code is %d, body:%s", resp.StatusCode, string(data))
}
if resp != nil && resp.Body != nil {
defer resp.Body.Close()
}
body, err = ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
return body, nil
}
输出
result: {"status":0,"msg":"ok","result":{"shouji":"13434343439","province":"广东","city":"广州","company":"中国移动","cardtype":null,"areacode":"020"}}
result: {"status":0,"msg":"ok","result":{"shouji":"13438522395","province":"四川","city":"攀枝花","company":"中国移动","cardtype":null,"areacode":"0812"}}
result: {"status":0,"msg":"ok","result":{"shouji":"13419317885","province":"四川","city":"广安","company":"中国移动","cardtype":null,"areacode":"0826"}}
result: {"status":0,"msg":"ok","result":{"shouji":"13456755448","province":"浙江","city":"杭州","company":"中国移动","cardtype":null,"areacode":"0571"}}
result: {"status":0,"msg":"ok","result":{"shouji":"13419385751","province":"四川","city":"遂宁","company":"中国移动","cardtype":null,"areacode":"0825"}}
time consumed: 0.399975s
3. 使用带缓冲区的chan控制
先上代码
package golimit
type Glimit struct {
Num int
C chan struct{}
}
func NewG(num int) *Glimit {
return &Glimit{
Num: num,
C : make(chan struct{}, num),
}
}
func (g *Glimit) Run(f func()){
g.C <- struct{}{}
go func() {
f()
<-g.C
}()
}
使用方式
func main() {
number := 10
// 限制线程数2
g := NewG(2)
wg := sync.WaitGroup{}
for i := 0; i < number; i++ {
wg.Add(1)
value :=i // 闭包传值方式
goFunc := func() {
// 做一些业务逻辑处理
fmt.Printf("go func: %d\n", value)
time.Sleep(time.Second)
wg.Done()
}
g.Run(goFunc)
}
wg.Wait()
}
并发请求的用法
package main
import (
"fmt"
"io/ioutil"
"net/http"
"sync"
"time"
"./golimit"
)
const (
routineCountTotal = 5 //限制线程数
)
func main() {
var numberTasks = [5]string{"13456755448", " 13419385751", "13419317885", " 13434343439", "13438522395"}
g := golimit.NewG(routineCountTotal)
wg := &sync.WaitGroup{}
client = &http.Client{}
beg := time.Now()
for i := 0; i < len(numberTasks); i++ {
wg.Add(1)
task := numberTasks[i]
g.Run(func() {
respBody, err := NumberQueryRequest(task)
if err != nil {
fmt.Printf("error occurred in NumberQueryRequest: %s\n", task)
} else {
fmt.Printf("response data: %s\n", string(respBody))
}
wg.Done()
})
}
wg.Wait()
fmt.Printf("time consumed: %fs", time.Now().Sub(beg).Seconds())
}
var client *http.Client
func NumberQueryRequest(keyword string) (body []byte, err error) {
url := fmt.Sprintf("https://api.binstd.com/shouji/query?appkey=df2720f76a0991fa&shouji=%s", keyword)
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, err
}
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36")
resp, err := client.Get(url)
if err != nil {
return nil, err
}
if resp.StatusCode != http.StatusOK {
data, _ := ioutil.ReadAll(resp.Body)
return nil, fmt.Errorf("response status code is not OK, response code is %d, body:%s", resp.StatusCode, string(data))
}
if resp != nil && resp.Body != nil {
defer resp.Body.Close()
}
body, err = ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
return body, nil
}
输出
response data: {"status":0,"msg":"ok","result":{"shouji":"13456755448","province":"浙江","city":"杭州","company":"中国移动","cardtype":null,"areacode":"0571"}}
response data: {"status":0,"msg":"ok","result":{"shouji":"13438522395","province":"四川","city":"攀枝花","company":"中国移动","cardtype":null,"areacode":"0812"}}
response data: {"status":0,"msg":"ok","result":{"shouji":"13434343439","province":"广东","city":"广州","company":"中国移动","cardtype":null,"areacode":"020"}}
response data: {"status":0,"msg":"ok","result":{"shouji":"13419385751","province":"四川","city":"遂宁","company":"中国移动","cardtype":null,"areacode":"0825"}}
response data: {"status":0,"msg":"ok","result":{"shouji":"13419317885","province":"四川","city":"广安","company":"中国移动","cardtype":null,"areacode":"0826"}}
time consumed: 0.937952s%
解析
- 比第一种方式更规范一些, 线程数量可以控制.
- 两处close通道的地方都用了传特殊空值的方式, 似乎不太优雅, 欢迎有更好的关闭通道的方式互相交流, 谢谢.
第三种方式解析(2020.7.15更新)
- 巧妙地使用go带缓冲区的通道来实现goroutine控制, 更加简洁, 并且这种方式在多个项目中都可以复用. 不必像第二种方式一样每次都实现一个worker函数.