golang 使用线程池进行高并发爬虫

golang 使用线程池进行高并发爬虫

gospider 介绍

gospider 是一个golang 爬虫神器，拥有python到golang爬虫过渡的所有必需库。用于python爬虫从业者快速且无坑的过渡到golang

安装

go get -u gitee.com/baixudong/gospider

gitee地址

https://gitee.com/baixudong/gospider

github地址

https://github.com/baixudong007/gospider

代码示例

package main

import (
    "context"
    "log"
    "net/url"

    "gitee.com/baixudong/gospider/requests"
    "gitee.com/baixudong/gospider/thread"
)

var reqCli *requests.Client

func init() {
    var err error
    reqCli, err = requests.NewClient(nil, requests.ClientOption{
        GetProxy: func(ctx context.Context, url *url.URL) (string, error) { //自动获取代理
            return "", nil
        },
    })
    if err != nil {
        log.Panic(err)
    }
}
func test(ctx context.Context, num int) {
    log.Printf("第%d个请求开始", num)
    resp, err := reqCli.Request(ctx, "get", "http://myip.top")
    if err != nil {
        log.Printf("第%d个请求失败%s", num, err.Error())
    } else {
        log.Printf("第%d个请求成功%d", num, resp.StatusCode())
    }
    log.Printf("第%d个请求结束", num)
}
func main() {
    threadCli := thread.NewClient(nil, 3) //限制并发为3
    for i := 0; i < 10; i++ {
        //读取任务
        threadCli.Write(&thread.Task{
            Func: test,
            Args: []any{i},
        })
    }
    threadCli.Join()
}