gocolly的使用 · 音视频/C++/k8s/Docker等等 学习笔记 · 看云
[TOC]
参考：https://blog.csdn.net/weixin_43968923/article/details/108994707
## 概述
gocolly是golang语言开发的爬虫包，通过gocolly来进行模拟网站的登录，
目标已经确立，开始行动。
### 下载
```
go get -u github.com/gocolly/colly
```
## cookie登录
登录网站：https://www.bilibili.com/
获取cookie：
* google浏览器访问`bilibili`
* 按F12
* 查看`NetWork`选项
* 点击`Doc`
* 查看 cookie信息
## colly爬虫代码流程
初始化一个`Collector`收集器
```
c := colly.NewCollector(
colly.AllowedDomains("www.bilibili.com"),
colly.AllowURLRevisit(),
colly.UserAgent("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.75 Safari/537.36"))
```
爬取网站的规则设置：
```
err := c.Limit(&colly.LimitRule{
// Filter domains affected by this rule
// 筛选受此规则影响的域
DomainGlob: "bilibili.com/*",
// Set a delay between requests to these domains
// 设置对这些域的请求之间的延迟
Delay: 1 * time.Second,
// Add an additional random delay
// 添加额外的随机延迟
RandomDelay: 1 * time.Second,
// 设置并发
Parallelism: 5,
})
```
开始爬取
```
err = c.Visit(url)
c.Wait()
```
代码示例
```
package main
import (
"fmt"
"github.com/gocolly/colly"
"net/http"
"os"
"strings"
"time"
)
/*
请求执行之前调用
- OnRequest
响应返回之后调用
- OnResponse
监听执行 selector
- OnHTML
监听执行 selector
- OnXML
错误回调
- OnError
完成抓取后执行，完成所有工作后执行
- OnScraped
取消监听，参数为 selector 字符串
- OnHTMLDetach
取消监听，参数为 selector 字符串
- OnXMLDetach
*/
// set cookies raw
func setCookieRaw(cookieRaw string) []*http.Cookie {
// 可以添加多个cookie
var cookies []*http.Cookie
cookieList := strings.Split(cookieRaw, "; ")
for _, item := range cookieList {
keyValue := strings.Split(item, "=")
// fmt.Println(keyValue)
name := keyValue[0]
valueList := keyValue[1:]
cookieItem := http.Cookie{
Name: name,
Value: strings.Join(valueList, "="),
}
cookies = append(cookies, &cookieItem)
}
return cookies
}
func main() {
c := colly.NewCollector(
colly.AllowedDomains("www.bilibili.com"),
colly.AllowURLRevisit(),
colly.UserAgent("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.75 Safari/537.36"))
err := c.Limit(&colly.LimitRule{
// Filter domains affected by this rule
// 筛选受此规则影响的域
DomainGlob: "bilibili.com/*",
// Set a delay between requests to these domains
// 设置对这些域的请求之间的延迟
Delay: 1 * time.Second,
// Add an additional random delay
// 添加额外的随机延迟
RandomDelay: 1 * time.Second,
// 设置并发
Parallelism: 5,
})
if err != nil {
fmt.Println("fad:", err)
}
header := map[string]string{
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
"Connection": "keep-alive",
"Host": "https://www.bilibili.com/",
"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.121 Safari/537.36",
}
url := "https://www.bilibili.com/"
// 在提出请求之前打印 "访问…"
c.OnRequest(func(r *colly.Request) {
for key, value := range header {
r.Headers.Add(key, value)
}
fmt.Println("url: ", r.URL.String())
cookie := "xxx"
err := c.SetCookies(url, setCookieRaw(cookie))
if err != nil {
fmt.Println("fad:", err)
}
})
c.OnHTML("a[href]", func(e *colly.HTMLElement) {
target := e.Attr("target")
name := e.ChildText("span")
if len(name) > 0 {
fmt.Println("name: ", name)
if target == "_blank" && name == "动态"{
link := e.Attr("href")
fmt.Println("link: ", link)
}
}
})
err = c.Visit(url)
if err != nil {
fmt.Errorf("fffffff %s\n", err.Error())
os.Exit(-1)
}
c.Wait()
fmt.Println("程序结束")
}
```