Golang爬虫如何实现HTTP请求

packagemain

import(

    "net/url"

    "net/http"

    "bytes"

    "fmt"

    "io/ioutil"

)

// 代理服务器(产品官网 www.16yun.cn)

constProxyServer = "t.16yun.cn:31111"

typeProxyAuth struct{

    Username string

    Password string

}

func(p ProxyAuth) ProxyClient() http.Client {

    varproxyURL *url.URL

    ifp.Username != ""&& p.Password!=""{

        proxyURL, _ = url.Parse("http://"+ p.Username + ":"+ p.Password + "@"+ ProxyServer)

    }else{

        proxyURL, _ = url.Parse("http://"+ ProxyServer)

    }

    returnhttp.Client{Transport: &http.Transport{Proxy:http.ProxyURL(proxyURL)}}

}

funcmain()  {

    targetURI := "https://httpbin.org/ip"

    // 初始化 proxy http client

    client := ProxyAuth{"username",  "password"}.ProxyClient()

    request, _ := http.NewRequest("GET", targetURI, bytes.NewBuffer([] byte(``)))

    // 设置Proxy-Tunnel

    // rand.Seed(time.Now().UnixNano())

    // tunnel := rand.Intn(10000)

    // request.Header.Set("Proxy-Tunnel", strconv.Itoa(tunnel) )

    response, err := client.Do(request)

    iferr != nil{

        panic("failed to connect: "+ err.Error())

    } else{

        bodyByte, err := ioutil.ReadAll(response.Body)

        iferr != nil{

            fmt.Println("读取 Body 时出错", err)

            return

        }

        response.Body.Close()

        body := string(bodyByte)

        fmt.Println("Response Status:", response.Status)

        fmt.Println("Response Header:", response.Header)

        fmt.Println("Response Body:\n", body)

    }

}