压力测试下的 golang http client 造成的连接数过多

看博客，自己运行验证

loadtest.go

package main

import (
  "fmt"
  "io"
  "io/ioutil"
  "net/http"
)

func startWebserver() {
  http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
    fmt.Fprintf(w, "Hello, %q", r.URL.Path)
  })

  http.ListenAndServe(":8080", nil)
}

func startLoadTest() {
  for count := 0; ; count++ {
    resp, err := http.Get("http://localhost:8080/")
    if err != nil {
      panic(fmt.Sprintf("Got error: %v", err))
    }

    // io.Copy(ioutil.Discard, resp.Body) // <-- v2 add this line
    resp.Body.Close()
    fmt.Printf("Finished GET request #%v\n", count)
  }
}

func main() {
  // start a webserver in a goroutine
  go startWebserver()
  startLoadTest()
}

[2020-08-28 11:32:39.459] ❯ go run loadtest.go
2020/08/28 11:32:39 Finished GET request #96105
panic: Got error: Get "http://localhost:8080/": dial tcp [::1]:8080: connect: resource temporarily unavailable

goroutine 1 [running]:
main.startLoadTest()
  /Users/bingoobjca/github/weeklyreport/2020/loadtest.go:22 +0x168
main.main()
  /Users/bingoobjca/github/weeklyreport/2020/loadtest.go:32 +0x3a
exit status 2

[2020-08-28 11:30:18.925] ❯ while true; do date "+%Y-%m-%d %H:%M:%S"; netstat -n | grep -i 8080 | grep -i time_wait | wc -l; sleep 3; done
2020-08-28 11:30:27
     576
2020-08-28 11:30:30
    7229
2020-08-28 11:30:33
   13068
2020-08-28 11:30:36
   17130
2020-08-28 11:30:39
   19069
2020-08-28 11:30:42
   20892
2020-08-28 11:30:45
   22642
2020-08-28 11:30:48
   24283
2020-08-28 11:30:51
   25677
2020-08-28 11:30:55
   27111
2020-08-28 11:30:58
   28496
2020-08-28 11:31:01
   29842
2020-08-28 11:31:04
   31131
2020-08-28 11:31:07
    6979
2020-08-28 11:31:26
   11667
2020-08-28 11:31:29
   15000
2020-08-28 11:31:33
   18616
2020-08-28 11:31:36
   20287
2020-08-28 11:31:39
   20926
2020-08-28 11:31:42
   22444
2020-08-28 11:31:45
   24109
2020-08-28 11:31:48
   25698
2020-08-28 11:31:51
   27123
2020-08-28 11:31:54
   28364
2020-08-28 11:31:57
   29710
2020-08-28 11:32:01
   31009
2020-08-28 11:32:04
    6434
2020-08-28 11:32:23
    9741
2020-08-28 11:32:26
   13645
2020-08-28 11:32:29
   18158
2020-08-28 11:32:32
   20768
2020-08-28 11:32:35
   26138
2020-08-28 11:32:38
   32443
2020-08-28 11:32:41
   32689
2020-08-28 11:32:45
   32689
2020-08-28 11:32:48
   32689
2020-08-28 11:32:51
   32689
2020-08-28 11:32:54
       0

loadtest.go

io.Copy(ioutil.Discard, resp.Body) // <-- v2 add this lineresp.Body.Close()

[2020-08-28 11:33:12.700] ❯ while true; do date "+%Y-%m-%d %H:%M:%S"; netstat -n | grep -i 8080 | grep -i time_wait | wc -l; sleep 3; done
2020-08-28 11:36:09
       0
2020-08-28 11:36:12
       0
2020-08-28 11:36:15
       0
2020-08-28 11:36:18
       0
2020-08-28 11:36:21
       0

the io.Copy() drains the body meaning it can be reused via keepalive.
if there's still data pending, the Close() will actually close it and it can't be reused

并发 client

package main

import (
  "fmt"
  "io"
  "io/ioutil"
  "net/http"
  "time"
)

func startWebserver() {
  http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
    time.Sleep(time.Millisecond * 50)
    fmt.Fprintf(w, "Hello, %q", r.URL.Path)
  })

  http.ListenAndServe(":8080", nil)
}

func startLoadTest() {
  for count := 0; ; count++ {
    resp, err := http.Get("http://localhost:8080/")
    if err != nil {
      panic(fmt.Sprintf("Got error: %v", err))
    }

    io.Copy(ioutil.Discard, resp.Body) // <-- v2 add this line
    resp.Body.Close()
    fmt.Printf("Finished GET request #%v\n", count)
  }
}

func resetClient() {
  // Customize the Transport to have larger connection pool
  defaultRoundTripper := http.DefaultTransport
  defaultTransportPointer, ok := defaultRoundTripper.(*http.Transport)
  if !ok {
    panic(fmt.Sprintf("defaultRoundTripper not an *http.Transport"))
  }
  defaultTransport := *defaultTransportPointer // dereference it to get a copy of the struct that the pointer points to
  defaultTransport.MaxIdleConns = 100
  defaultTransport.MaxIdleConnsPerHost = 100

  http.DefaultClient = &http.Client{Transport: &defaultTransport}
}

func main() {
  // start a webserver in a goroutine
  go startWebserver()
  // startLoadTest()

  // resetClient()
  for i := 0; i < 100; i++ {
    go startLoadTest()
  }

  time.Sleep(time.Second * 2400)
}

panic: Got error: Get "http://localhost:8080/": dial tcp [::1]:8080: socket: too many open files

// startLoadTest()

const DefaultMaxIdleConnsPerHost = 2

压力测试下的 golang http client 造成的连接数过多

loadtest.go

loadtest.go

并发 client

更多资源