Go调度器学习之协作与抢占详解

0. 简介

在上篇博客——《Golang调度器(4)—goroutine调度》中一直遗留了一个没有解答的问题：如果某个G执行时间过长，其他的G如何才能被正常调度，这就引出了接下来的话题：协作与抢占。

v1.2

sysmon调用（call）

但是这种调度方式是协程主动的，是基于协作的，但是他无法面对一些场景，比如在死循环中没有任何调用发生，那么这个协程将永远执行下去，永远不会发生调度，这显然是不可接受的。

v1.14

1. 用户主动让出CPU：runtime.Gosched函数

runtime.Gosched

// Gosched yields the processor, allowing other goroutines to run. It does not
// suspend the current goroutine, so execution resumes automatically.
func Gosched() {
   checkTimeouts()
   mcall(gosched_m)
}

runtime.Gosched

mcallg0gosched_m

// Gosched continuation on g0.
func gosched_m(gp *g) {
   if trace.enabled {
      traceGoSched()
   }
   goschedImpl(gp)
}

gosched_mgoschedImplgpgp

func goschedImpl(gp *g) {
   status := readgstatus(gp)
   if status&^_Gscan != _Grunning {
      dumpgstatus(gp)
      throw("bad g status")
   }
   casgstatus(gp, _Grunning, _Grunnable)
   dropg()            // 使当前m放弃gp，就是其参数 curg
   lock(&sched.lock)
   globrunqput(gp)    // 并且把gp放到全局队列中，等待调度
   unlock(&sched.lock)

   schedule()
}

runtime.Gosched

2. 基于协作的抢占式调度

2.1 场景

package main

import (
   "fmt"
   "runtime"
   "sync"
   "time"
)

var once = sync.Once{}

func f() {
   once.Do(func() {
      fmt.Println("I am go routine 1!")
   })
}

func main() {
   defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(1))

   go func() {
      for {
         f()
      }
   }()

   time.Sleep(10 * time.Millisecond)
   fmt.Println("I am main goroutine!")
}

fmt.Println("I am main goroutine!")

2.2 栈扩张与抢占标记

$ go tool compile -N -l main.go
$ go tool objdump main.o >> main.i

TEXT "".f(SB) gofile../home/chenyiguo/smb_share/go_routine_test/main.go
main.go:12 0x151a 493b6610 CMPQ 0x10(R14), SP
main.go:12 0x151e 762b JBE 0x154b
main.go:12 0x1520 4883ec18 SUBQ $0x18, SP
main.go:12 0x1524 48896c2410 MOVQ BP, 0x10(SP)
main.go:12 0x1529 488d6c2410 LEAQ 0x10(SP), BP
main.go:13 0x152e 488d0500000000 LEAQ 0(IP), AX [3:7]R_PCREL:"".once
main.go:13 0x1535 488d1d00000000 LEAQ 0(IP), BX [3:7]R_PCREL:"".f.func1·f
main.go:13 0x153c e800000000 CALL 0x1541 [1:5]R_CALL:sync.(*Once).Do
main.go:16 0x1541 488b6c2410 MOVQ 0x10(SP), BP
main.go:16 0x1546 4883c418 ADDQ $0x18, SP
main.go:16 0x154a c3 RET
main.go:12 0x154b e800000000 CALL 0x1550 [1:5]R_CALL:runtime.morestack_noctxt
main.go:12 0x1550 ebc8 JMP "".f(SB)

CMPQ 0x10(R14), SPSP0x10(R14)stackguard0AT&TCMPSP0x10(R14)0x154bruntime.morestack_noctxt//go:nosplit

接下来，我们将关注于两点来打通整个链路，即：

栈扩张怎么重新调度，让出CPU的执行权？
何时会设置栈扩张标记？

2.3 栈扩张怎么触发重新调度

// morestack but not preserving ctxt.
TEXT runtime·morestack_noctxt(SB),NOSPLIT,$0
   MOVL   $0, DX
   JMP    runtime·morestack(SB)

TEXT runtime·morestack(SB),NOSPLIT,$0-0
   ...

   // Set g->sched to context in f.
   MOVQ   0(SP), AX // f's PC
   MOVQ   AX, (g_sched+gobuf_pc)(SI)
   LEAQ   8(SP), AX // f's SP
   MOVQ   AX, (g_sched+gobuf_sp)(SI)
   MOVQ   BP, (g_sched+gobuf_bp)(SI)
   MOVQ   DX, (g_sched+gobuf_ctxt)(SI)

   ...
   CALL   runtime·newstack(SB)
   CALL   runtime·abort(SB)  // crash if newstack returns
   RET

runtime·morestack_noctxtruntime·morestackruntime·morestackruntime.newstack

func newstack() {
   ...

   gp := thisg.m.curg
   
   ...
   stackguard0 := atomic.Loaduintptr(&gp.stackguard0)

   ...
   preempt := stackguard0 == stackPreempt
   ...

   if preempt {
      if gp == thisg.m.g0 {
         throw("runtime: preempt g0")
      }
      if thisg.m.p == 0 && thisg.m.locks == 0 {
         throw("runtime: g is running but p is not")
      }

      if gp.preemptShrink {
         // We're at a synchronous safe point now, so
         // do the pending stack shrink.
         gp.preemptShrink = false
         shrinkstack(gp)
      }

      if gp.preemptStop {
         preemptPark(gp) // never returns
      }

      // Act like goroutine called runtime.Gosched.
      gopreempt_m(gp) // never return
   }

   ...
}

runtime.newstackstackguard0gopreempt_m(gp)

func gopreempt_m(gp *g) {
   if trace.enabled {
      traceGoPreempt()
   }
   goschedImpl(gp)
}

gopreempt_mGoschedgosched_mgoschedImplgpgp

这里我们就明白了，一旦发生栈扩张，就有可能会发生让渡出执行权，进行重新调度的可能性，那什么时候会发生栈扩张呢？

2.4 何时设置栈扩张标记

stackguard0stackPreemptsysmongoroutineretake

func sysmon() {
   ...

   for {
      ...
      // retake P's blocked in syscalls
      // and preempt long running G's
      if retake(now) != 0 {
         idle = 0
      } else {
         idle++
      }
      ...
   }
}

func retake(now int64) uint32 {
   ...
   for i := 0; i < len(allp); i++ {
      ...
      s := _p_.status
      sysretake := false
      if s == _Prunning || s == _Psyscall {
         // Preempt G if it's running for too long.
         t := int64(_p_.schedtick)
         if int64(pd.schedtick) != t {
            pd.schedtick = uint32(t)
            pd.schedwhen = now
         } else if pd.schedwhen+forcePreemptNS <= now { // forcePreemptNS=10ms
            preemptone(_p_) // 在这里设置栈扩张标记
            // In case of syscall, preemptone() doesn't
            // work, because there is no M wired to P.
            sysretake = true
         }
      }
      ...
   }
   unlock(&allpLock)
   return uint32(n)
}

preemptone

func preemptone(_p_ *p) bool {
   mp := _p_.m.ptr()
   if mp == nil || mp == getg().m {
      return false
   }
   gp := mp.curg
   if gp == nil || gp == mp.g0 {
      return false
   }

   gp.preempt = true

   // Every call in a goroutine checks for stack overflow by
   // comparing the current stack pointer to gp->stackguard0.
   // Setting gp->stackguard0 to StackPreempt folds
   // preemption into the normal stack overflow check.
   gp.stackguard0 = stackPreempt // 设置栈扩张标记

   // Request an async preemption of this P.
   if preemptMSupported && debug.asyncpreemptoff == 0 {
      _p_.preempt = true
      preemptM(mp)
   }

   return true
}

goroutine

≥10ms

3. 基于信号的抢占式调度

"I am main goroutine!"

package main

import (
   "fmt"
   "runtime"
   "sync"
   "time"
)

var once = sync.Once{}

func main() {
   defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(1))

   go func() {
      for {
         once.Do(func() {
            fmt.Println("I am go routine 1!")
         })
      }
   }()

   time.Sleep(10 * time.Millisecond)
   fmt.Println("I am main goroutine!")
}

for

3.1 发送抢占信号

Go SDKpreemptone

if preemptMSupported && debug.asyncpreemptoff == 0 {
   _p_.preempt = true
   preemptM(mp)
}

preemptM_SIGURG

const sigPreempt = _SIGURG


func preemptM(mp *m) {
   // On Darwin, don't try to preempt threads during exec.
   // Issue #41702.
   if GOOS == "darwin" || GOOS == "ios" {
      execLock.rlock()
   }

   if atomic.Cas(&mp.signalPending, 0, 1) {
      if GOOS == "darwin" || GOOS == "ios" {
         atomic.Xadd(&pendingPreemptSignals, 1)
      }

      // If multiple threads are preempting the same M, it may send many
      // signals to the same M such that it hardly make progress, causing
      // live-lock problem. Apparently this could happen on darwin. See
      // issue #37741.
      // Only send a signal if there isn't already one pending.
      signalM(mp, sigPreempt)
   }

   if GOOS == "darwin" || GOOS == "ios" {
      execLock.runlock()
   }
}

3.2 抢占调用的注入

m0mstartmstartm0initsig

func initsig(preinit bool) {
  ...

   for i := uint32(0); i < _NSIG; i++ {
      ...

      handlingSig[i] = 1
      setsig(i, abi.FuncPCABIInternal(sighandler))
   }
}

sighandler

func sighandler(sig uint32, info *siginfo, ctxt unsafe.Pointer, gp *g) {
   ...

   if sig == sigPreempt && debug.asyncpreemptoff == 0 {
      // Might be a preemption signal.
      doSigPreempt(gp, c)
      // Even if this was definitely a preemption signal, it
      // may have been coalesced with another signal, so we
      // still let it through to the application.
   }

   ...
}

sigPreemptdoSigPreempt

func doSigPreempt(gp *g, ctxt *sigctxt) {
   // Check if this G wants to be preempted and is safe to
   // preempt.
   if wantAsyncPreempt(gp) {
      if ok, newpc := isAsyncSafePoint(gp, ctxt.sigpc(), ctxt.sigsp(), ctxt.siglr()); ok {
         // Adjust the PC and inject a call to asyncPreempt.
         ctxt.pushCall(abi.FuncPCABI0(asyncPreempt), newpc) // 插入抢占调用
      }
   }

   // Acknowledge the preemption.
   atomic.Xadd(&gp.m.preemptGen, 1)
   atomic.Store(&gp.m.signalPending, 0)

   if GOOS == "darwin" || GOOS == "ios" {
      atomic.Xadd(&pendingPreemptSignals, -1)
   }
}

doSigPreempt—>asyncPreempt->asyncPreempt2

func asyncPreempt2() {
   gp := getg()
   gp.asyncSafePoint = true
   if gp.preemptStop {
      mcall(preemptPark)
   } else {
      mcall(gopreempt_m)
   }
   gp.asyncSafePoint = false
}

gopreempt_m

所以对于基于信号的抢占调度，总结如下：

_SIGURGgoroutine

4. 小结

Go

您可能感兴趣的文章: