go源码中关于系统调用的定义如下:

func Syscall(trap, a1, a2, a3 uintptr) (r1, r2 uintptr, err Errno)
func Syscall6(trap, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2 uintptr, err Errno)
func RawSyscall(trap, a1, a2, a3 uintptr) (r1, r2 uintptr, err Errno)
func RawSyscall6(trap, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2 uintptr, err Errno)

其中Syscall和RawSyscall区别在于Syscall开始和结束,分别调用了 runtime 中的进入系统调用和退出系统调用的函数,说明Syscall函数受调度器控制,不会造成系统堵塞,而RawSyscall函数没有调用runtime,因此可能会造成堵塞,一般我们使用Syscall就可以了,RawSyscall最好用在不会堵塞的情况下。

 

func Syscall(trap, a1, a2, a3 uintptr) (r1, r2 uintptr, err Errno)

Syscall 的定义位于 src/syscall/asm_linux_amd64.s, 是用汇编写成的,封装了对linux底层的调用。接收4个参数,其中trap为中断信号,a1,a2,a3为底层调用函数对应的参数

举例说明:Go调用底层ioctl函数

trap中断类型传入syscall.SYS_IOCTL,SYS_IOCTL中断号表示调用linux底层ioctl函数
Syscall函数中剩下三个参数a1,a2,a3分别对应ioctl的三个参数。可以man命令查看linux ioctl函数参数,如下

int ioctl(int d, int request, ...);

第一个参数d指定一个由open/socket创建的文件描述符,即socket套接字
第二个参数request指定操作的类型,即对该文件描述符执行何种操作,设备相关的请求的代码
第三个参数为一块内存区域,通常依赖于request指定的操作类型

具体过程如下:
1 通过socket创建套接字
2 初始化struct ifconf与/或struct ifreq结构
3 调用ioctl函数,执行相应类型的SIO操作
4 获取返回至truct ifconf与/或struct ifreq结构中的相关信息

调用底层socket函数创建socket套接字,linux下用man命令查看socket函数用法

 

int socket(int domain, int type, int protocol);

其中domain为协议类型,type为套接字类型,protocol指定某个协议类型常值
domain的值有:

AF_INET IPv4协议
AF_INET6 Ipv6协议
AF_ROUTE 路由套接字
...

type的值有:

SOCK_STREAM 字节流套接字
SOCK_DGRAM 数据报套接字
SOCK_RAW 原始套接字
...

protocol的值有:

IPPROTO_IP IP传输协议
IPPROTO_TCP TCP传输协议
IPPROTO_UDP UDP传输协议
...

 

因此linux下调用socket生成套接字写法:

fd = socket(AF_INET, SOCK_DGRAM, IPPROTO_IP);

综上,转换成go语言中系统调用写法

fd, _, err := syscall.RawSyscall(syscall.SYS_SOCKET, syscall.AF_INET, syscall.SOCK_DGRAM, syscall.IPPROTO_IP)

此时即生成了的socket套接字fd
我们传给int ioctl(int d, int request, …);函数作为第一个参数,第二个参数request操作的类型我们传入SIOCETHTOOL,获取ethtool信息
SIOCETHTOOL 在源码中宏定义为

#define SIOCETHTOOL     0x8946

第三个参数为struct ifreq结构内存地址
Struct ifreq结构如下:

Struct ifreq{
Char ifr_name[IFNAMSIZ];
Union{
    Struct  sockaddr  ifru_addr;
    Struct  sockaddr  ifru_dstaddr;
    Struct  sockaddr  ifru_broadaddr;
    Struct  sockaddr  ifru_netmask;
    Struct  sockaddr  ifru_hwaddr;
    Short  ifru_flags;
    Int     ifru_metric;
    Caddr_t ifru_data;
}ifr_ifru;
};
#define ifr_addr        ifr_ifru.ifru_addr
#define ifr_broadaddr   ifr_ifru.ifru_broadadd
#define ifr_hwaddr      ifr_ifru_hwaddr

 

 

综上,linux调用ioctl函数如下:

fd = socket(AF_INET, SOCK_DGRAM, IPPROTO_IP);
ioctl(fd, SIOCETHTOOL, &ifreq);

go语言:

fd, _, err := syscall.RawSyscall(syscall.SYS_SOCKET, syscall.AF_INET, syscall.SOCK_DGRAM, syscall.IPPROTO_IP)
if err != 0 {
        return syscall.Errno(err)
    }

_, _, ep := syscall.Syscall(syscall.SYS_IOCTL, uintptr(e.fd), SIOCETHTOOL, uintptr(unsafe.Pointer(&ifreq)))
if ep != 0 {
        return syscall.Errno(ep)
    }

 

 

和调度的交互

 

这里只列出Syscall和RawSyscall的源码:

//Syscall
TEXT ·Syscall(SB),NOSPLIT,$0-56
    CALL    runtime·entersyscall(SB)
    MOVQ    a1+8(FP), DI
    MOVQ    a2+16(FP), SI
    MOVQ    a3+24(FP), DX
    MOVQ    $0, R10
    MOVQ    $0, R8
    MOVQ    $0, R9
    MOVQ    trap+0(FP), AX    // syscall entry
    SYSCALL
    CMPQ    AX, $0xfffffffffffff001
    JLS    ok
    MOVQ    $-1, r1+32(FP)
    MOVQ    $0, r2+40(FP)
    NEGQ    AX
    MOVQ    AX, err+48(FP)
    CALL    runtime·exitsyscall(SB)
    RET
ok:
    MOVQ    AX, r1+32(FP)
    MOVQ    DX, r2+40(FP)
    MOVQ    $0, err+48(FP)
    CALL    runtime·exitsyscall(SB)
    RET

 

//RawSyscall
TEXT ·RawSyscall(SB),NOSPLIT,$0-56
    MOVQ    a1+8(FP), DI
    MOVQ    a2+16(FP), SI
    MOVQ    a3+24(FP), DX
    MOVQ    $0, R10
    MOVQ    $0, R8
    MOVQ    $0, R9
    MOVQ    trap+0(FP), AX    // syscall entry
    SYSCALL
    CMPQ    AX, $0xfffffffffffff001
    JLS    ok1
    MOVQ    $-1, r1+32(FP)
    MOVQ    $0, r2+40(FP)
    NEGQ    AX
    MOVQ    AX, err+48(FP)
    RET
ok1:
    MOVQ    AX, r1+32(FP)
    MOVQ    DX, r2+40(FP)
    MOVQ    $0, err+48(FP)
    RET

 

Syscall和RawSyscall的实现比较典型,可以看到这两个实现最主要的区别在于:
Syscall在进入系统调用的时候,调用了runtime·entersyscall(SB)函数,在结束系统调用的时候调用了runtime·exitsyscall(SB)。做到进入和退出syscall的时候通知runtime。

这两个函数runtime·entersyscall和runtime·exitsyscall的实现在proc.go文件里面。其实在runtime·entersyscall函数里面,通知系统调用时候,是会将g的M的P解绑,P可以去继续获取M执行其余的g,这样提升效率。

所以如果用户代码使用了 RawSyscall 来做一些阻塞的系统调用,是有可能阻塞其它的 g 的。RawSyscall 只是为了在执行那些一定不会阻塞的系统调用时,能节省两次对 runtime 的函数调用消耗

runtime·entersyscall和runtime·exitsyscall这两个函数也是与scheduler交互的地方,后面会对源码进行分析

 

 

运行时支持

runtime·entersyscallruntime·exitsyscallsrc/pkg/runtime/proc.c
src/pkg/runtime/runtime.h
  1.  
    void runtime·entersyscall(void);
  2.  
    void runtime·entersyscallblock(void);
  3.  
    void runtime·exitsyscall(void);
  4.  
     
void runtime·entersyscallblock(void)
void runtime·exitsyscall(void)
  1.  
    void ·entersyscall(int dummy) { ... }
  2.  
    void ·entersyscallblock(int dummy) { ... }
  3.  
     
runtime·entersyscallruntime·entersyscallblock·entersyscall·entersyscallblockdummy

runtime·entersyscall

好了,我们回到函数实现分析上来,看看进入系统调用前,runtime究竟都做了那些特别处理。下面将这个函数分成3段进行分析:

dummysaveg->sched.spg->sched.pcsyscallspsyscallpcsyscallstacksyscallguardentersyscallGsyscall
  1.  
    #pragma textflag NOSPLIT
  2.  
    void
  3.  
    ·entersyscall(int32 dummy)
  4.  
    {
  5.  
    // Disable preemption because during this function g is in Gsyscall status,
  6.  
    // but can have inconsistent g->sched, do not let GC observe it.
  7.  
    m->locks++;
  8.  
     
  9.  
    // Leave SP around for GC and traceback.
  10.  
    save(runtime·getcallerpc(&dummy), runtime·getcallersp(&dummy));
  11.  
    g->syscallsp = g->sched.sp;
  12.  
    g->syscallpc = g->sched.pc;
  13.  
    g->syscallstack = g->stackbase;
  14.  
    g->syscallguard = g->stackguard;
  15.  
    g->status = Gsyscall;
  16.  
    if(g->syscallsp < g->syscallguard-StackGuard || g->syscallstack < g->syscallsp) {
  17.  
    // runtime·printf("entersyscall inconsistent %p [%p,%p]\n",
  18.  
    // g->syscallsp, g->syscallguard-StackGuard, g->syscallstack);
  19.  
    runtime·throw("entersyscall");
  20.  
    }
  21.  
     
sysmonsysmon
  1.  
    if(runtime·atomicload(&runtime·sched.sysmonwait)) { // TODO: fast atomic
  2.  
    runtime·lock(&runtime·sched);
  3.  
    if(runtime·atomicload(&runtime·sched.sysmonwait)) {
  4.  
    runtime·atomicstore(&runtime·sched.sysmonwait, 0);
  5.  
    runtime·notewakeup(&runtime·sched.sysmonnote);
  6.  
    }
  7.  
    runtime·unlock(&runtime·sched);
  8.  
    save(runtime·getcallerpc(&dummy), runtime·getcallersp(&dummy));
  9.  
    }
  10.  
     
mcachemPsyscallPsyscallPgcstopg->stackguard0 = StackPreemptmorestack
  1.  
    m->mcache = nil;
  2.  
    m->p->m = nil;
  3.  
    runtime·atomicstore(&m->p->status, Psyscall);
  4.  
    if(runtime·sched.gcwaiting) {
  5.  
    runtime·lock(&runtime·sched);
  6.  
    if (runtime·sched.stopwait > 0 && runtime·cas(&m->p->status, Psyscall, Pgcstop)) {
  7.  
    if(--runtime·sched.stopwait == 0)
  8.  
    runtime·notewakeup(&runtime·sched.stopnote);
  9.  
    }
  10.  
    runtime·unlock(&runtime·sched);
  11.  
    save(runtime·getcallerpc(&dummy), runtime·getcallersp(&dummy));
  12.  
    }
  13.  
     
  14.  
    // Goroutines must not split stacks in Gsyscall status (it would corrupt g->sched).
  15.  
    // We set stackguard to StackPreempt so that first split stack check calls morestack.
  16.  
    // Morestack detects this case and throws.
  17.  
    g->stackguard0 = StackPreempt;
  18.  
    m->locks--;
  19.  
    }
  20.  
     
runtime·lock(&runtime.sched)runtime·unlock(&runtime·sched)save

runtime·entersyscallblock

·entersyscall·entersyscallblocksysmon
·entersyscall
  1.  
    #pragma textflag NOSPLIT
  2.  
    void
  3.  
    ·entersyscallblock(int32 dummy)
  4.  
    {
  5.  
    P *p;
  6.  
     
  7.  
    m->locks++; // see comment in entersyscall
  8.  
     
  9.  
    // Leave SP around for GC and traceback.
  10.  
    save(runtime·getcallerpc(&dummy), runtime·getcallersp(&dummy));
  11.  
    g->syscallsp = g->sched.sp;
  12.  
    g->syscallpc = g->sched.pc;
  13.  
    g->syscallstack = g->stackbase;
  14.  
    g->syscallguard = g->stackguard;
  15.  
    g->status = Gsyscall;
  16.  
    if(g->syscallsp < g->syscallguard-StackGuard || g->syscallstack < g->syscallsp) {
  17.  
    // runtime·printf("entersyscall inconsistent %p [%p,%p]\n",
  18.  
    // g->syscallsp, g->syscallguard-StackGuard, g->syscallstack);
  19.  
    runtime·throw("entersyscallblock");
  20.  
    }
  21.  
     
Pidle
  1.  
    p = releasep();
  2.  
    handoffp(p);
  3.  
    if(g->isbackground) // do not consider blocked scavenger for deadlock detection
  4.  
    incidlelocked(1);
  5.  
     
  6.  
    // Resave for traceback during blocked call.
  7.  
    save(runtime·getcallerpc(&dummy), runtime·getcallersp(&dummy));
  8.  
     
  9.  
    g->stackguard0 = StackPreempt; // see comment in entersyscall
  10.  
    m->locks--;
  11.  
    }
  12.  
     
syscallruntime·entersyscallruntime·entersyscallblock
runtime·entersyscallblockbool runtime.notetsleepg(Note *n, int64 ns)NoteNote

上述机制在runtime中多有使用,比如在“定时器”模块中 —— 后面有机会会详细介绍。

runtime·exitsyscall

该函数主要的功能是从syscall状态恢复,其结构比较清晰,主要分为两个步骤:

exitsyscallfasttruefalseruntime·exitsyscall
  1.  
    // The goroutine g exited its system call.
  2.  
    // Arrange for it to run on a cpu again.
  3.  
    // This is called only from the go syscall library, not
  4.  
    // from the low-level system calls used by the runtime.
  5.  
    #pragma textflag NOSPLIT
  6.  
    void
  7.  
    runtime·exitsyscall(void)
  8.  
    {
  9.  
    m->locks++; // see comment in entersyscall
  10.  
     
  11.  
    if(g->isbackground) // do not consider blocked scavenger for deadlock detection
  12.  
    incidlelocked(-1);
  13.  
     
  14.  
    if(exitsyscallfast()) {
  15.  
    // There's a cpu for us, so we can run.
  16.  
    m->p->syscalltick++;
  17.  
    g->status = Grunning;
  18.  
    // Garbage collector isn't running (since we are),
  19.  
    // so okay to clear gcstack and gcsp.
  20.  
    g->syscallstack = (uintptr)nil;
  21.  
    g->syscallsp = (uintptr)nil;
  22.  
    m->locks--;
  23.  
    if(g->preempt) {
  24.  
    // restore the preemption request in case we've cleared it in newstack
  25.  
    g->stackguard0 = StackPreempt;
  26.  
    } else {
  27.  
    // otherwise restore the real stackguard, we've spoiled it in entersyscall/entersyscallblock
  28.  
    g->stackguard0 = g->stackguard;
  29.  
    }
  30.  
    return;
  31.  
    }
  32.  
     
  33.  
    m->locks--;
  34.  
     
exitsyscallfastruntime.mcall
  1.  
    // Call the scheduler.
  2.  
    runtime·mcall(exitsyscall0);
  3.  
     
  4.  
    // Scheduler returned, so we're allowed to run now.
  5.  
    // Delete the gcstack information that we left for
  6.  
    // the garbage collector during the system call.
  7.  
    // Must wait until now because until gosched returns
  8.  
    // we don't know for sure that the garbage collector
  9.  
    // is not running.
  10.  
    g->syscallstack = (uintptr)nil;
  11.  
    g->syscallsp = (uintptr)nil;
  12.  
    m->p->syscalltick++;
  13.  
    }
  14.  
     
exitsyscall0runtime.mcallsyscallstacksyscallspsyscalltick

一点说明

Go语言之所以设计了M及P这两个概念,并对执行syscall的线程进行特别处理,适当进行M和P的解耦,主要是为了提高并发度,降低频繁、长时间的阻塞syscall带来的问题。但是必须意识到,这种机制本身也存在一定的开销,比如任务迁移可能影响CACHE、TLB的性能。

·entersyscall

对于runtime中的一些底层syscall,比如所有的底层锁操作 —— 在Linux中使用的是Futex机制 —— 相应的Lock/Unlock操作都使用了底层系统调用,此时线程会直接调用syscall而不需要其他的操作,这样主要是保证底层代码的高效执行。

syscallRawSyscallruntime·entersyscallruntime·exitsyscall