I use syscall prlimit to set resource limit to process, it works for limit CPU time but when test memory usage, I come with the problem.

package sandbox

import (
    "syscall"
    "unsafe"
)

func prLimit(pid int, limit uintptr, rlimit *syscall.Rlimit) error {
    _, _, errno := syscall.RawSyscall6(syscall.SYS_PRLIMIT64, uintptr(pid), limit, uintptr(unsafe.Pointer(rlimit)), 0, 0, 0)
    var err error
    if errno != 0 {
        err = errno
        return err
    } else {
        return nil
    }
}

and it's my test.

func TestMemoryLimit(t *testing.T) {
    proc, err := os.StartProcess("test/memo", []string{"memo"}, &os.ProcAttr{})
    if err != nil {
        panic(err)
    }
    defer proc.Kill()
    var rlimit syscall.Rlimit
    rlimit.Cur = 10
    rlimit.Max = 10 + 1024
    prLimit(proc.Pid, syscall.RLIMIT_DATA, &rlimit)
    status, err := proc.Wait()
    if status.Success() {
        t.Fatal("memory test failed")
    }
}

this is memo:

package main

func main() {
    var a [10000][]int
    for i := 0; i < 1000; i++ {
        a[i] = make([]int, 1024)
    }
}

I make a large amount of memory and set only 10 bytes for the memory , but it won't signal segment fault signal any way.