panic on win7_32bit

[xujiachuang0723]

I encountered a panic while using winio on a 32-bit, x86 Windows 7 environment. The usage is quite straightforward—simply continuously read from and write to the pipe in a loop.

The panic info:

fatal error: unexpected signal during runtime execution
[signal 0xc0000005 code=0x0 addr=0xa04e000 pc=0x461df1]

runtime stack:
runtime.throw({0x51b4fa, 0x2a})
        C:/Program Files/Go/src/runtime/panic.go:1047 +0x4d fp=0x9f0ff14 sp=0x9f0ff00 pc=0x43525d
runtime.sigpanic()
        C:/Program Files/Go/src/runtime/signal_windows.go:251 +0x2ba fp=0x9f0ff38 sp=0x9f0ff14 pc=0x448fda
runtime.asmstdcall()
        C:/Program Files/Go/src/runtime/sys_windows_386.s:26 +0x21 fp=0x9f0ff3c sp=0x9f0ff38 pc=0x461df1

goroutine 6 [syscall, locked to thread]:
runtime.cgocall(0x461dd0, 0xa00ca90)
        C:/Program Files/Go/src/runtime/cgocall.go:157 +0x58 fp=0xa032f50 sp=0xa032f38 pc=0x403af8
syscall.SyscallN(0x76a050a8, {0xa04e000, 0x5, 0x5})
        C:/Program Files/Go/src/runtime/syscall_windows.go:557 +0x11d fp=0xa032f7c sp=0xa032f50 pc=0x45f45d
github.com/Microsoft/go-winio.getQueuedCompletionStatus(0xe4, 0xa032fd0, 0xa032fcc, 0xa032fd4, 0xffffffff)
        C:/Users/User/go/pkg/mod/github.com/!microsoft/[email protected]/zsyscall_windows.go:327 +0xba fp=0xa032fb0 sp=0xa032f7c pc=0x4e843a
github.com/Microsoft/go-winio.ioCompletionProcessor(0xe4)
        C:/Users/User/go/pkg/mod/github.com/!microsoft/[email protected]/file.go:163 +0x9e fp=0xa032fe8 sp=0xa032fb0 pc=0x4e687e
github.com/Microsoft/go-winio.initIO.func1()
        C:/Users/User/go/pkg/mod/github.com/!microsoft/[email protected]/file.go:59 +0x27 fp=0xa032ff0 sp=0xa032fe8 pc=0x4e6447
runtime.goexit()
        C:/Program Files/Go/src/runtime/asm_386.s:1326 +0x1 fp=0xa032ff4 sp=0xa032ff0 pc=0x461161
created by github.com/Microsoft/go-winio.initIO
        C:/Users/User/go/pkg/mod/github.com/!microsoft/[email protected]/file.go:59 +0x84

goroutine 1 [select]:
runtime.gopark(0x51da88, 0x0, 0x9, 0x18, 0x1)
        C:/Program Files/Go/src/runtime/proc.go:381 +0xff fp=0xa4a7c80 sp=0xa4a7c6c pc=0x437caf
runtime.selectgo(0xa4a7dd0, 0xa4a7da0, 0x0, 0x0, 0x2, 0x1)
        C:/Program Files/Go/src/runtime/select.go:327 +0xbc0 fp=0xa4a7d70 sp=0xa4a7c80 pc=0x446c70
github.com/Microsoft/go-winio.(*win32File).asyncIO(0xa050120, 0xa0bea20, 0xa050184, 0x0, {0x545bc4, 0x5dd298})
        C:/Users/User/go/pkg/mod/github.com/!microsoft/[email protected]/file.go:195 +0x2ec fp=0xa4a7e38 sp=0xa4a7d70 pc=0x4e6bac
github.com/Microsoft/go-winio.(*win32File).Write(0xa050120, {0xa152000, 0x2800, 0x2800})
        C:/Users/User/go/pkg/mod/github.com/!microsoft/[email protected]/file.go:264 +0x151 fp=0xa4a7e84 sp=0xa4a7e38 pc=0x4e7191
github.com/Microsoft/go-winio.(*win32Pipe).Write(0xa0081b0, {0xa152000, 0x2800, 0x2800})
        <autogenerated>:1 +0x42 fp=0xa4a7ea4 sp=0xa4a7e84 pc=0x4e9ad2
main.main.func1({0xa06a000, 0x2800})
        D:/code/tmp/gopractice/client/client.go:99 +0x92 fp=0xa4a7ef4 sp=0xa4a7ea4 pc=0x4eab02
main.main.func2({0xa06a000, 0x2800})
        D:/code/tmp/gopractice/client/client.go:113 +0x49 fp=0xa4a7f10 sp=0xa4a7ef4 pc=0x4eaa19
main.main()
        D:/code/tmp/gopractice/client/client.go:125 +0x30a fp=0xa4a7fc4 sp=0xa4a7f10 pc=0x4ea8aa
runtime.main()
        C:/Program Files/Go/src/runtime/proc.go:250 +0x22e fp=0xa4a7ff0 sp=0xa4a7fc4 pc=0x43787e
runtime.goexit()
        C:/Program Files/Go/src/runtime/asm_386.s:1326 +0x1 fp=0xa4a7ff4 sp=0xa4a7ff0 pc=0x461161

goroutine 2 [force gc (idle), 5728 minutes]:
runtime.gopark(0x51da58, 0x5f3fb8, 0x11, 0x14, 0x1)
        C:/Program Files/Go/src/runtime/proc.go:381 +0xff fp=0xa037fdc sp=0xa037fc8 pc=0x437caf
runtime.goparkunlock(...)
        C:/Program Files/Go/src/runtime/proc.go:387
runtime.forcegchelper()
        C:/Program Files/Go/src/runtime/proc.go:305 +0xcf fp=0xa037ff0 sp=0xa037fdc pc=0x437adf
runtime.goexit()
        C:/Program Files/Go/src/runtime/asm_386.s:1326 +0x1 fp=0xa037ff4 sp=0xa037ff0 pc=0x461161
created by runtime.init.5
        C:/Program Files/Go/src/runtime/proc.go:293 +0x23

goroutine 3 [GC sweep wait]:
runtime.gopark(0x51da58, 0x5f4130, 0xc, 0x14, 0x1)
        C:/Program Files/Go/src/runtime/proc.go:381 +0xff fp=0xa038fcc sp=0xa038fb8 pc=0x437caf
runtime.goparkunlock(...)
        C:/Program Files/Go/src/runtime/proc.go:387
runtime.bgsweep(0xa012200)
        C:/Program Files/Go/src/runtime/mgcsweep.go:319 +0xf2 fp=0xa038fe8 sp=0xa038fcc pc=0x422932
runtime.gcenable.func1()
        C:/Program Files/Go/src/runtime/mgc.go:178 +0x27 fp=0xa038ff0 sp=0xa038fe8 pc=0x414e17
runtime.goexit()
        C:/Program Files/Go/src/runtime/asm_386.s:1326 +0x1 fp=0xa038ff4 sp=0xa038ff0 pc=0x461161
created by runtime.gcenable
        C:/Program Files/Go/src/runtime/mgc.go:178 +0x7c

goroutine 4 [sleep]:
runtime.gopark(0x51da58, 0x5f42e0, 0x13, 0x13, 0x2)
        C:/Program Files/Go/src/runtime/proc.go:381 +0xff fp=0xa039f64 sp=0xa039f50 pc=0x437caf
runtime.goparkunlock(...)
        C:/Program Files/Go/src/runtime/proc.go:387
runtime.(*scavengerState).sleep(0x5f42e0, 0x40fd4c0000000000)
        C:/Program Files/Go/src/runtime/mgcscavenge.go:479 +0x167 fp=0xa039fcc sp=0xa039f64 pc=0x4204d7
runtime.bgscavenge(0xa012200)
        C:/Program Files/Go/src/runtime/mgcscavenge.go:637 +0xa5 fp=0xa039fe8 sp=0xa039fcc pc=0x420985
runtime.gcenable.func2()
        C:/Program Files/Go/src/runtime/mgc.go:179 +0x27 fp=0xa039ff0 sp=0xa039fe8 pc=0x414dd7
runtime.goexit()
        C:/Program Files/Go/src/runtime/asm_386.s:1326 +0x1 fp=0xa039ff4 sp=0xa039ff0 pc=0x461161
created by runtime.gcenable
        C:/Program Files/Go/src/runtime/mgc.go:179 +0xc1

goroutine 5 [finalizer wait, 5728 minutes]:
runtime.gopark(0x51d994, 0x618d38, 0x10, 0x14, 0x1)
        C:/Program Files/Go/src/runtime/proc.go:381 +0xff fp=0xa036f94 sp=0xa036f80 pc=0x437caf
runtime.runfinq()
        C:/Program Files/Go/src/runtime/mfinal.go:193 +0xf4 fp=0xa036ff0 sp=0xa036f94 pc=0x413fa4
runtime.goexit()
        C:/Program Files/Go/src/runtime/asm_386.s:1326 +0x1 fp=0xa036ff4 sp=0xa036ff0 pc=0x461161
created by runtime.createfing
        C:/Program Files/Go/src/runtime/mfinal.go:163 +0x60

goroutine 7 [select]:
runtime.gopark(0x51da88, 0x0, 0x9, 0x18, 0x1)
        C:/Program Files/Go/src/runtime/proc.go:381 +0xff fp=0xa4a2c4c sp=0xa4a2c38 pc=0x437caf
runtime.selectgo(0xa4a2d9c, 0xa4a2d6c, 0x0, 0x0, 0x2, 0x1)
        C:/Program Files/Go/src/runtime/select.go:327 +0xbc0 fp=0xa4a2d3c sp=0xa4a2c4c pc=0x446c70
github.com/Microsoft/go-winio.(*win32File).asyncIO(0xa050120, 0xa0be2d0, 0xa050158, 0x0, {0x545bc4, 0x5dd298})
        C:/Users/User/go/pkg/mod/github.com/!microsoft/[email protected]/file.go:195 +0x2ec fp=0xa4a2e04 sp=0xa4a2d3c pc=0x4e6bac
github.com/Microsoft/go-winio.(*win32File).Read(0xa050120, {0xa12c000, 0x2800, 0x2800})
        C:/Users/User/go/pkg/mod/github.com/!microsoft/[email protected]/file.go:238 +0x125 fp=0xa4a2e60 sp=0xa4a2e04 pc=0x4e6ea5
github.com/Microsoft/go-winio.(*win32Pipe).Read(0xa0081b0, {0xa12c000, 0x2800, 0x2800})
        <autogenerated>:1 +0x42 fp=0xa4a2e80 sp=0xa4a2e60 pc=0x4e99e2
bufio.(*Reader).Read(0xa128000, {0xa12c000, 0x2800, 0x2800})
        C:/Program Files/Go/src/bufio/bufio.go:223 +0x106 fp=0xa4a2ea4 sp=0xa4a2e80 pc=0x46f8c6
io.ReadAtLeast({0x54596c, 0xa128000}, {0xa12c000, 0x2800, 0x2800}, 0x2800)
        C:/Program Files/Go/src/io/io.go:332 +0x97 fp=0xa4a2ed4 sp=0xa4a2ea4 pc=0x468c07
io.ReadFull(...)
        C:/Program Files/Go/src/io/io.go:351
main.handlerConnection({0x545ee4, 0xa0081b0}, 0xa012540)
        D:/code/tmp/gopractice/client/client.go:70 +0x289 fp=0xa4a2fe0 sp=0xa4a2ed4 pc=0x4ea2f9
main.main.func3()
        D:/code/tmp/gopractice/client/client.go:122 +0x35 fp=0xa4a2ff0 sp=0xa4a2fe0 pc=0x4ea9b5
runtime.goexit()
        C:/Program Files/Go/src/runtime/asm_386.s:1326 +0x1 fp=0xa4a2ff4 sp=0xa4a2ff0 pc=0x461161
created by main.main
        D:/code/tmp/gopractice/client/client.go:122 +0x217

goroutine 8 [GC worker (idle)]:
runtime.gopark(0x51d9a0, 0xa014210, 0x1a, 0x14, 0x0)
        C:/Program Files/Go/src/runtime/proc.go:381 +0xff fp=0xa034f94 sp=0xa034f80 pc=0x437caf
runtime.gcBgMarkWorker()
        C:/Program Files/Go/src/runtime/mgc.go:1275 +0xee fp=0xa034ff0 sp=0xa034f94 pc=0x41722e
runtime.goexit()
        C:/Program Files/Go/src/runtime/asm_386.s:1326 +0x1 fp=0xa034ff4 sp=0xa034ff0 pc=0x461161
created by runtime.gcBgMarkStartWorkers
        C:/Program Files/Go/src/runtime/mgc.go:1199 +0x25
		

goroutine 18 [GC worker (idle)]:
runtime.gopark(0x51d9a0, 0xa014228, 0x1a, 0x14, 0x0)
        C:/Program Files/Go/src/runtime/proc.go:381 +0xff fp=0xa033f94 sp=0xa033f80 pc=0x437caf
runtime.gcBgMarkWorker()
        C:/Program Files/Go/src/runtime/mgc.go:1275 +0xee fp=0xa033ff0 sp=0xa033f94 pc=0x41722e
runtime.goexit()
        C:/Program Files/Go/src/runtime/asm_386.s:1326 +0x1 fp=0xa033ff4 sp=0xa033ff0 pc=0x461161
created by runtime.gcBgMarkStartWorkers
        C:/Program Files/Go/src/runtime/mgc.go:1199 +0x25

goroutine 9 [GC worker (idle)]:
runtime.gopark(0x51d9a0, 0xa49c000, 0x1a, 0x14, 0x0)
        C:/Program Files/Go/src/runtime/proc.go:381 +0xff fp=0xa035f94 sp=0xa035f80 pc=0x437caf
runtime.gcBgMarkWorker()
        C:/Program Files/Go/src/runtime/mgc.go:1275 +0xee fp=0xa035ff0 sp=0xa035f94 pc=0x41722e
runtime.goexit()
        C:/Program Files/Go/src/runtime/asm_386.s:1326 +0x1 fp=0xa035ff4 sp=0xa035ff0 pc=0x461161
created by runtime.gcBgMarkStartWorkers
        C:/Program Files/Go/src/runtime/mgc.go:1199 +0x25

goroutine 34 [GC worker (idle)]:
runtime.gopark(0x51d9a0, 0xa49c018, 0x1a, 0x14, 0x0)
        C:/Program Files/Go/src/runtime/proc.go:381 +0xff fp=0xa4a4f94 sp=0xa4a4f80 pc=0x437caf
runtime.gcBgMarkWorker()
        C:/Program Files/Go/src/runtime/mgc.go:1275 +0xee fp=0xa4a4ff0 sp=0xa4a4f94 pc=0x41722e
runtime.goexit()
        C:/Program Files/Go/src/runtime/asm_386.s:1326 +0x1 fp=0xa4a4ff4 sp=0xa4a4ff0 pc=0x461161
created by runtime.gcBgMarkStartWorkers
        C:/Program Files/Go/src/runtime/mgc.go:1199 +0x25

client.go(who panic):

package main

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"net"
	"net/url"
	"runtime"
	"strings"
	"time"

	winio "github.com/Microsoft/go-winio"
)

func ParseAddress(address string) (scheme string, addr string, err error) {
	if strings.HasPrefix(address, "pipe://") {
		scheme = "pipe"
		addr = address[len("pipe://"):]
		return
	}
	if strings.HasPrefix(address, "unix://") {
		scheme = "unix"
		addr = address[len("unix://"):]
		return
	}
	uri, err := url.Parse(address)
	if err != nil || len(uri.Scheme) == 0 || len(uri.Host) == 0 {
		_, _, err = net.SplitHostPort(address)
		if err != nil {
			return "", "", errors.New("invalid address:" + address)
		}
		scheme = "tcp"
		addr = address
		err = nil
		return
	}
	scheme = uri.Scheme
	addr = uri.Host
	err = nil
	return
}

func ValidScheme(scheme string) bool {
	scheme = strings.ToLower(scheme)
	if runtime.GOOS == "windows" {
		return scheme == "tcp" || scheme == "udp" || scheme == "pipe"
	}
	return scheme == "tcp" || scheme == "udp" || scheme == "unix"
}

func DialTimeout(scheme string, address string, timeout time.Duration) (net.Conn, error) {
	if !ValidScheme(scheme) {
		return nil, errors.New("invalid scheme:" + scheme)
	}
	if scheme == "pipe" {
		return winio.DialPipe(address, &timeout)
	}
	return net.DialTimeout(scheme, address, timeout)
}

func handlerConnection(con net.Conn, ch chan struct{}) {
	defer con.Close()
	bio := bufio.NewReader(con)
	buf := make([]byte, 10*1024)
	cnt := 0
	for {
		con.SetDeadline(time.Now().Add(time.Second * 60))
		_, err := io.ReadFull(bio, buf)
		if err != nil {
			fmt.Println("ReadFull fail: ", err)
			break
		}
		cnt++
		if cnt%100000 == 0 {
			fmt.Println(time.Now().UTC(), "Read data: ", cnt, "...")
		}
	}
	ch <- struct{}{}


func main() {
	scheme, addr, err := ParseAddress("pipe://\\\\.\\pipe\\UEOS_TEST_WINIO")
	if err != nil {
		fmt.Println(err)
		return
	}
	con, err := DialTimeout(scheme, addr, time.Second*6)
	if err != nil {
		fmt.Println(err)
		return
	}

	writeFull := func(data string) error {
		write := 0
		pp := []byte(data)
		for {
			n, err := con.Write(pp)
			if err != nil {
				fmt.Println("con.Write fail: ", err)
				return err
			}
			write += n
			if n >= len(pp) {
				return err
			}
			pp = pp[n:]
		}
	}
	sendMsgs := func(data string) error {
		for i := 0; i < 10000; i++ {
			err := writeFull(data)
			if err != nil {
				return err
			}
		}
		time.Sleep(time.Millisecond * 7)
		return nil
	}
	ch := make(chan struct{})
	go handlerConnection(con, ch)
	data := strings.Repeat("x", 10*1024)
	for {
		err := sendMsgs(data)
		if err != nil {
			break
		}
		fmt.Println(time.Now().UTC(), "sendMsgs...")
	}
	con.Close()
	_ = <-ch
}

server.go:

package main

import (
	"bufio"
	"fmt"
	"io"
	"net"
	"strings"
	"time"

	winio "github.com/Microsoft/go-winio"
)

func handlerConnection(con net.Conn, ch chan struct{}) {
	defer con.Close()
	bio := bufio.NewReader(con)
	buf := make([]byte, 10*1024)
	cnt := 0
	for {
		con.SetDeadline(time.Now().Add(time.Second * 60))
		_, err := io.ReadFull(bio, buf)
		if err != nil {
			fmt.Println("ReadFull fail: ", err)
			break
		}
		cnt++
		if cnt%100000 == 0 {
			fmt.Println(time.Now().UTC(), "Read data: ", cnt, "...")
		}
	}
	ch <- struct{}{}
}

func oneConn(con net.Conn) {
	writeFull := func(data string) error {
		write := 0
		pp := []byte(data)
		for {
			n, err := con.Write(pp)
			if err != nil {
				fmt.Println("con.Write fail: ", err)
				return err
			}
			write += n
			if n >= len(pp) {
				return err
			}
			pp = pp[n:]
		}
	}
	sendMsgs := func(data string) error {
		for i := 0; i < 10000; i++ {
			err := writeFull(data)
			if err != nil {
				return err
			}
		}
		time.Sleep(time.Millisecond * 7)
		return nil
	}
	ch := make(chan struct{})
	go handlerConnection(con, ch)
	data := strings.Repeat("x", 10*1024)
	for {
		err := sendMsgs(data)
		if err != nil {
			break
		}
		fmt.Println(time.Now().UTC(), "sendMsgs...")
	}
	con.Close()
	_ = <-ch
}


func main() {
	pls, err := winio.ListenPipe("\\\\.\\pipe\\UEOS_TEST_WINIO", &winio.PipeConfig{
		SecurityDescriptor: "D:(A;;GA;;;WD)",
		InputBufferSize:    1024 * 1024 * 4,
		OutputBufferSize:   1024 * 1024 * 4,
	})
	if err != nil {
		fmt.Println("listen pipe failed, addr err: ", err)
		return
	}

	fmt.Println(time.Now().UTC(), "Center Server start...")
	for {
		con, err := pls.Accept()
		if err != nil {
			fmt.Println(err)
			return
		}
		fmt.Println(time.Now().UTC(), "new connection accept")
		go oneConn(con)
	}
}

[xujiachuang0723]

The occurrence rate is very low.

[xujiachuang0723]

I've found the cause of the problem. In file.go under asyncIO, if err != windows.ERROR_IO_PENDING, it directly returns (besides success, there are other non-pending errors). This can lead to the memory invalidation of c *ioOperation. Meanwhile, the getQueuedCompletionStatus called by ioCompletionProcessor might still use this c *ioOperation, which can cause the process to crash.