Black Lives Matter. Support the Equal Justice Initiative.

Source file src/runtime/panic.go

Documentation: runtime

     1  // Copyright 2014 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package runtime
     6  
     7  import (
     8  	"internal/abi"
     9  	"internal/goexperiment"
    10  	"runtime/internal/atomic"
    11  	"runtime/internal/sys"
    12  	"unsafe"
    13  )
    14  
    15  // We have two different ways of doing defers. The older way involves creating a
    16  // defer record at the time that a defer statement is executing and adding it to a
    17  // defer chain. This chain is inspected by the deferreturn call at all function
    18  // exits in order to run the appropriate defer calls. A cheaper way (which we call
    19  // open-coded defers) is used for functions in which no defer statements occur in
    20  // loops. In that case, we simply store the defer function/arg information into
    21  // specific stack slots at the point of each defer statement, as well as setting a
    22  // bit in a bitmask. At each function exit, we add inline code to directly make
    23  // the appropriate defer calls based on the bitmask and fn/arg information stored
    24  // on the stack. During panic/Goexit processing, the appropriate defer calls are
    25  // made using extra funcdata info that indicates the exact stack slots that
    26  // contain the bitmask and defer fn/args.
    27  
    28  // Check to make sure we can really generate a panic. If the panic
    29  // was generated from the runtime, or from inside malloc, then convert
    30  // to a throw of msg.
    31  // pc should be the program counter of the compiler-generated code that
    32  // triggered this panic.
    33  func panicCheck1(pc uintptr, msg string) {
    34  	if sys.GoarchWasm == 0 && hasPrefix(funcname(findfunc(pc)), "runtime.") {
    35  		// Note: wasm can't tail call, so we can't get the original caller's pc.
    36  		throw(msg)
    37  	}
    38  	// TODO: is this redundant? How could we be in malloc
    39  	// but not in the runtime? runtime/internal/*, maybe?
    40  	gp := getg()
    41  	if gp != nil && gp.m != nil && gp.m.mallocing != 0 {
    42  		throw(msg)
    43  	}
    44  }
    45  
    46  // Same as above, but calling from the runtime is allowed.
    47  //
    48  // Using this function is necessary for any panic that may be
    49  // generated by runtime.sigpanic, since those are always called by the
    50  // runtime.
    51  func panicCheck2(err string) {
    52  	// panic allocates, so to avoid recursive malloc, turn panics
    53  	// during malloc into throws.
    54  	gp := getg()
    55  	if gp != nil && gp.m != nil && gp.m.mallocing != 0 {
    56  		throw(err)
    57  	}
    58  }
    59  
    60  // Many of the following panic entry-points turn into throws when they
    61  // happen in various runtime contexts. These should never happen in
    62  // the runtime, and if they do, they indicate a serious issue and
    63  // should not be caught by user code.
    64  //
    65  // The panic{Index,Slice,divide,shift} functions are called by
    66  // code generated by the compiler for out of bounds index expressions,
    67  // out of bounds slice expressions, division by zero, and shift by negative.
    68  // The panicdivide (again), panicoverflow, panicfloat, and panicmem
    69  // functions are called by the signal handler when a signal occurs
    70  // indicating the respective problem.
    71  //
    72  // Since panic{Index,Slice,shift} are never called directly, and
    73  // since the runtime package should never have an out of bounds slice
    74  // or array reference or negative shift, if we see those functions called from the
    75  // runtime package we turn the panic into a throw. That will dump the
    76  // entire runtime stack for easier debugging.
    77  //
    78  // The entry points called by the signal handler will be called from
    79  // runtime.sigpanic, so we can't disallow calls from the runtime to
    80  // these (they always look like they're called from the runtime).
    81  // Hence, for these, we just check for clearly bad runtime conditions.
    82  //
    83  // The panic{Index,Slice} functions are implemented in assembly and tail call
    84  // to the goPanic{Index,Slice} functions below. This is done so we can use
    85  // a space-minimal register calling convention.
    86  
    87  // failures in the comparisons for s[x], 0 <= x < y (y == len(s))
    88  func goPanicIndex(x int, y int) {
    89  	panicCheck1(getcallerpc(), "index out of range")
    90  	panic(boundsError{x: int64(x), signed: true, y: y, code: boundsIndex})
    91  }
    92  func goPanicIndexU(x uint, y int) {
    93  	panicCheck1(getcallerpc(), "index out of range")
    94  	panic(boundsError{x: int64(x), signed: false, y: y, code: boundsIndex})
    95  }
    96  
    97  // failures in the comparisons for s[:x], 0 <= x <= y (y == len(s) or cap(s))
    98  func goPanicSliceAlen(x int, y int) {
    99  	panicCheck1(getcallerpc(), "slice bounds out of range")
   100  	panic(boundsError{x: int64(x), signed: true, y: y, code: boundsSliceAlen})
   101  }
   102  func goPanicSliceAlenU(x uint, y int) {
   103  	panicCheck1(getcallerpc(), "slice bounds out of range")
   104  	panic(boundsError{x: int64(x), signed: false, y: y, code: boundsSliceAlen})
   105  }
   106  func goPanicSliceAcap(x int, y int) {
   107  	panicCheck1(getcallerpc(), "slice bounds out of range")
   108  	panic(boundsError{x: int64(x), signed: true, y: y, code: boundsSliceAcap})
   109  }
   110  func goPanicSliceAcapU(x uint, y int) {
   111  	panicCheck1(getcallerpc(), "slice bounds out of range")
   112  	panic(boundsError{x: int64(x), signed: false, y: y, code: boundsSliceAcap})
   113  }
   114  
   115  // failures in the comparisons for s[x:y], 0 <= x <= y
   116  func goPanicSliceB(x int, y int) {
   117  	panicCheck1(getcallerpc(), "slice bounds out of range")
   118  	panic(boundsError{x: int64(x), signed: true, y: y, code: boundsSliceB})
   119  }
   120  func goPanicSliceBU(x uint, y int) {
   121  	panicCheck1(getcallerpc(), "slice bounds out of range")
   122  	panic(boundsError{x: int64(x), signed: false, y: y, code: boundsSliceB})
   123  }
   124  
   125  // failures in the comparisons for s[::x], 0 <= x <= y (y == len(s) or cap(s))
   126  func goPanicSlice3Alen(x int, y int) {
   127  	panicCheck1(getcallerpc(), "slice bounds out of range")
   128  	panic(boundsError{x: int64(x), signed: true, y: y, code: boundsSlice3Alen})
   129  }
   130  func goPanicSlice3AlenU(x uint, y int) {
   131  	panicCheck1(getcallerpc(), "slice bounds out of range")
   132  	panic(boundsError{x: int64(x), signed: false, y: y, code: boundsSlice3Alen})
   133  }
   134  func goPanicSlice3Acap(x int, y int) {
   135  	panicCheck1(getcallerpc(), "slice bounds out of range")
   136  	panic(boundsError{x: int64(x), signed: true, y: y, code: boundsSlice3Acap})
   137  }
   138  func goPanicSlice3AcapU(x uint, y int) {
   139  	panicCheck1(getcallerpc(), "slice bounds out of range")
   140  	panic(boundsError{x: int64(x), signed: false, y: y, code: boundsSlice3Acap})
   141  }
   142  
   143  // failures in the comparisons for s[:x:y], 0 <= x <= y
   144  func goPanicSlice3B(x int, y int) {
   145  	panicCheck1(getcallerpc(), "slice bounds out of range")
   146  	panic(boundsError{x: int64(x), signed: true, y: y, code: boundsSlice3B})
   147  }
   148  func goPanicSlice3BU(x uint, y int) {
   149  	panicCheck1(getcallerpc(), "slice bounds out of range")
   150  	panic(boundsError{x: int64(x), signed: false, y: y, code: boundsSlice3B})
   151  }
   152  
   153  // failures in the comparisons for s[x:y:], 0 <= x <= y
   154  func goPanicSlice3C(x int, y int) {
   155  	panicCheck1(getcallerpc(), "slice bounds out of range")
   156  	panic(boundsError{x: int64(x), signed: true, y: y, code: boundsSlice3C})
   157  }
   158  func goPanicSlice3CU(x uint, y int) {
   159  	panicCheck1(getcallerpc(), "slice bounds out of range")
   160  	panic(boundsError{x: int64(x), signed: false, y: y, code: boundsSlice3C})
   161  }
   162  
   163  // failures in the conversion (*[x]T)s, 0 <= x <= y, x == cap(s)
   164  func goPanicSliceConvert(x int, y int) {
   165  	panicCheck1(getcallerpc(), "slice length too short to convert to pointer to array")
   166  	panic(boundsError{x: int64(x), signed: true, y: y, code: boundsConvert})
   167  }
   168  
   169  // Implemented in assembly, as they take arguments in registers.
   170  // Declared here to mark them as ABIInternal.
   171  func panicIndex(x int, y int)
   172  func panicIndexU(x uint, y int)
   173  func panicSliceAlen(x int, y int)
   174  func panicSliceAlenU(x uint, y int)
   175  func panicSliceAcap(x int, y int)
   176  func panicSliceAcapU(x uint, y int)
   177  func panicSliceB(x int, y int)
   178  func panicSliceBU(x uint, y int)
   179  func panicSlice3Alen(x int, y int)
   180  func panicSlice3AlenU(x uint, y int)
   181  func panicSlice3Acap(x int, y int)
   182  func panicSlice3AcapU(x uint, y int)
   183  func panicSlice3B(x int, y int)
   184  func panicSlice3BU(x uint, y int)
   185  func panicSlice3C(x int, y int)
   186  func panicSlice3CU(x uint, y int)
   187  func panicSliceConvert(x int, y int)
   188  
   189  var shiftError = error(errorString("negative shift amount"))
   190  
   191  func panicshift() {
   192  	panicCheck1(getcallerpc(), "negative shift amount")
   193  	panic(shiftError)
   194  }
   195  
   196  var divideError = error(errorString("integer divide by zero"))
   197  
   198  func panicdivide() {
   199  	panicCheck2("integer divide by zero")
   200  	panic(divideError)
   201  }
   202  
   203  var overflowError = error(errorString("integer overflow"))
   204  
   205  func panicoverflow() {
   206  	panicCheck2("integer overflow")
   207  	panic(overflowError)
   208  }
   209  
   210  var floatError = error(errorString("floating point error"))
   211  
   212  func panicfloat() {
   213  	panicCheck2("floating point error")
   214  	panic(floatError)
   215  }
   216  
   217  var memoryError = error(errorString("invalid memory address or nil pointer dereference"))
   218  
   219  func panicmem() {
   220  	panicCheck2("invalid memory address or nil pointer dereference")
   221  	panic(memoryError)
   222  }
   223  
   224  func panicmemAddr(addr uintptr) {
   225  	panicCheck2("invalid memory address or nil pointer dereference")
   226  	panic(errorAddressString{msg: "invalid memory address or nil pointer dereference", addr: addr})
   227  }
   228  
   229  // Create a new deferred function fn with siz bytes of arguments.
   230  // The compiler turns a defer statement into a call to this.
   231  //go:nosplit
   232  func deferproc(siz int32, fn *funcval) { // arguments of fn follow fn
   233  	gp := getg()
   234  	if gp.m.curg != gp {
   235  		// go code on the system stack can't defer
   236  		throw("defer on system stack")
   237  	}
   238  
   239  	if goexperiment.RegabiDefer && siz != 0 {
   240  		// TODO: Make deferproc just take a func().
   241  		throw("defer with non-empty frame")
   242  	}
   243  
   244  	// the arguments of fn are in a perilous state. The stack map
   245  	// for deferproc does not describe them. So we can't let garbage
   246  	// collection or stack copying trigger until we've copied them out
   247  	// to somewhere safe. The memmove below does that.
   248  	// Until the copy completes, we can only call nosplit routines.
   249  	sp := getcallersp()
   250  	argp := uintptr(unsafe.Pointer(&fn)) + unsafe.Sizeof(fn)
   251  	callerpc := getcallerpc()
   252  
   253  	d := newdefer(siz)
   254  	if d._panic != nil {
   255  		throw("deferproc: d.panic != nil after newdefer")
   256  	}
   257  	d.link = gp._defer
   258  	gp._defer = d
   259  	d.fn = fn
   260  	d.pc = callerpc
   261  	d.sp = sp
   262  	switch siz {
   263  	case 0:
   264  		// Do nothing.
   265  	case sys.PtrSize:
   266  		*(*uintptr)(deferArgs(d)) = *(*uintptr)(unsafe.Pointer(argp))
   267  	default:
   268  		memmove(deferArgs(d), unsafe.Pointer(argp), uintptr(siz))
   269  	}
   270  
   271  	// deferproc returns 0 normally.
   272  	// a deferred func that stops a panic
   273  	// makes the deferproc return 1.
   274  	// the code the compiler generates always
   275  	// checks the return value and jumps to the
   276  	// end of the function if deferproc returns != 0.
   277  	return0()
   278  	// No code can go here - the C return register has
   279  	// been set and must not be clobbered.
   280  }
   281  
   282  // deferprocStack queues a new deferred function with a defer record on the stack.
   283  // The defer record must have its siz and fn fields initialized.
   284  // All other fields can contain junk.
   285  // The defer record must be immediately followed in memory by
   286  // the arguments of the defer.
   287  // Nosplit because the arguments on the stack won't be scanned
   288  // until the defer record is spliced into the gp._defer list.
   289  //go:nosplit
   290  func deferprocStack(d *_defer) {
   291  	gp := getg()
   292  	if gp.m.curg != gp {
   293  		// go code on the system stack can't defer
   294  		throw("defer on system stack")
   295  	}
   296  	if goexperiment.RegabiDefer && d.siz != 0 {
   297  		throw("defer with non-empty frame")
   298  	}
   299  	// siz and fn are already set.
   300  	// The other fields are junk on entry to deferprocStack and
   301  	// are initialized here.
   302  	d.started = false
   303  	d.heap = false
   304  	d.openDefer = false
   305  	d.sp = getcallersp()
   306  	d.pc = getcallerpc()
   307  	d.framepc = 0
   308  	d.varp = 0
   309  	// The lines below implement:
   310  	//   d.panic = nil
   311  	//   d.fd = nil
   312  	//   d.link = gp._defer
   313  	//   gp._defer = d
   314  	// But without write barriers. The first three are writes to
   315  	// the stack so they don't need a write barrier, and furthermore
   316  	// are to uninitialized memory, so they must not use a write barrier.
   317  	// The fourth write does not require a write barrier because we
   318  	// explicitly mark all the defer structures, so we don't need to
   319  	// keep track of pointers to them with a write barrier.
   320  	*(*uintptr)(unsafe.Pointer(&d._panic)) = 0
   321  	*(*uintptr)(unsafe.Pointer(&d.fd)) = 0
   322  	*(*uintptr)(unsafe.Pointer(&d.link)) = uintptr(unsafe.Pointer(gp._defer))
   323  	*(*uintptr)(unsafe.Pointer(&gp._defer)) = uintptr(unsafe.Pointer(d))
   324  
   325  	return0()
   326  	// No code can go here - the C return register has
   327  	// been set and must not be clobbered.
   328  }
   329  
   330  // Small malloc size classes >= 16 are the multiples of 16: 16, 32, 48, 64, 80, 96, 112, 128, 144, ...
   331  // Each P holds a pool for defers with small arg sizes.
   332  // Assign defer allocations to pools by rounding to 16, to match malloc size classes.
   333  
   334  const (
   335  	deferHeaderSize = unsafe.Sizeof(_defer{})
   336  	minDeferAlloc   = (deferHeaderSize + 15) &^ 15
   337  	minDeferArgs    = minDeferAlloc - deferHeaderSize
   338  )
   339  
   340  // defer size class for arg size sz
   341  //go:nosplit
   342  func deferclass(siz uintptr) uintptr {
   343  	if siz <= minDeferArgs {
   344  		return 0
   345  	}
   346  	return (siz - minDeferArgs + 15) / 16
   347  }
   348  
   349  // total size of memory block for defer with arg size sz
   350  func totaldefersize(siz uintptr) uintptr {
   351  	if siz <= minDeferArgs {
   352  		return minDeferAlloc
   353  	}
   354  	return deferHeaderSize + siz
   355  }
   356  
   357  // Ensure that defer arg sizes that map to the same defer size class
   358  // also map to the same malloc size class.
   359  func testdefersizes() {
   360  	var m [len(p{}.deferpool)]int32
   361  
   362  	for i := range m {
   363  		m[i] = -1
   364  	}
   365  	for i := uintptr(0); ; i++ {
   366  		defersc := deferclass(i)
   367  		if defersc >= uintptr(len(m)) {
   368  			break
   369  		}
   370  		siz := roundupsize(totaldefersize(i))
   371  		if m[defersc] < 0 {
   372  			m[defersc] = int32(siz)
   373  			continue
   374  		}
   375  		if m[defersc] != int32(siz) {
   376  			print("bad defer size class: i=", i, " siz=", siz, " defersc=", defersc, "\n")
   377  			throw("bad defer size class")
   378  		}
   379  	}
   380  }
   381  
   382  // The arguments associated with a deferred call are stored
   383  // immediately after the _defer header in memory.
   384  //go:nosplit
   385  func deferArgs(d *_defer) unsafe.Pointer {
   386  	if d.siz == 0 {
   387  		// Avoid pointer past the defer allocation.
   388  		return nil
   389  	}
   390  	return add(unsafe.Pointer(d), unsafe.Sizeof(*d))
   391  }
   392  
   393  // deferFunc returns d's deferred function. This is temporary while we
   394  // support both modes of GOEXPERIMENT=regabidefer. Once we commit to
   395  // that experiment, we should change the type of d.fn.
   396  //go:nosplit
   397  func deferFunc(d *_defer) func() {
   398  	if !goexperiment.RegabiDefer {
   399  		throw("requires GOEXPERIMENT=regabidefer")
   400  	}
   401  	var fn func()
   402  	*(**funcval)(unsafe.Pointer(&fn)) = d.fn
   403  	return fn
   404  }
   405  
   406  var deferType *_type // type of _defer struct
   407  
   408  func init() {
   409  	var x interface{}
   410  	x = (*_defer)(nil)
   411  	deferType = (*(**ptrtype)(unsafe.Pointer(&x))).elem
   412  }
   413  
   414  // Allocate a Defer, usually using per-P pool.
   415  // Each defer must be released with freedefer.  The defer is not
   416  // added to any defer chain yet.
   417  //
   418  // This must not grow the stack because there may be a frame without
   419  // stack map information when this is called.
   420  //
   421  //go:nosplit
   422  func newdefer(siz int32) *_defer {
   423  	var d *_defer
   424  	sc := deferclass(uintptr(siz))
   425  	gp := getg()
   426  	if sc < uintptr(len(p{}.deferpool)) {
   427  		pp := gp.m.p.ptr()
   428  		if len(pp.deferpool[sc]) == 0 && sched.deferpool[sc] != nil {
   429  			// Take the slow path on the system stack so
   430  			// we don't grow newdefer's stack.
   431  			systemstack(func() {
   432  				lock(&sched.deferlock)
   433  				for len(pp.deferpool[sc]) < cap(pp.deferpool[sc])/2 && sched.deferpool[sc] != nil {
   434  					d := sched.deferpool[sc]
   435  					sched.deferpool[sc] = d.link
   436  					d.link = nil
   437  					pp.deferpool[sc] = append(pp.deferpool[sc], d)
   438  				}
   439  				unlock(&sched.deferlock)
   440  			})
   441  		}
   442  		if n := len(pp.deferpool[sc]); n > 0 {
   443  			d = pp.deferpool[sc][n-1]
   444  			pp.deferpool[sc][n-1] = nil
   445  			pp.deferpool[sc] = pp.deferpool[sc][:n-1]
   446  		}
   447  	}
   448  	if d == nil {
   449  		// Allocate new defer+args.
   450  		systemstack(func() {
   451  			total := roundupsize(totaldefersize(uintptr(siz)))
   452  			d = (*_defer)(mallocgc(total, deferType, true))
   453  		})
   454  	}
   455  	d.siz = siz
   456  	d.heap = true
   457  	return d
   458  }
   459  
   460  // Free the given defer.
   461  // The defer cannot be used after this call.
   462  //
   463  // This must not grow the stack because there may be a frame without a
   464  // stack map when this is called.
   465  //
   466  //go:nosplit
   467  func freedefer(d *_defer) {
   468  	if d._panic != nil {
   469  		freedeferpanic()
   470  	}
   471  	if d.fn != nil {
   472  		freedeferfn()
   473  	}
   474  	if !d.heap {
   475  		return
   476  	}
   477  	sc := deferclass(uintptr(d.siz))
   478  	if sc >= uintptr(len(p{}.deferpool)) {
   479  		return
   480  	}
   481  	pp := getg().m.p.ptr()
   482  	if len(pp.deferpool[sc]) == cap(pp.deferpool[sc]) {
   483  		// Transfer half of local cache to the central cache.
   484  		//
   485  		// Take this slow path on the system stack so
   486  		// we don't grow freedefer's stack.
   487  		systemstack(func() {
   488  			var first, last *_defer
   489  			for len(pp.deferpool[sc]) > cap(pp.deferpool[sc])/2 {
   490  				n := len(pp.deferpool[sc])
   491  				d := pp.deferpool[sc][n-1]
   492  				pp.deferpool[sc][n-1] = nil
   493  				pp.deferpool[sc] = pp.deferpool[sc][:n-1]
   494  				if first == nil {
   495  					first = d
   496  				} else {
   497  					last.link = d
   498  				}
   499  				last = d
   500  			}
   501  			lock(&sched.deferlock)
   502  			last.link = sched.deferpool[sc]
   503  			sched.deferpool[sc] = first
   504  			unlock(&sched.deferlock)
   505  		})
   506  	}
   507  
   508  	// These lines used to be simply `*d = _defer{}` but that
   509  	// started causing a nosplit stack overflow via typedmemmove.
   510  	d.siz = 0
   511  	d.started = false
   512  	d.openDefer = false
   513  	d.sp = 0
   514  	d.pc = 0
   515  	d.framepc = 0
   516  	d.varp = 0
   517  	d.fd = nil
   518  	// d._panic and d.fn must be nil already.
   519  	// If not, we would have called freedeferpanic or freedeferfn above,
   520  	// both of which throw.
   521  	d.link = nil
   522  
   523  	pp.deferpool[sc] = append(pp.deferpool[sc], d)
   524  }
   525  
   526  // Separate function so that it can split stack.
   527  // Windows otherwise runs out of stack space.
   528  func freedeferpanic() {
   529  	// _panic must be cleared before d is unlinked from gp.
   530  	throw("freedefer with d._panic != nil")
   531  }
   532  
   533  func freedeferfn() {
   534  	// fn must be cleared before d is unlinked from gp.
   535  	throw("freedefer with d.fn != nil")
   536  }
   537  
   538  // Run a deferred function if there is one.
   539  // The compiler inserts a call to this at the end of any
   540  // function which calls defer.
   541  // If there is a deferred function, this will call runtime·jmpdefer,
   542  // which will jump to the deferred function such that it appears
   543  // to have been called by the caller of deferreturn at the point
   544  // just before deferreturn was called. The effect is that deferreturn
   545  // is called again and again until there are no more deferred functions.
   546  //
   547  // Declared as nosplit, because the function should not be preempted once we start
   548  // modifying the caller's frame in order to reuse the frame to call the deferred
   549  // function.
   550  //
   551  //go:nosplit
   552  func deferreturn() {
   553  	gp := getg()
   554  	d := gp._defer
   555  	if d == nil {
   556  		return
   557  	}
   558  	sp := getcallersp()
   559  	if d.sp != sp {
   560  		return
   561  	}
   562  	if d.openDefer {
   563  		done := runOpenDeferFrame(gp, d)
   564  		if !done {
   565  			throw("unfinished open-coded defers in deferreturn")
   566  		}
   567  		gp._defer = d.link
   568  		freedefer(d)
   569  		return
   570  	}
   571  
   572  	// Moving arguments around.
   573  	//
   574  	// Everything called after this point must be recursively
   575  	// nosplit because the garbage collector won't know the form
   576  	// of the arguments until the jmpdefer can flip the PC over to
   577  	// fn.
   578  	argp := getcallersp() + sys.MinFrameSize
   579  	switch d.siz {
   580  	case 0:
   581  		// Do nothing.
   582  	case sys.PtrSize:
   583  		*(*uintptr)(unsafe.Pointer(argp)) = *(*uintptr)(deferArgs(d))
   584  	default:
   585  		memmove(unsafe.Pointer(argp), deferArgs(d), uintptr(d.siz))
   586  	}
   587  	fn := d.fn
   588  	d.fn = nil
   589  	gp._defer = d.link
   590  	freedefer(d)
   591  	// If the defer function pointer is nil, force the seg fault to happen
   592  	// here rather than in jmpdefer. gentraceback() throws an error if it is
   593  	// called with a callback on an LR architecture and jmpdefer is on the
   594  	// stack, because the stack trace can be incorrect in that case - see
   595  	// issue #8153).
   596  	_ = fn.fn
   597  	jmpdefer(fn, argp)
   598  }
   599  
   600  // Goexit terminates the goroutine that calls it. No other goroutine is affected.
   601  // Goexit runs all deferred calls before terminating the goroutine. Because Goexit
   602  // is not a panic, any recover calls in those deferred functions will return nil.
   603  //
   604  // Calling Goexit from the main goroutine terminates that goroutine
   605  // without func main returning. Since func main has not returned,
   606  // the program continues execution of other goroutines.
   607  // If all other goroutines exit, the program crashes.
   608  func Goexit() {
   609  	// Run all deferred functions for the current goroutine.
   610  	// This code is similar to gopanic, see that implementation
   611  	// for detailed comments.
   612  	gp := getg()
   613  
   614  	// Create a panic object for Goexit, so we can recognize when it might be
   615  	// bypassed by a recover().
   616  	var p _panic
   617  	p.goexit = true
   618  	p.link = gp._panic
   619  	gp._panic = (*_panic)(noescape(unsafe.Pointer(&p)))
   620  
   621  	addOneOpenDeferFrame(gp, getcallerpc(), unsafe.Pointer(getcallersp()))
   622  	for {
   623  		d := gp._defer
   624  		if d == nil {
   625  			break
   626  		}
   627  		if d.started {
   628  			if d._panic != nil {
   629  				d._panic.aborted = true
   630  				d._panic = nil
   631  			}
   632  			if !d.openDefer {
   633  				d.fn = nil
   634  				gp._defer = d.link
   635  				freedefer(d)
   636  				continue
   637  			}
   638  		}
   639  		d.started = true
   640  		d._panic = (*_panic)(noescape(unsafe.Pointer(&p)))
   641  		if d.openDefer {
   642  			done := runOpenDeferFrame(gp, d)
   643  			if !done {
   644  				// We should always run all defers in the frame,
   645  				// since there is no panic associated with this
   646  				// defer that can be recovered.
   647  				throw("unfinished open-coded defers in Goexit")
   648  			}
   649  			if p.aborted {
   650  				// Since our current defer caused a panic and may
   651  				// have been already freed, just restart scanning
   652  				// for open-coded defers from this frame again.
   653  				addOneOpenDeferFrame(gp, getcallerpc(), unsafe.Pointer(getcallersp()))
   654  			} else {
   655  				addOneOpenDeferFrame(gp, 0, nil)
   656  			}
   657  		} else {
   658  			if goexperiment.RegabiDefer {
   659  				// Save the pc/sp in deferCallSave(), so we can "recover" back to this
   660  				// loop if necessary.
   661  				deferCallSave(&p, deferFunc(d))
   662  			} else {
   663  				// Save the pc/sp in reflectcallSave(), so we can "recover" back to this
   664  				// loop if necessary.
   665  				reflectcallSave(&p, unsafe.Pointer(d.fn), deferArgs(d), uint32(d.siz))
   666  			}
   667  		}
   668  		if p.aborted {
   669  			// We had a recursive panic in the defer d we started, and
   670  			// then did a recover in a defer that was further down the
   671  			// defer chain than d. In the case of an outstanding Goexit,
   672  			// we force the recover to return back to this loop. d will
   673  			// have already been freed if completed, so just continue
   674  			// immediately to the next defer on the chain.
   675  			p.aborted = false
   676  			continue
   677  		}
   678  		if gp._defer != d {
   679  			throw("bad defer entry in Goexit")
   680  		}
   681  		d._panic = nil
   682  		d.fn = nil
   683  		gp._defer = d.link
   684  		freedefer(d)
   685  		// Note: we ignore recovers here because Goexit isn't a panic
   686  	}
   687  	goexit1()
   688  }
   689  
   690  // Call all Error and String methods before freezing the world.
   691  // Used when crashing with panicking.
   692  func preprintpanics(p *_panic) {
   693  	defer func() {
   694  		if recover() != nil {
   695  			throw("panic while printing panic value")
   696  		}
   697  	}()
   698  	for p != nil {
   699  		switch v := p.arg.(type) {
   700  		case error:
   701  			p.arg = v.Error()
   702  		case stringer:
   703  			p.arg = v.String()
   704  		}
   705  		p = p.link
   706  	}
   707  }
   708  
   709  // Print all currently active panics. Used when crashing.
   710  // Should only be called after preprintpanics.
   711  func printpanics(p *_panic) {
   712  	if p.link != nil {
   713  		printpanics(p.link)
   714  		if !p.link.goexit {
   715  			print("\t")
   716  		}
   717  	}
   718  	if p.goexit {
   719  		return
   720  	}
   721  	print("panic: ")
   722  	printany(p.arg)
   723  	if p.recovered {
   724  		print(" [recovered]")
   725  	}
   726  	print("\n")
   727  }
   728  
   729  // addOneOpenDeferFrame scans the stack for the first frame (if any) with
   730  // open-coded defers and if it finds one, adds a single record to the defer chain
   731  // for that frame. If sp is non-nil, it starts the stack scan from the frame
   732  // specified by sp. If sp is nil, it uses the sp from the current defer record
   733  // (which has just been finished). Hence, it continues the stack scan from the
   734  // frame of the defer that just finished. It skips any frame that already has an
   735  // open-coded _defer record, which would have been created from a previous
   736  // (unrecovered) panic.
   737  //
   738  // Note: All entries of the defer chain (including this new open-coded entry) have
   739  // their pointers (including sp) adjusted properly if the stack moves while
   740  // running deferred functions. Also, it is safe to pass in the sp arg (which is
   741  // the direct result of calling getcallersp()), because all pointer variables
   742  // (including arguments) are adjusted as needed during stack copies.
   743  func addOneOpenDeferFrame(gp *g, pc uintptr, sp unsafe.Pointer) {
   744  	var prevDefer *_defer
   745  	if sp == nil {
   746  		prevDefer = gp._defer
   747  		pc = prevDefer.framepc
   748  		sp = unsafe.Pointer(prevDefer.sp)
   749  	}
   750  	systemstack(func() {
   751  		gentraceback(pc, uintptr(sp), 0, gp, 0, nil, 0x7fffffff,
   752  			func(frame *stkframe, unused unsafe.Pointer) bool {
   753  				if prevDefer != nil && prevDefer.sp == frame.sp {
   754  					// Skip the frame for the previous defer that
   755  					// we just finished (and was used to set
   756  					// where we restarted the stack scan)
   757  					return true
   758  				}
   759  				f := frame.fn
   760  				fd := funcdata(f, _FUNCDATA_OpenCodedDeferInfo)
   761  				if fd == nil {
   762  					return true
   763  				}
   764  				// Insert the open defer record in the
   765  				// chain, in order sorted by sp.
   766  				d := gp._defer
   767  				var prev *_defer
   768  				for d != nil {
   769  					dsp := d.sp
   770  					if frame.sp < dsp {
   771  						break
   772  					}
   773  					if frame.sp == dsp {
   774  						if !d.openDefer {
   775  							throw("duplicated defer entry")
   776  						}
   777  						return true
   778  					}
   779  					prev = d
   780  					d = d.link
   781  				}
   782  				if frame.fn.deferreturn == 0 {
   783  					throw("missing deferreturn")
   784  				}
   785  
   786  				maxargsize, _ := readvarintUnsafe(fd)
   787  				d1 := newdefer(int32(maxargsize))
   788  				d1.openDefer = true
   789  				d1._panic = nil
   790  				// These are the pc/sp to set after we've
   791  				// run a defer in this frame that did a
   792  				// recover. We return to a special
   793  				// deferreturn that runs any remaining
   794  				// defers and then returns from the
   795  				// function.
   796  				d1.pc = frame.fn.entry + uintptr(frame.fn.deferreturn)
   797  				d1.varp = frame.varp
   798  				d1.fd = fd
   799  				// Save the SP/PC associated with current frame,
   800  				// so we can continue stack trace later if needed.
   801  				d1.framepc = frame.pc
   802  				d1.sp = frame.sp
   803  				d1.link = d
   804  				if prev == nil {
   805  					gp._defer = d1
   806  				} else {
   807  					prev.link = d1
   808  				}
   809  				// Stop stack scanning after adding one open defer record
   810  				return false
   811  			},
   812  			nil, 0)
   813  	})
   814  }
   815  
   816  // readvarintUnsafe reads the uint32 in varint format starting at fd, and returns the
   817  // uint32 and a pointer to the byte following the varint.
   818  //
   819  // There is a similar function runtime.readvarint, which takes a slice of bytes,
   820  // rather than an unsafe pointer. These functions are duplicated, because one of
   821  // the two use cases for the functions would get slower if the functions were
   822  // combined.
   823  func readvarintUnsafe(fd unsafe.Pointer) (uint32, unsafe.Pointer) {
   824  	var r uint32
   825  	var shift int
   826  	for {
   827  		b := *(*uint8)((unsafe.Pointer(fd)))
   828  		fd = add(fd, unsafe.Sizeof(b))
   829  		if b < 128 {
   830  			return r + uint32(b)<<shift, fd
   831  		}
   832  		r += ((uint32(b) &^ 128) << shift)
   833  		shift += 7
   834  		if shift > 28 {
   835  			panic("Bad varint")
   836  		}
   837  	}
   838  }
   839  
   840  // runOpenDeferFrame runs the active open-coded defers in the frame specified by
   841  // d. It normally processes all active defers in the frame, but stops immediately
   842  // if a defer does a successful recover. It returns true if there are no
   843  // remaining defers to run in the frame.
   844  func runOpenDeferFrame(gp *g, d *_defer) bool {
   845  	done := true
   846  	fd := d.fd
   847  
   848  	// Skip the maxargsize
   849  	_, fd = readvarintUnsafe(fd)
   850  	deferBitsOffset, fd := readvarintUnsafe(fd)
   851  	nDefers, fd := readvarintUnsafe(fd)
   852  	deferBits := *(*uint8)(unsafe.Pointer(d.varp - uintptr(deferBitsOffset)))
   853  
   854  	for i := int(nDefers) - 1; i >= 0; i-- {
   855  		// read the funcdata info for this defer
   856  		var argWidth, closureOffset, nArgs uint32
   857  		argWidth, fd = readvarintUnsafe(fd)
   858  		closureOffset, fd = readvarintUnsafe(fd)
   859  		nArgs, fd = readvarintUnsafe(fd)
   860  		if goexperiment.RegabiDefer && argWidth != 0 {
   861  			throw("defer with non-empty frame")
   862  		}
   863  		if deferBits&(1<<i) == 0 {
   864  			for j := uint32(0); j < nArgs; j++ {
   865  				_, fd = readvarintUnsafe(fd)
   866  				_, fd = readvarintUnsafe(fd)
   867  				_, fd = readvarintUnsafe(fd)
   868  			}
   869  			continue
   870  		}
   871  		closure := *(**funcval)(unsafe.Pointer(d.varp - uintptr(closureOffset)))
   872  		d.fn = closure
   873  		deferArgs := deferArgs(d)
   874  		// If there is an interface receiver or method receiver, it is
   875  		// described/included as the first arg.
   876  		for j := uint32(0); j < nArgs; j++ {
   877  			var argOffset, argLen, argCallOffset uint32
   878  			argOffset, fd = readvarintUnsafe(fd)
   879  			argLen, fd = readvarintUnsafe(fd)
   880  			argCallOffset, fd = readvarintUnsafe(fd)
   881  			memmove(unsafe.Pointer(uintptr(deferArgs)+uintptr(argCallOffset)),
   882  				unsafe.Pointer(d.varp-uintptr(argOffset)),
   883  				uintptr(argLen))
   884  		}
   885  		deferBits = deferBits &^ (1 << i)
   886  		*(*uint8)(unsafe.Pointer(d.varp - uintptr(deferBitsOffset))) = deferBits
   887  		p := d._panic
   888  		if goexperiment.RegabiDefer {
   889  			deferCallSave(p, deferFunc(d))
   890  		} else {
   891  			reflectcallSave(p, unsafe.Pointer(closure), deferArgs, argWidth)
   892  		}
   893  		if p != nil && p.aborted {
   894  			break
   895  		}
   896  		d.fn = nil
   897  		// These args are just a copy, so can be cleared immediately
   898  		memclrNoHeapPointers(deferArgs, uintptr(argWidth))
   899  		if d._panic != nil && d._panic.recovered {
   900  			done = deferBits == 0
   901  			break
   902  		}
   903  	}
   904  
   905  	return done
   906  }
   907  
   908  // reflectcallSave calls reflectcall after saving the caller's pc and sp in the
   909  // panic record. This allows the runtime to return to the Goexit defer processing
   910  // loop, in the unusual case where the Goexit may be bypassed by a successful
   911  // recover.
   912  //
   913  // This is marked as a wrapper by the compiler so it doesn't appear in
   914  // tracebacks.
   915  func reflectcallSave(p *_panic, fn, arg unsafe.Pointer, argsize uint32) {
   916  	if goexperiment.RegabiDefer {
   917  		throw("not allowed with GOEXPERIMENT=regabidefer")
   918  	}
   919  	if p != nil {
   920  		p.argp = unsafe.Pointer(getargp())
   921  		p.pc = getcallerpc()
   922  		p.sp = unsafe.Pointer(getcallersp())
   923  	}
   924  	// Pass a dummy RegArgs since we'll only take this path if
   925  	// we're not using the register ABI.
   926  	var regs abi.RegArgs
   927  	reflectcall(nil, fn, arg, argsize, argsize, argsize, &regs)
   928  	if p != nil {
   929  		p.pc = 0
   930  		p.sp = unsafe.Pointer(nil)
   931  	}
   932  }
   933  
   934  // deferCallSave calls fn() after saving the caller's pc and sp in the
   935  // panic record. This allows the runtime to return to the Goexit defer
   936  // processing loop, in the unusual case where the Goexit may be
   937  // bypassed by a successful recover.
   938  //
   939  // This is marked as a wrapper by the compiler so it doesn't appear in
   940  // tracebacks.
   941  func deferCallSave(p *_panic, fn func()) {
   942  	if !goexperiment.RegabiDefer {
   943  		throw("only allowed with GOEXPERIMENT=regabidefer")
   944  	}
   945  	if p != nil {
   946  		p.argp = unsafe.Pointer(getargp())
   947  		p.pc = getcallerpc()
   948  		p.sp = unsafe.Pointer(getcallersp())
   949  	}
   950  	fn()
   951  	if p != nil {
   952  		p.pc = 0
   953  		p.sp = unsafe.Pointer(nil)
   954  	}
   955  }
   956  
   957  // The implementation of the predeclared function panic.
   958  func gopanic(e interface{}) {
   959  	gp := getg()
   960  	if gp.m.curg != gp {
   961  		print("panic: ")
   962  		printany(e)
   963  		print("\n")
   964  		throw("panic on system stack")
   965  	}
   966  
   967  	if gp.m.mallocing != 0 {
   968  		print("panic: ")
   969  		printany(e)
   970  		print("\n")
   971  		throw("panic during malloc")
   972  	}
   973  	if gp.m.preemptoff != "" {
   974  		print("panic: ")
   975  		printany(e)
   976  		print("\n")
   977  		print("preempt off reason: ")
   978  		print(gp.m.preemptoff)
   979  		print("\n")
   980  		throw("panic during preemptoff")
   981  	}
   982  	if gp.m.locks != 0 {
   983  		print("panic: ")
   984  		printany(e)
   985  		print("\n")
   986  		throw("panic holding locks")
   987  	}
   988  
   989  	var p _panic
   990  	p.arg = e
   991  	p.link = gp._panic
   992  	gp._panic = (*_panic)(noescape(unsafe.Pointer(&p)))
   993  
   994  	atomic.Xadd(&runningPanicDefers, 1)
   995  
   996  	// By calculating getcallerpc/getcallersp here, we avoid scanning the
   997  	// gopanic frame (stack scanning is slow...)
   998  	addOneOpenDeferFrame(gp, getcallerpc(), unsafe.Pointer(getcallersp()))
   999  
  1000  	for {
  1001  		d := gp._defer
  1002  		if d == nil {
  1003  			break
  1004  		}
  1005  
  1006  		// If defer was started by earlier panic or Goexit (and, since we're back here, that triggered a new panic),
  1007  		// take defer off list. An earlier panic will not continue running, but we will make sure below that an
  1008  		// earlier Goexit does continue running.
  1009  		if d.started {
  1010  			if d._panic != nil {
  1011  				d._panic.aborted = true
  1012  			}
  1013  			d._panic = nil
  1014  			if !d.openDefer {
  1015  				// For open-coded defers, we need to process the
  1016  				// defer again, in case there are any other defers
  1017  				// to call in the frame (not including the defer
  1018  				// call that caused the panic).
  1019  				d.fn = nil
  1020  				gp._defer = d.link
  1021  				freedefer(d)
  1022  				continue
  1023  			}
  1024  		}
  1025  
  1026  		// Mark defer as started, but keep on list, so that traceback
  1027  		// can find and update the defer's argument frame if stack growth
  1028  		// or a garbage collection happens before executing d.fn.
  1029  		d.started = true
  1030  
  1031  		// Record the panic that is running the defer.
  1032  		// If there is a new panic during the deferred call, that panic
  1033  		// will find d in the list and will mark d._panic (this panic) aborted.
  1034  		d._panic = (*_panic)(noescape(unsafe.Pointer(&p)))
  1035  
  1036  		done := true
  1037  		if d.openDefer {
  1038  			done = runOpenDeferFrame(gp, d)
  1039  			if done && !d._panic.recovered {
  1040  				addOneOpenDeferFrame(gp, 0, nil)
  1041  			}
  1042  		} else {
  1043  			p.argp = unsafe.Pointer(getargp())
  1044  
  1045  			if goexperiment.RegabiDefer {
  1046  				fn := deferFunc(d)
  1047  				fn()
  1048  			} else {
  1049  				// Pass a dummy RegArgs since we'll only take this path if
  1050  				// we're not using the register ABI.
  1051  				var regs abi.RegArgs
  1052  				reflectcall(nil, unsafe.Pointer(d.fn), deferArgs(d), uint32(d.siz), uint32(d.siz), uint32(d.siz), &regs)
  1053  			}
  1054  		}
  1055  		p.argp = nil
  1056  
  1057  		// Deferred function did not panic. Remove d.
  1058  		if gp._defer != d {
  1059  			throw("bad defer entry in panic")
  1060  		}
  1061  		d._panic = nil
  1062  
  1063  		// trigger shrinkage to test stack copy. See stack_test.go:TestStackPanic
  1064  		//GC()
  1065  
  1066  		pc := d.pc
  1067  		sp := unsafe.Pointer(d.sp) // must be pointer so it gets adjusted during stack copy
  1068  		if done {
  1069  			d.fn = nil
  1070  			gp._defer = d.link
  1071  			freedefer(d)
  1072  		}
  1073  		if p.recovered {
  1074  			gp._panic = p.link
  1075  			if gp._panic != nil && gp._panic.goexit && gp._panic.aborted {
  1076  				// A normal recover would bypass/abort the Goexit.  Instead,
  1077  				// we return to the processing loop of the Goexit.
  1078  				gp.sigcode0 = uintptr(gp._panic.sp)
  1079  				gp.sigcode1 = uintptr(gp._panic.pc)
  1080  				mcall(recovery)
  1081  				throw("bypassed recovery failed") // mcall should not return
  1082  			}
  1083  			atomic.Xadd(&runningPanicDefers, -1)
  1084  
  1085  			// Remove any remaining non-started, open-coded
  1086  			// defer entries after a recover, since the
  1087  			// corresponding defers will be executed normally
  1088  			// (inline). Any such entry will become stale once
  1089  			// we run the corresponding defers inline and exit
  1090  			// the associated stack frame.
  1091  			d := gp._defer
  1092  			var prev *_defer
  1093  			if !done {
  1094  				// Skip our current frame, if not done. It is
  1095  				// needed to complete any remaining defers in
  1096  				// deferreturn()
  1097  				prev = d
  1098  				d = d.link
  1099  			}
  1100  			for d != nil {
  1101  				if d.started {
  1102  					// This defer is started but we
  1103  					// are in the middle of a
  1104  					// defer-panic-recover inside of
  1105  					// it, so don't remove it or any
  1106  					// further defer entries
  1107  					break
  1108  				}
  1109  				if d.openDefer {
  1110  					if prev == nil {
  1111  						gp._defer = d.link
  1112  					} else {
  1113  						prev.link = d.link
  1114  					}
  1115  					newd := d.link
  1116  					freedefer(d)
  1117  					d = newd
  1118  				} else {
  1119  					prev = d
  1120  					d = d.link
  1121  				}
  1122  			}
  1123  
  1124  			gp._panic = p.link
  1125  			// Aborted panics are marked but remain on the g.panic list.
  1126  			// Remove them from the list.
  1127  			for gp._panic != nil && gp._panic.aborted {
  1128  				gp._panic = gp._panic.link
  1129  			}
  1130  			if gp._panic == nil { // must be done with signal
  1131  				gp.sig = 0
  1132  			}
  1133  			// Pass information about recovering frame to recovery.
  1134  			gp.sigcode0 = uintptr(sp)
  1135  			gp.sigcode1 = pc
  1136  			mcall(recovery)
  1137  			throw("recovery failed") // mcall should not return
  1138  		}
  1139  	}
  1140  
  1141  	// ran out of deferred calls - old-school panic now
  1142  	// Because it is unsafe to call arbitrary user code after freezing
  1143  	// the world, we call preprintpanics to invoke all necessary Error
  1144  	// and String methods to prepare the panic strings before startpanic.
  1145  	preprintpanics(gp._panic)
  1146  
  1147  	fatalpanic(gp._panic) // should not return
  1148  	*(*int)(nil) = 0      // not reached
  1149  }
  1150  
  1151  // getargp returns the location where the caller
  1152  // writes outgoing function call arguments.
  1153  //go:nosplit
  1154  //go:noinline
  1155  func getargp() uintptr {
  1156  	return getcallersp() + sys.MinFrameSize
  1157  }
  1158  
  1159  // The implementation of the predeclared function recover.
  1160  // Cannot split the stack because it needs to reliably
  1161  // find the stack segment of its caller.
  1162  //
  1163  // TODO(rsc): Once we commit to CopyStackAlways,
  1164  // this doesn't need to be nosplit.
  1165  //go:nosplit
  1166  func gorecover(argp uintptr) interface{} {
  1167  	// Must be in a function running as part of a deferred call during the panic.
  1168  	// Must be called from the topmost function of the call
  1169  	// (the function used in the defer statement).
  1170  	// p.argp is the argument pointer of that topmost deferred function call.
  1171  	// Compare against argp reported by caller.
  1172  	// If they match, the caller is the one who can recover.
  1173  	gp := getg()
  1174  	p := gp._panic
  1175  	if p != nil && !p.goexit && !p.recovered && argp == uintptr(p.argp) {
  1176  		p.recovered = true
  1177  		return p.arg
  1178  	}
  1179  	return nil
  1180  }
  1181  
  1182  //go:linkname sync_throw sync.throw
  1183  func sync_throw(s string) {
  1184  	throw(s)
  1185  }
  1186  
  1187  //go:nosplit
  1188  func throw(s string) {
  1189  	// Everything throw does should be recursively nosplit so it
  1190  	// can be called even when it's unsafe to grow the stack.
  1191  	systemstack(func() {
  1192  		print("fatal error: ", s, "\n")
  1193  	})
  1194  	gp := getg()
  1195  	if gp.m.throwing == 0 {
  1196  		gp.m.throwing = 1
  1197  	}
  1198  	fatalthrow()
  1199  	*(*int)(nil) = 0 // not reached
  1200  }
  1201  
  1202  // runningPanicDefers is non-zero while running deferred functions for panic.
  1203  // runningPanicDefers is incremented and decremented atomically.
  1204  // This is used to try hard to get a panic stack trace out when exiting.
  1205  var runningPanicDefers uint32
  1206  
  1207  // panicking is non-zero when crashing the program for an unrecovered panic.
  1208  // panicking is incremented and decremented atomically.
  1209  var panicking uint32
  1210  
  1211  // paniclk is held while printing the panic information and stack trace,
  1212  // so that two concurrent panics don't overlap their output.
  1213  var paniclk mutex
  1214  
  1215  // Unwind the stack after a deferred function calls recover
  1216  // after a panic. Then arrange to continue running as though
  1217  // the caller of the deferred function returned normally.
  1218  func recovery(gp *g) {
  1219  	// Info about defer passed in G struct.
  1220  	sp := gp.sigcode0
  1221  	pc := gp.sigcode1
  1222  
  1223  	// d's arguments need to be in the stack.
  1224  	if sp != 0 && (sp < gp.stack.lo || gp.stack.hi < sp) {
  1225  		print("recover: ", hex(sp), " not in [", hex(gp.stack.lo), ", ", hex(gp.stack.hi), "]\n")
  1226  		throw("bad recovery")
  1227  	}
  1228  
  1229  	// Make the deferproc for this d return again,
  1230  	// this time returning 1. The calling function will
  1231  	// jump to the standard return epilogue.
  1232  	gp.sched.sp = sp
  1233  	gp.sched.pc = pc
  1234  	gp.sched.lr = 0
  1235  	gp.sched.ret = 1
  1236  	gogo(&gp.sched)
  1237  }
  1238  
  1239  // fatalthrow implements an unrecoverable runtime throw. It freezes the
  1240  // system, prints stack traces starting from its caller, and terminates the
  1241  // process.
  1242  //
  1243  //go:nosplit
  1244  func fatalthrow() {
  1245  	pc := getcallerpc()
  1246  	sp := getcallersp()
  1247  	gp := getg()
  1248  	// Switch to the system stack to avoid any stack growth, which
  1249  	// may make things worse if the runtime is in a bad state.
  1250  	systemstack(func() {
  1251  		startpanic_m()
  1252  
  1253  		if dopanic_m(gp, pc, sp) {
  1254  			// crash uses a decent amount of nosplit stack and we're already
  1255  			// low on stack in throw, so crash on the system stack (unlike
  1256  			// fatalpanic).
  1257  			crash()
  1258  		}
  1259  
  1260  		exit(2)
  1261  	})
  1262  
  1263  	*(*int)(nil) = 0 // not reached
  1264  }
  1265  
  1266  // fatalpanic implements an unrecoverable panic. It is like fatalthrow, except
  1267  // that if msgs != nil, fatalpanic also prints panic messages and decrements
  1268  // runningPanicDefers once main is blocked from exiting.
  1269  //
  1270  //go:nosplit
  1271  func fatalpanic(msgs *_panic) {
  1272  	pc := getcallerpc()
  1273  	sp := getcallersp()
  1274  	gp := getg()
  1275  	var docrash bool
  1276  	// Switch to the system stack to avoid any stack growth, which
  1277  	// may make things worse if the runtime is in a bad state.
  1278  	systemstack(func() {
  1279  		if startpanic_m() && msgs != nil {
  1280  			// There were panic messages and startpanic_m
  1281  			// says it's okay to try to print them.
  1282  
  1283  			// startpanic_m set panicking, which will
  1284  			// block main from exiting, so now OK to
  1285  			// decrement runningPanicDefers.
  1286  			atomic.Xadd(&runningPanicDefers, -1)
  1287  
  1288  			printpanics(msgs)
  1289  		}
  1290  
  1291  		docrash = dopanic_m(gp, pc, sp)
  1292  	})
  1293  
  1294  	if docrash {
  1295  		// By crashing outside the above systemstack call, debuggers
  1296  		// will not be confused when generating a backtrace.
  1297  		// Function crash is marked nosplit to avoid stack growth.
  1298  		crash()
  1299  	}
  1300  
  1301  	systemstack(func() {
  1302  		exit(2)
  1303  	})
  1304  
  1305  	*(*int)(nil) = 0 // not reached
  1306  }
  1307  
  1308  // startpanic_m prepares for an unrecoverable panic.
  1309  //
  1310  // It returns true if panic messages should be printed, or false if
  1311  // the runtime is in bad shape and should just print stacks.
  1312  //
  1313  // It must not have write barriers even though the write barrier
  1314  // explicitly ignores writes once dying > 0. Write barriers still
  1315  // assume that g.m.p != nil, and this function may not have P
  1316  // in some contexts (e.g. a panic in a signal handler for a signal
  1317  // sent to an M with no P).
  1318  //
  1319  //go:nowritebarrierrec
  1320  func startpanic_m() bool {
  1321  	_g_ := getg()
  1322  	if mheap_.cachealloc.size == 0 { // very early
  1323  		print("runtime: panic before malloc heap initialized\n")
  1324  	}
  1325  	// Disallow malloc during an unrecoverable panic. A panic
  1326  	// could happen in a signal handler, or in a throw, or inside
  1327  	// malloc itself. We want to catch if an allocation ever does
  1328  	// happen (even if we're not in one of these situations).
  1329  	_g_.m.mallocing++
  1330  
  1331  	// If we're dying because of a bad lock count, set it to a
  1332  	// good lock count so we don't recursively panic below.
  1333  	if _g_.m.locks < 0 {
  1334  		_g_.m.locks = 1
  1335  	}
  1336  
  1337  	switch _g_.m.dying {
  1338  	case 0:
  1339  		// Setting dying >0 has the side-effect of disabling this G's writebuf.
  1340  		_g_.m.dying = 1
  1341  		atomic.Xadd(&panicking, 1)
  1342  		lock(&paniclk)
  1343  		if debug.schedtrace > 0 || debug.scheddetail > 0 {
  1344  			schedtrace(true)
  1345  		}
  1346  		freezetheworld()
  1347  		return true
  1348  	case 1:
  1349  		// Something failed while panicking.
  1350  		// Just print a stack trace and exit.
  1351  		_g_.m.dying = 2
  1352  		print("panic during panic\n")
  1353  		return false
  1354  	case 2:
  1355  		// This is a genuine bug in the runtime, we couldn't even
  1356  		// print the stack trace successfully.
  1357  		_g_.m.dying = 3
  1358  		print("stack trace unavailable\n")
  1359  		exit(4)
  1360  		fallthrough
  1361  	default:
  1362  		// Can't even print! Just exit.
  1363  		exit(5)
  1364  		return false // Need to return something.
  1365  	}
  1366  }
  1367  
  1368  var didothers bool
  1369  var deadlock mutex
  1370  
  1371  func dopanic_m(gp *g, pc, sp uintptr) bool {
  1372  	if gp.sig != 0 {
  1373  		signame := signame(gp.sig)
  1374  		if signame != "" {
  1375  			print("[signal ", signame)
  1376  		} else {
  1377  			print("[signal ", hex(gp.sig))
  1378  		}
  1379  		print(" code=", hex(gp.sigcode0), " addr=", hex(gp.sigcode1), " pc=", hex(gp.sigpc), "]\n")
  1380  	}
  1381  
  1382  	level, all, docrash := gotraceback()
  1383  	_g_ := getg()
  1384  	if level > 0 {
  1385  		if gp != gp.m.curg {
  1386  			all = true
  1387  		}
  1388  		if gp != gp.m.g0 {
  1389  			print("\n")
  1390  			goroutineheader(gp)
  1391  			traceback(pc, sp, 0, gp)
  1392  		} else if level >= 2 || _g_.m.throwing > 0 {
  1393  			print("\nruntime stack:\n")
  1394  			traceback(pc, sp, 0, gp)
  1395  		}
  1396  		if !didothers && all {
  1397  			didothers = true
  1398  			tracebackothers(gp)
  1399  		}
  1400  	}
  1401  	unlock(&paniclk)
  1402  
  1403  	if atomic.Xadd(&panicking, -1) != 0 {
  1404  		// Some other m is panicking too.
  1405  		// Let it print what it needs to print.
  1406  		// Wait forever without chewing up cpu.
  1407  		// It will exit when it's done.
  1408  		lock(&deadlock)
  1409  		lock(&deadlock)
  1410  	}
  1411  
  1412  	printDebugLog()
  1413  
  1414  	return docrash
  1415  }
  1416  
  1417  // canpanic returns false if a signal should throw instead of
  1418  // panicking.
  1419  //
  1420  //go:nosplit
  1421  func canpanic(gp *g) bool {
  1422  	// Note that g is m->gsignal, different from gp.
  1423  	// Note also that g->m can change at preemption, so m can go stale
  1424  	// if this function ever makes a function call.
  1425  	_g_ := getg()
  1426  	_m_ := _g_.m
  1427  
  1428  	// Is it okay for gp to panic instead of crashing the program?
  1429  	// Yes, as long as it is running Go code, not runtime code,
  1430  	// and not stuck in a system call.
  1431  	if gp == nil || gp != _m_.curg {
  1432  		return false
  1433  	}
  1434  	if _m_.locks != 0 || _m_.mallocing != 0 || _m_.throwing != 0 || _m_.preemptoff != "" || _m_.dying != 0 {
  1435  		return false
  1436  	}
  1437  	status := readgstatus(gp)
  1438  	if status&^_Gscan != _Grunning || gp.syscallsp != 0 {
  1439  		return false
  1440  	}
  1441  	if GOOS == "windows" && _m_.libcallsp != 0 {
  1442  		return false
  1443  	}
  1444  	return true
  1445  }
  1446  
  1447  // shouldPushSigpanic reports whether pc should be used as sigpanic's
  1448  // return PC (pushing a frame for the call). Otherwise, it should be
  1449  // left alone so that LR is used as sigpanic's return PC, effectively
  1450  // replacing the top-most frame with sigpanic. This is used by
  1451  // preparePanic.
  1452  func shouldPushSigpanic(gp *g, pc, lr uintptr) bool {
  1453  	if pc == 0 {
  1454  		// Probably a call to a nil func. The old LR is more
  1455  		// useful in the stack trace. Not pushing the frame
  1456  		// will make the trace look like a call to sigpanic
  1457  		// instead. (Otherwise the trace will end at sigpanic
  1458  		// and we won't get to see who faulted.)
  1459  		return false
  1460  	}
  1461  	// If we don't recognize the PC as code, but we do recognize
  1462  	// the link register as code, then this assumes the panic was
  1463  	// caused by a call to non-code. In this case, we want to
  1464  	// ignore this call to make unwinding show the context.
  1465  	//
  1466  	// If we running C code, we're not going to recognize pc as a
  1467  	// Go function, so just assume it's good. Otherwise, traceback
  1468  	// may try to read a stale LR that looks like a Go code
  1469  	// pointer and wander into the woods.
  1470  	if gp.m.incgo || findfunc(pc).valid() {
  1471  		// This wasn't a bad call, so use PC as sigpanic's
  1472  		// return PC.
  1473  		return true
  1474  	}
  1475  	if findfunc(lr).valid() {
  1476  		// This was a bad call, but the LR is good, so use the
  1477  		// LR as sigpanic's return PC.
  1478  		return false
  1479  	}
  1480  	// Neither the PC or LR is good. Hopefully pushing a frame
  1481  	// will work.
  1482  	return true
  1483  }
  1484  
  1485  // isAbortPC reports whether pc is the program counter at which
  1486  // runtime.abort raises a signal.
  1487  //
  1488  // It is nosplit because it's part of the isgoexception
  1489  // implementation.
  1490  //
  1491  //go:nosplit
  1492  func isAbortPC(pc uintptr) bool {
  1493  	f := findfunc(pc)
  1494  	if !f.valid() {
  1495  		return false
  1496  	}
  1497  	return f.funcID == funcID_abort
  1498  }
  1499  

View as plain text