mkpreempt.go

Documentation: runtime

     1  // Copyright 2019 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  //go:build ignore
     6  // +build ignore
     7  
     8  // mkpreempt generates the asyncPreempt functions for each
     9  // architecture.
    10  package main
    11  
    12  import (
    13  	"flag"
    14  	"fmt"
    15  	"io"
    16  	"log"
    17  	"os"
    18  	"strings"
    19  )
    20  
    21  // Copied from cmd/compile/internal/ssa/gen/*Ops.go
    22  
    23  var regNames386 = []string{
    24  	"AX",
    25  	"CX",
    26  	"DX",
    27  	"BX",
    28  	"SP",
    29  	"BP",
    30  	"SI",
    31  	"DI",
    32  	"X0",
    33  	"X1",
    34  	"X2",
    35  	"X3",
    36  	"X4",
    37  	"X5",
    38  	"X6",
    39  	"X7",
    40  }
    41  
    42  var regNamesAMD64 = []string{
    43  	"AX",
    44  	"CX",
    45  	"DX",
    46  	"BX",
    47  	"SP",
    48  	"BP",
    49  	"SI",
    50  	"DI",
    51  	"R8",
    52  	"R9",
    53  	"R10",
    54  	"R11",
    55  	"R12",
    56  	"R13",
    57  	"R14",
    58  	"R15",
    59  	"X0",
    60  	"X1",
    61  	"X2",
    62  	"X3",
    63  	"X4",
    64  	"X5",
    65  	"X6",
    66  	"X7",
    67  	"X8",
    68  	"X9",
    69  	"X10",
    70  	"X11",
    71  	"X12",
    72  	"X13",
    73  	"X14",
    74  	"X15",
    75  }
    76  
    77  var out io.Writer
    78  
    79  var arches = map[string]func(){
    80  	"386":     gen386,
    81  	"amd64":   genAMD64,
    82  	"arm":     genARM,
    83  	"arm64":   genARM64,
    84  	"mips64x": func() { genMIPS(true) },
    85  	"mipsx":   func() { genMIPS(false) },
    86  	"ppc64x":  genPPC64,
    87  	"riscv64": genRISCV64,
    88  	"s390x":   genS390X,
    89  	"wasm":    genWasm,
    90  }
    91  var beLe = map[string]bool{"mips64x": true, "mipsx": true, "ppc64x": true}
    92  
    93  func main() {
    94  	flag.Parse()
    95  	if flag.NArg() > 0 {
    96  		out = os.Stdout
    97  		for _, arch := range flag.Args() {
    98  			gen, ok := arches[arch]
    99  			if !ok {
   100  				log.Fatalf("unknown arch %s", arch)
   101  			}
   102  			header(arch)
   103  			gen()
   104  		}
   105  		return
   106  	}
   107  
   108  	for arch, gen := range arches {
   109  		f, err := os.Create(fmt.Sprintf("preempt_%s.s", arch))
   110  		if err != nil {
   111  			log.Fatal(err)
   112  		}
   113  		out = f
   114  		header(arch)
   115  		gen()
   116  		if err := f.Close(); err != nil {
   117  			log.Fatal(err)
   118  		}
   119  	}
   120  }
   121  
   122  func header(arch string) {
   123  	fmt.Fprintf(out, "// Code generated by mkpreempt.go; DO NOT EDIT.\n\n")
   124  	if beLe[arch] {
   125  		base := arch[:len(arch)-1]
   126  		fmt.Fprintf(out, "//go:build %s || %sle\n", base, base)
   127  		fmt.Fprintf(out, "// +build %s %sle\n\n", base, base)
   128  	}
   129  	fmt.Fprintf(out, "#include \"go_asm.h\"\n")
   130  	fmt.Fprintf(out, "#include \"textflag.h\"\n\n")
   131  	fmt.Fprintf(out, "// Note: asyncPreempt doesn't use the internal ABI, but we must be able to inject calls to it from the signal handler, so Go code has to see the PC of this function literally.\n")
   132  	fmt.Fprintf(out, "TEXT ·asyncPreempt<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-0\n")
   133  }
   134  
   135  func p(f string, args ...interface{}) {
   136  	fmted := fmt.Sprintf(f, args...)
   137  	fmt.Fprintf(out, "\t%s\n", strings.ReplaceAll(fmted, "\n", "\n\t"))
   138  }
   139  
   140  func label(l string) {
   141  	fmt.Fprintf(out, "%s\n", l)
   142  }
   143  
   144  type layout struct {
   145  	stack int
   146  	regs  []regPos
   147  	sp    string // stack pointer register
   148  }
   149  
   150  type regPos struct {
   151  	pos int
   152  
   153  	op  string
   154  	reg string
   155  
   156  	// If this register requires special save and restore, these
   157  	// give those operations with a %d placeholder for the stack
   158  	// offset.
   159  	save, restore string
   160  }
   161  
   162  func (l *layout) add(op, reg string, size int) {
   163  	l.regs = append(l.regs, regPos{op: op, reg: reg, pos: l.stack})
   164  	l.stack += size
   165  }
   166  
   167  func (l *layout) addSpecial(save, restore string, size int) {
   168  	l.regs = append(l.regs, regPos{save: save, restore: restore, pos: l.stack})
   169  	l.stack += size
   170  }
   171  
   172  func (l *layout) save() {
   173  	for _, reg := range l.regs {
   174  		if reg.save != "" {
   175  			p(reg.save, reg.pos)
   176  		} else {
   177  			p("%s %s, %d(%s)", reg.op, reg.reg, reg.pos, l.sp)
   178  		}
   179  	}
   180  }
   181  
   182  func (l *layout) restore() {
   183  	for i := len(l.regs) - 1; i >= 0; i-- {
   184  		reg := l.regs[i]
   185  		if reg.restore != "" {
   186  			p(reg.restore, reg.pos)
   187  		} else {
   188  			p("%s %d(%s), %s", reg.op, reg.pos, l.sp, reg.reg)
   189  		}
   190  	}
   191  }
   192  
   193  func gen386() {
   194  	p("PUSHFL")
   195  	// Save general purpose registers.
   196  	var l = layout{sp: "SP"}
   197  	for _, reg := range regNames386 {
   198  		if reg == "SP" || strings.HasPrefix(reg, "X") {
   199  			continue
   200  		}
   201  		l.add("MOVL", reg, 4)
   202  	}
   203  
   204  	// Save SSE state only if supported.
   205  	lSSE := layout{stack: l.stack, sp: "SP"}
   206  	for i := 0; i < 8; i++ {
   207  		lSSE.add("MOVUPS", fmt.Sprintf("X%d", i), 16)
   208  	}
   209  
   210  	p("ADJSP $%d", lSSE.stack)
   211  	p("NOP SP")
   212  	l.save()
   213  	p("CMPB internal∕cpu·X86+const_offsetX86HasSSE2(SB), $1\nJNE nosse")
   214  	lSSE.save()
   215  	label("nosse:")
   216  	p("CALL ·asyncPreempt2(SB)")
   217  	p("CMPB internal∕cpu·X86+const_offsetX86HasSSE2(SB), $1\nJNE nosse2")
   218  	lSSE.restore()
   219  	label("nosse2:")
   220  	l.restore()
   221  	p("ADJSP $%d", -lSSE.stack)
   222  
   223  	p("POPFL")
   224  	p("RET")
   225  }
   226  
   227  func genAMD64() {
   228  	// Assign stack offsets.
   229  	var l = layout{sp: "SP"}
   230  	for _, reg := range regNamesAMD64 {
   231  		if reg == "SP" || reg == "BP" {
   232  			continue
   233  		}
   234  		if !strings.HasPrefix(reg, "X") {
   235  			l.add("MOVQ", reg, 8)
   236  		}
   237  	}
   238  	lSSE := layout{stack: l.stack, sp: "SP"}
   239  	for _, reg := range regNamesAMD64 {
   240  		if strings.HasPrefix(reg, "X") {
   241  			lSSE.add("MOVUPS", reg, 16)
   242  		}
   243  	}
   244  
   245  	// TODO: MXCSR register?
   246  
   247  	p("PUSHQ BP")
   248  	p("MOVQ SP, BP")
   249  	p("// Save flags before clobbering them")
   250  	p("PUSHFQ")
   251  	p("// obj doesn't understand ADD/SUB on SP, but does understand ADJSP")
   252  	p("ADJSP $%d", lSSE.stack)
   253  	p("// But vet doesn't know ADJSP, so suppress vet stack checking")
   254  	p("NOP SP")
   255  
   256  	l.save()
   257  
   258  	// Apparently, the signal handling code path in darwin kernel leaves
   259  	// the upper bits of Y registers in a dirty state, which causes
   260  	// many SSE operations (128-bit and narrower) become much slower.
   261  	// Clear the upper bits to get to a clean state. See issue #37174.
   262  	// It is safe here as Go code don't use the upper bits of Y registers.
   263  	p("#ifdef GOOS_darwin")
   264  	p("CMPB internal∕cpu·X86+const_offsetX86HasAVX(SB), $0")
   265  	p("JE 2(PC)")
   266  	p("VZEROUPPER")
   267  	p("#endif")
   268  
   269  	lSSE.save()
   270  	p("CALL ·asyncPreempt2(SB)")
   271  	lSSE.restore()
   272  	l.restore()
   273  	p("ADJSP $%d", -lSSE.stack)
   274  	p("POPFQ")
   275  	p("POPQ BP")
   276  	p("RET")
   277  }
   278  
   279  func genARM() {
   280  	// Add integer registers R0-R12.
   281  	// R13 (SP), R14 (LR), R15 (PC) are special and not saved here.
   282  	var l = layout{sp: "R13", stack: 4} // add LR slot
   283  	for i := 0; i <= 12; i++ {
   284  		reg := fmt.Sprintf("R%d", i)
   285  		if i == 10 {
   286  			continue // R10 is g register, no need to save/restore
   287  		}
   288  		l.add("MOVW", reg, 4)
   289  	}
   290  	// Add flag register.
   291  	l.addSpecial(
   292  		"MOVW CPSR, R0\nMOVW R0, %d(R13)",
   293  		"MOVW %d(R13), R0\nMOVW R0, CPSR",
   294  		4)
   295  
   296  	// Add floating point registers F0-F15 and flag register.
   297  	var lfp = layout{stack: l.stack, sp: "R13"}
   298  	lfp.addSpecial(
   299  		"MOVW FPCR, R0\nMOVW R0, %d(R13)",
   300  		"MOVW %d(R13), R0\nMOVW R0, FPCR",
   301  		4)
   302  	for i := 0; i <= 15; i++ {
   303  		reg := fmt.Sprintf("F%d", i)
   304  		lfp.add("MOVD", reg, 8)
   305  	}
   306  
   307  	p("MOVW.W R14, -%d(R13)", lfp.stack) // allocate frame, save LR
   308  	l.save()
   309  	p("MOVB ·goarm(SB), R0\nCMP $6, R0\nBLT nofp") // test goarm, and skip FP registers if goarm=5.
   310  	lfp.save()
   311  	label("nofp:")
   312  	p("CALL ·asyncPreempt2(SB)")
   313  	p("MOVB ·goarm(SB), R0\nCMP $6, R0\nBLT nofp2") // test goarm, and skip FP registers if goarm=5.
   314  	lfp.restore()
   315  	label("nofp2:")
   316  	l.restore()
   317  
   318  	p("MOVW %d(R13), R14", lfp.stack)     // sigctxt.pushCall pushes LR on stack, restore it
   319  	p("MOVW.P %d(R13), R15", lfp.stack+4) // load PC, pop frame (including the space pushed by sigctxt.pushCall)
   320  	p("UNDEF")                            // shouldn't get here
   321  }
   322  
   323  func genARM64() {
   324  	// Add integer registers R0-R26
   325  	// R27 (REGTMP), R28 (g), R29 (FP), R30 (LR), R31 (SP) are special
   326  	// and not saved here.
   327  	var l = layout{sp: "RSP", stack: 8} // add slot to save PC of interrupted instruction
   328  	for i := 0; i <= 26; i++ {
   329  		if i == 18 {
   330  			continue // R18 is not used, skip
   331  		}
   332  		reg := fmt.Sprintf("R%d", i)
   333  		l.add("MOVD", reg, 8)
   334  	}
   335  	// Add flag registers.
   336  	l.addSpecial(
   337  		"MOVD NZCV, R0\nMOVD R0, %d(RSP)",
   338  		"MOVD %d(RSP), R0\nMOVD R0, NZCV",
   339  		8)
   340  	l.addSpecial(
   341  		"MOVD FPSR, R0\nMOVD R0, %d(RSP)",
   342  		"MOVD %d(RSP), R0\nMOVD R0, FPSR",
   343  		8)
   344  	// TODO: FPCR? I don't think we'll change it, so no need to save.
   345  	// Add floating point registers F0-F31.
   346  	for i := 0; i <= 31; i++ {
   347  		reg := fmt.Sprintf("F%d", i)
   348  		l.add("FMOVD", reg, 8)
   349  	}
   350  	if l.stack%16 != 0 {
   351  		l.stack += 8 // SP needs 16-byte alignment
   352  	}
   353  
   354  	// allocate frame, save PC of interrupted instruction (in LR)
   355  	p("MOVD R30, %d(RSP)", -l.stack)
   356  	p("SUB $%d, RSP", l.stack)
   357  	p("#ifdef GOOS_linux")
   358  	p("MOVD R29, -8(RSP)") // save frame pointer (only used on Linux)
   359  	p("SUB $8, RSP, R29")  // set up new frame pointer
   360  	p("#endif")
   361  	// On iOS, save the LR again after decrementing SP. We run the
   362  	// signal handler on the G stack (as it doesn't support sigaltstack),
   363  	// so any writes below SP may be clobbered.
   364  	p("#ifdef GOOS_ios")
   365  	p("MOVD R30, (RSP)")
   366  	p("#endif")
   367  
   368  	l.save()
   369  	p("CALL ·asyncPreempt2(SB)")
   370  	l.restore()
   371  
   372  	p("MOVD %d(RSP), R30", l.stack) // sigctxt.pushCall has pushed LR (at interrupt) on stack, restore it
   373  	p("#ifdef GOOS_linux")
   374  	p("MOVD -8(RSP), R29") // restore frame pointer
   375  	p("#endif")
   376  	p("MOVD (RSP), R27")          // load PC to REGTMP
   377  	p("ADD $%d, RSP", l.stack+16) // pop frame (including the space pushed by sigctxt.pushCall)
   378  	p("JMP (R27)")
   379  }
   380  
   381  func genMIPS(_64bit bool) {
   382  	mov := "MOVW"
   383  	movf := "MOVF"
   384  	add := "ADD"
   385  	sub := "SUB"
   386  	r28 := "R28"
   387  	regsize := 4
   388  	softfloat := "GOMIPS_softfloat"
   389  	if _64bit {
   390  		mov = "MOVV"
   391  		movf = "MOVD"
   392  		add = "ADDV"
   393  		sub = "SUBV"
   394  		r28 = "RSB"
   395  		regsize = 8
   396  		softfloat = "GOMIPS64_softfloat"
   397  	}
   398  
   399  	// Add integer registers R1-R22, R24-R25, R28
   400  	// R0 (zero), R23 (REGTMP), R29 (SP), R30 (g), R31 (LR) are special,
   401  	// and not saved here. R26 and R27 are reserved by kernel and not used.
   402  	var l = layout{sp: "R29", stack: regsize} // add slot to save PC of interrupted instruction (in LR)
   403  	for i := 1; i <= 25; i++ {
   404  		if i == 23 {
   405  			continue // R23 is REGTMP
   406  		}
   407  		reg := fmt.Sprintf("R%d", i)
   408  		l.add(mov, reg, regsize)
   409  	}
   410  	l.add(mov, r28, regsize)
   411  	l.addSpecial(
   412  		mov+" HI, R1\n"+mov+" R1, %d(R29)",
   413  		mov+" %d(R29), R1\n"+mov+" R1, HI",
   414  		regsize)
   415  	l.addSpecial(
   416  		mov+" LO, R1\n"+mov+" R1, %d(R29)",
   417  		mov+" %d(R29), R1\n"+mov+" R1, LO",
   418  		regsize)
   419  
   420  	// Add floating point control/status register FCR31 (FCR0-FCR30 are irrelevant)
   421  	var lfp = layout{sp: "R29", stack: l.stack}
   422  	lfp.addSpecial(
   423  		mov+" FCR31, R1\n"+mov+" R1, %d(R29)",
   424  		mov+" %d(R29), R1\n"+mov+" R1, FCR31",
   425  		regsize)
   426  	// Add floating point registers F0-F31.
   427  	for i := 0; i <= 31; i++ {
   428  		reg := fmt.Sprintf("F%d", i)
   429  		lfp.add(movf, reg, regsize)
   430  	}
   431  
   432  	// allocate frame, save PC of interrupted instruction (in LR)
   433  	p(mov+" R31, -%d(R29)", lfp.stack)
   434  	p(sub+" $%d, R29", lfp.stack)
   435  
   436  	l.save()
   437  	p("#ifndef %s", softfloat)
   438  	lfp.save()
   439  	p("#endif")
   440  	p("CALL ·asyncPreempt2(SB)")
   441  	p("#ifndef %s", softfloat)
   442  	lfp.restore()
   443  	p("#endif")
   444  	l.restore()
   445  
   446  	p(mov+" %d(R29), R31", lfp.stack)     // sigctxt.pushCall has pushed LR (at interrupt) on stack, restore it
   447  	p(mov + " (R29), R23")                // load PC to REGTMP
   448  	p(add+" $%d, R29", lfp.stack+regsize) // pop frame (including the space pushed by sigctxt.pushCall)
   449  	p("JMP (R23)")
   450  }
   451  
   452  func genPPC64() {
   453  	// Add integer registers R3-R29
   454  	// R0 (zero), R1 (SP), R30 (g) are special and not saved here.
   455  	// R2 (TOC pointer in PIC mode), R12 (function entry address in PIC mode) have been saved in sigctxt.pushCall.
   456  	// R31 (REGTMP) will be saved manually.
   457  	var l = layout{sp: "R1", stack: 32 + 8} // MinFrameSize on PPC64, plus one word for saving R31
   458  	for i := 3; i <= 29; i++ {
   459  		if i == 12 || i == 13 {
   460  			// R12 has been saved in sigctxt.pushCall.
   461  			// R13 is TLS pointer, not used by Go code. we must NOT
   462  			// restore it, otherwise if we parked and resumed on a
   463  			// different thread we'll mess up TLS addresses.
   464  			continue
   465  		}
   466  		reg := fmt.Sprintf("R%d", i)
   467  		l.add("MOVD", reg, 8)
   468  	}
   469  	l.addSpecial(
   470  		"MOVW CR, R31\nMOVW R31, %d(R1)",
   471  		"MOVW %d(R1), R31\nMOVFL R31, $0xff", // this is MOVW R31, CR
   472  		8)                                    // CR is 4-byte wide, but just keep the alignment
   473  	l.addSpecial(
   474  		"MOVD XER, R31\nMOVD R31, %d(R1)",
   475  		"MOVD %d(R1), R31\nMOVD R31, XER",
   476  		8)
   477  	// Add floating point registers F0-F31.
   478  	for i := 0; i <= 31; i++ {
   479  		reg := fmt.Sprintf("F%d", i)
   480  		l.add("FMOVD", reg, 8)
   481  	}
   482  	// Add floating point control/status register FPSCR.
   483  	l.addSpecial(
   484  		"MOVFL FPSCR, F0\nFMOVD F0, %d(R1)",
   485  		"FMOVD %d(R1), F0\nMOVFL F0, FPSCR",
   486  		8)
   487  
   488  	p("MOVD R31, -%d(R1)", l.stack-32) // save R31 first, we'll use R31 for saving LR
   489  	p("MOVD LR, R31")
   490  	p("MOVDU R31, -%d(R1)", l.stack) // allocate frame, save PC of interrupted instruction (in LR)
   491  
   492  	l.save()
   493  	p("CALL ·asyncPreempt2(SB)")
   494  	l.restore()
   495  
   496  	p("MOVD %d(R1), R31", l.stack) // sigctxt.pushCall has pushed LR, R2, R12 (at interrupt) on stack, restore them
   497  	p("MOVD R31, LR")
   498  	p("MOVD %d(R1), R2", l.stack+8)
   499  	p("MOVD %d(R1), R12", l.stack+16)
   500  	p("MOVD (R1), R31") // load PC to CTR
   501  	p("MOVD R31, CTR")
   502  	p("MOVD 32(R1), R31")        // restore R31
   503  	p("ADD $%d, R1", l.stack+32) // pop frame (including the space pushed by sigctxt.pushCall)
   504  	p("JMP (CTR)")
   505  }
   506  
   507  func genRISCV64() {
   508  	// X0 (zero), X1 (LR), X2 (SP), X4 (TP), X27 (g), X31 (TMP) are special.
   509  	var l = layout{sp: "X2", stack: 8}
   510  
   511  	// Add integer registers (X3, X5-X26, X28-30).
   512  	for i := 3; i < 31; i++ {
   513  		if i == 4 || i == 27 {
   514  			continue
   515  		}
   516  		reg := fmt.Sprintf("X%d", i)
   517  		l.add("MOV", reg, 8)
   518  	}
   519  
   520  	// Add floating point registers (F0-F31).
   521  	for i := 0; i <= 31; i++ {
   522  		reg := fmt.Sprintf("F%d", i)
   523  		l.add("MOVD", reg, 8)
   524  	}
   525  
   526  	p("MOV X1, -%d(X2)", l.stack)
   527  	p("ADD $-%d, X2", l.stack)
   528  	l.save()
   529  	p("CALL ·asyncPreempt2(SB)")
   530  	l.restore()
   531  	p("MOV %d(X2), X1", l.stack)
   532  	p("MOV (X2), X31")
   533  	p("ADD $%d, X2", l.stack+8)
   534  	p("JMP (X31)")
   535  }
   536  
   537  func genS390X() {
   538  	// Add integer registers R0-R12
   539  	// R13 (g), R14 (LR), R15 (SP) are special, and not saved here.
   540  	// Saving R10 (REGTMP) is not necessary, but it is saved anyway.
   541  	var l = layout{sp: "R15", stack: 16} // add slot to save PC of interrupted instruction and flags
   542  	l.addSpecial(
   543  		"STMG R0, R12, %d(R15)",
   544  		"LMG %d(R15), R0, R12",
   545  		13*8)
   546  	// Add floating point registers F0-F31.
   547  	for i := 0; i <= 15; i++ {
   548  		reg := fmt.Sprintf("F%d", i)
   549  		l.add("FMOVD", reg, 8)
   550  	}
   551  
   552  	// allocate frame, save PC of interrupted instruction (in LR) and flags (condition code)
   553  	p("IPM R10") // save flags upfront, as ADD will clobber flags
   554  	p("MOVD R14, -%d(R15)", l.stack)
   555  	p("ADD $-%d, R15", l.stack)
   556  	p("MOVW R10, 8(R15)") // save flags
   557  
   558  	l.save()
   559  	p("CALL ·asyncPreempt2(SB)")
   560  	l.restore()
   561  
   562  	p("MOVD %d(R15), R14", l.stack)    // sigctxt.pushCall has pushed LR (at interrupt) on stack, restore it
   563  	p("ADD $%d, R15", l.stack+8)       // pop frame (including the space pushed by sigctxt.pushCall)
   564  	p("MOVWZ -%d(R15), R10", l.stack)  // load flags to REGTMP
   565  	p("TMLH R10, $(3<<12)")            // restore flags
   566  	p("MOVD -%d(R15), R10", l.stack+8) // load PC to REGTMP
   567  	p("JMP (R10)")
   568  }
   569  
   570  func genWasm() {
   571  	p("// No async preemption on wasm")
   572  	p("UNDEF")
   573  }
   574  
   575  func notImplemented() {
   576  	p("// Not implemented yet")
   577  	p("JMP ·abort(SB)")
   578  }
   579
View as plain text