Text file
src/runtime/asm_riscv64.s
Documentation: runtime
1 // Copyright 2017 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4
5 #include "go_asm.h"
6 #include "funcdata.h"
7 #include "textflag.h"
8
9 // func rt0_go()
10 TEXT runtime·rt0_go(SB),NOSPLIT|TOPFRAME,$0
11 // X2 = stack; A0 = argc; A1 = argv
12 ADD $-24, X2
13 MOV A0, 8(X2) // argc
14 MOV A1, 16(X2) // argv
15
16 // create istack out of the given (operating system) stack.
17 // _cgo_init may update stackguard.
18 MOV $runtime·g0(SB), g
19 MOV $(-64*1024), T0
20 ADD T0, X2, T1
21 MOV T1, g_stackguard0(g)
22 MOV T1, g_stackguard1(g)
23 MOV T1, (g_stack+stack_lo)(g)
24 MOV X2, (g_stack+stack_hi)(g)
25
26 // if there is a _cgo_init, call it using the gcc ABI.
27 MOV _cgo_init(SB), T0
28 BEQ T0, ZERO, nocgo
29
30 MOV ZERO, A3 // arg 3: not used
31 MOV ZERO, A2 // arg 2: not used
32 MOV $setg_gcc<>(SB), A1 // arg 1: setg
33 MOV g, A0 // arg 0: G
34 JALR RA, T0
35
36 nocgo:
37 // update stackguard after _cgo_init
38 MOV (g_stack+stack_lo)(g), T0
39 ADD $const__StackGuard, T0
40 MOV T0, g_stackguard0(g)
41 MOV T0, g_stackguard1(g)
42
43 // set the per-goroutine and per-mach "registers"
44 MOV $runtime·m0(SB), T0
45
46 // save m->g0 = g0
47 MOV g, m_g0(T0)
48 // save m0 to g0->m
49 MOV T0, g_m(g)
50
51 CALL runtime·check(SB)
52
53 // args are already prepared
54 CALL runtime·args(SB)
55 CALL runtime·osinit(SB)
56 CALL runtime·schedinit(SB)
57
58 // create a new goroutine to start program
59 MOV $runtime·mainPC(SB), T0 // entry
60 ADD $-24, X2
61 MOV T0, 16(X2)
62 MOV ZERO, 8(X2)
63 MOV ZERO, 0(X2)
64 CALL runtime·newproc(SB)
65 ADD $24, X2
66
67 // start this M
68 CALL runtime·mstart(SB)
69
70 WORD $0 // crash if reached
71 RET
72
73 TEXT runtime·mstart(SB),NOSPLIT|TOPFRAME,$0
74 CALL runtime·mstart0(SB)
75 RET // not reached
76
77 // void setg_gcc(G*); set g called from gcc with g in A0
78 TEXT setg_gcc<>(SB),NOSPLIT,$0-0
79 MOV A0, g
80 CALL runtime·save_g(SB)
81 RET
82
83 // func cputicks() int64
84 TEXT runtime·cputicks(SB),NOSPLIT,$0-8
85 RDTIME A0
86 MOV A0, ret+0(FP)
87 RET
88
89 // systemstack_switch is a dummy routine that systemstack leaves at the bottom
90 // of the G stack. We need to distinguish the routine that
91 // lives at the bottom of the G stack from the one that lives
92 // at the top of the system stack because the one at the top of
93 // the system stack terminates the stack walk (see topofstack()).
94 TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0
95 UNDEF
96 JALR RA, ZERO // make sure this function is not leaf
97 RET
98
99 // func systemstack(fn func())
100 TEXT runtime·systemstack(SB), NOSPLIT, $0-8
101 MOV fn+0(FP), CTXT // CTXT = fn
102 MOV g_m(g), T0 // T0 = m
103
104 MOV m_gsignal(T0), T1 // T1 = gsignal
105 BEQ g, T1, noswitch
106
107 MOV m_g0(T0), T1 // T1 = g0
108 BEQ g, T1, noswitch
109
110 MOV m_curg(T0), T2
111 BEQ g, T2, switch
112
113 // Bad: g is not gsignal, not g0, not curg. What is it?
114 // Hide call from linker nosplit analysis.
115 MOV $runtime·badsystemstack(SB), T1
116 JALR RA, T1
117
118 switch:
119 // save our state in g->sched. Pretend to
120 // be systemstack_switch if the G stack is scanned.
121 CALL gosave_systemstack_switch<>(SB)
122
123 // switch to g0
124 MOV T1, g
125 CALL runtime·save_g(SB)
126 MOV (g_sched+gobuf_sp)(g), T0
127 MOV T0, X2
128
129 // call target function
130 MOV 0(CTXT), T1 // code pointer
131 JALR RA, T1
132
133 // switch back to g
134 MOV g_m(g), T0
135 MOV m_curg(T0), g
136 CALL runtime·save_g(SB)
137 MOV (g_sched+gobuf_sp)(g), X2
138 MOV ZERO, (g_sched+gobuf_sp)(g)
139 RET
140
141 noswitch:
142 // already on m stack, just call directly
143 // Using a tail call here cleans up tracebacks since we won't stop
144 // at an intermediate systemstack.
145 MOV 0(CTXT), T1 // code pointer
146 ADD $8, X2
147 JMP (T1)
148
149 TEXT runtime·getcallerpc(SB),NOSPLIT|NOFRAME,$0-8
150 MOV 0(X2), T0 // LR saved by caller
151 MOV T0, ret+0(FP)
152 RET
153
154 /*
155 * support for morestack
156 */
157
158 // Called during function prolog when more stack is needed.
159 // Caller has already loaded:
160 // R1: framesize, R2: argsize, R3: LR
161 //
162 // The traceback routines see morestack on a g0 as being
163 // the top of a stack (for example, morestack calling newstack
164 // calling the scheduler calling newm calling gc), so we must
165 // record an argument size. For that purpose, it has no arguments.
166
167 // func morestack()
168 TEXT runtime·morestack(SB),NOSPLIT|NOFRAME,$0-0
169 // Cannot grow scheduler stack (m->g0).
170 MOV g_m(g), A0
171 MOV m_g0(A0), A1
172 BNE g, A1, 3(PC)
173 CALL runtime·badmorestackg0(SB)
174 CALL runtime·abort(SB)
175
176 // Cannot grow signal stack (m->gsignal).
177 MOV m_gsignal(A0), A1
178 BNE g, A1, 3(PC)
179 CALL runtime·badmorestackgsignal(SB)
180 CALL runtime·abort(SB)
181
182 // Called from f.
183 // Set g->sched to context in f.
184 MOV X2, (g_sched+gobuf_sp)(g)
185 MOV T0, (g_sched+gobuf_pc)(g)
186 MOV RA, (g_sched+gobuf_lr)(g)
187 MOV CTXT, (g_sched+gobuf_ctxt)(g)
188
189 // Called from f.
190 // Set m->morebuf to f's caller.
191 MOV RA, (m_morebuf+gobuf_pc)(A0) // f's caller's PC
192 MOV X2, (m_morebuf+gobuf_sp)(A0) // f's caller's SP
193 MOV g, (m_morebuf+gobuf_g)(A0)
194
195 // Call newstack on m->g0's stack.
196 MOV m_g0(A0), g
197 CALL runtime·save_g(SB)
198 MOV (g_sched+gobuf_sp)(g), X2
199 // Create a stack frame on g0 to call newstack.
200 MOV ZERO, -8(X2) // Zero saved LR in frame
201 ADD $-8, X2
202 CALL runtime·newstack(SB)
203
204 // Not reached, but make sure the return PC from the call to newstack
205 // is still in this function, and not the beginning of the next.
206 UNDEF
207
208 // func morestack_noctxt()
209 TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0
210 MOV ZERO, CTXT
211 JMP runtime·morestack(SB)
212
213 // AES hashing not implemented for riscv64
214 TEXT runtime·memhash(SB),NOSPLIT|NOFRAME,$0-32
215 JMP runtime·memhashFallback(SB)
216 TEXT runtime·strhash(SB),NOSPLIT|NOFRAME,$0-24
217 JMP runtime·strhashFallback(SB)
218 TEXT runtime·memhash32(SB),NOSPLIT|NOFRAME,$0-24
219 JMP runtime·memhash32Fallback(SB)
220 TEXT runtime·memhash64(SB),NOSPLIT|NOFRAME,$0-24
221 JMP runtime·memhash64Fallback(SB)
222
223 // func return0()
224 TEXT runtime·return0(SB), NOSPLIT, $0
225 MOV $0, A0
226 RET
227
228 // restore state from Gobuf; longjmp
229
230 // func gogo(buf *gobuf)
231 TEXT runtime·gogo(SB), NOSPLIT|NOFRAME, $0-8
232 MOV buf+0(FP), T0
233 MOV gobuf_g(T0), T1
234 MOV 0(T1), ZERO // make sure g != nil
235 JMP gogo<>(SB)
236
237 TEXT gogo<>(SB), NOSPLIT|NOFRAME, $0
238 MOV T1, g
239 CALL runtime·save_g(SB)
240
241 MOV gobuf_sp(T0), X2
242 MOV gobuf_lr(T0), RA
243 MOV gobuf_ret(T0), A0
244 MOV gobuf_ctxt(T0), CTXT
245 MOV ZERO, gobuf_sp(T0)
246 MOV ZERO, gobuf_ret(T0)
247 MOV ZERO, gobuf_lr(T0)
248 MOV ZERO, gobuf_ctxt(T0)
249 MOV gobuf_pc(T0), T0
250 JALR ZERO, T0
251
252 // func jmpdefer(fv *funcval, argp uintptr)
253 // called from deferreturn
254 // 1. grab stored return address from the caller's frame
255 // 2. sub 8 bytes to get back to JAL deferreturn
256 // 3. JMP to fn
257 TEXT runtime·jmpdefer(SB), NOSPLIT|NOFRAME, $0-16
258 MOV 0(X2), RA
259 ADD $-8, RA
260
261 MOV fv+0(FP), CTXT
262 MOV argp+8(FP), X2
263 ADD $-8, X2
264 MOV 0(CTXT), T0
265 JALR ZERO, T0
266
267 // func procyield(cycles uint32)
268 TEXT runtime·procyield(SB),NOSPLIT,$0-0
269 RET
270
271 // Switch to m->g0's stack, call fn(g).
272 // Fn must never return. It should gogo(&g->sched)
273 // to keep running g.
274
275 // func mcall(fn func(*g))
276 TEXT runtime·mcall(SB), NOSPLIT|NOFRAME, $0-8
277 // Save caller state in g->sched
278 MOV X2, (g_sched+gobuf_sp)(g)
279 MOV RA, (g_sched+gobuf_pc)(g)
280 MOV ZERO, (g_sched+gobuf_lr)(g)
281
282 // Switch to m->g0 & its stack, call fn.
283 MOV g, T0
284 MOV g_m(g), T1
285 MOV m_g0(T1), g
286 CALL runtime·save_g(SB)
287 BNE g, T0, 2(PC)
288 JMP runtime·badmcall(SB)
289 MOV fn+0(FP), CTXT // context
290 MOV 0(CTXT), T1 // code pointer
291 MOV (g_sched+gobuf_sp)(g), X2 // sp = m->g0->sched.sp
292 ADD $-16, X2
293 MOV T0, 8(X2)
294 MOV ZERO, 0(X2)
295 JALR RA, T1
296 JMP runtime·badmcall2(SB)
297
298 // Save state of caller into g->sched,
299 // but using fake PC from systemstack_switch.
300 // Must only be called from functions with no locals ($0)
301 // or else unwinding from systemstack_switch is incorrect.
302 // Smashes X31.
303 TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0
304 MOV $runtime·systemstack_switch(SB), X31
305 ADD $8, X31 // get past prologue
306 MOV X31, (g_sched+gobuf_pc)(g)
307 MOV X2, (g_sched+gobuf_sp)(g)
308 MOV ZERO, (g_sched+gobuf_lr)(g)
309 MOV ZERO, (g_sched+gobuf_ret)(g)
310 // Assert ctxt is zero. See func save.
311 MOV (g_sched+gobuf_ctxt)(g), X31
312 BEQ ZERO, X31, 2(PC)
313 CALL runtime·abort(SB)
314 RET
315
316 // func asmcgocall(fn, arg unsafe.Pointer) int32
317 // Call fn(arg) on the scheduler stack,
318 // aligned appropriately for the gcc ABI.
319 // See cgocall.go for more details.
320 TEXT ·asmcgocall(SB),NOSPLIT,$0-20
321 MOV fn+0(FP), X5
322 MOV arg+8(FP), X10
323
324 MOV X2, X8 // save original stack pointer
325 MOV g, X9
326
327 // Figure out if we need to switch to m->g0 stack.
328 // We get called to create new OS threads too, and those
329 // come in on the m->g0 stack already.
330 MOV g_m(g), X6
331 MOV m_g0(X6), X7
332 BEQ X7, g, g0
333
334 CALL gosave_systemstack_switch<>(SB)
335 MOV X7, g
336 CALL runtime·save_g(SB)
337 MOV (g_sched+gobuf_sp)(g), X2
338
339 // Now on a scheduling stack (a pthread-created stack).
340 g0:
341 // Save room for two of our pointers.
342 ADD $-16, X2
343 MOV X9, 0(X2) // save old g on stack
344 MOV (g_stack+stack_hi)(X9), X9
345 SUB X8, X9, X8
346 MOV X8, 8(X2) // save depth in old g stack (can't just save SP, as stack might be copied during a callback)
347
348 JALR RA, (X5)
349
350 // Restore g, stack pointer. X10 is return value.
351 MOV 0(X2), g
352 CALL runtime·save_g(SB)
353 MOV (g_stack+stack_hi)(g), X5
354 MOV 8(X2), X6
355 SUB X6, X5, X6
356 MOV X6, X2
357
358 MOVW X10, ret+16(FP)
359 RET
360
361 // func asminit()
362 TEXT runtime·asminit(SB),NOSPLIT|NOFRAME,$0-0
363 RET
364
365 // reflectcall: call a function with the given argument list
366 // func call(stackArgsType *_type, f *FuncVal, stackArgs *byte, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
367 // we don't have variable-sized frames, so we use a small number
368 // of constant-sized-frame functions to encode a few bits of size in the pc.
369 // Caution: ugly multiline assembly macros in your future!
370
371 #define DISPATCH(NAME,MAXSIZE) \
372 MOV $MAXSIZE, T1 \
373 BLTU T1, T0, 3(PC) \
374 MOV $NAME(SB), T2; \
375 JALR ZERO, T2
376 // Note: can't just "BR NAME(SB)" - bad inlining results.
377
378 // func call(stackArgsType *rtype, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
379 TEXT reflect·call(SB), NOSPLIT, $0-0
380 JMP ·reflectcall(SB)
381
382 // func call(stackArgsType *_type, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
383 TEXT ·reflectcall(SB), NOSPLIT|NOFRAME, $0-48
384 MOVWU frameSize+32(FP), T0
385 DISPATCH(runtime·call16, 16)
386 DISPATCH(runtime·call32, 32)
387 DISPATCH(runtime·call64, 64)
388 DISPATCH(runtime·call128, 128)
389 DISPATCH(runtime·call256, 256)
390 DISPATCH(runtime·call512, 512)
391 DISPATCH(runtime·call1024, 1024)
392 DISPATCH(runtime·call2048, 2048)
393 DISPATCH(runtime·call4096, 4096)
394 DISPATCH(runtime·call8192, 8192)
395 DISPATCH(runtime·call16384, 16384)
396 DISPATCH(runtime·call32768, 32768)
397 DISPATCH(runtime·call65536, 65536)
398 DISPATCH(runtime·call131072, 131072)
399 DISPATCH(runtime·call262144, 262144)
400 DISPATCH(runtime·call524288, 524288)
401 DISPATCH(runtime·call1048576, 1048576)
402 DISPATCH(runtime·call2097152, 2097152)
403 DISPATCH(runtime·call4194304, 4194304)
404 DISPATCH(runtime·call8388608, 8388608)
405 DISPATCH(runtime·call16777216, 16777216)
406 DISPATCH(runtime·call33554432, 33554432)
407 DISPATCH(runtime·call67108864, 67108864)
408 DISPATCH(runtime·call134217728, 134217728)
409 DISPATCH(runtime·call268435456, 268435456)
410 DISPATCH(runtime·call536870912, 536870912)
411 DISPATCH(runtime·call1073741824, 1073741824)
412 MOV $runtime·badreflectcall(SB), T2
413 JALR ZERO, T2
414
415 #define CALLFN(NAME,MAXSIZE) \
416 TEXT NAME(SB), WRAPPER, $MAXSIZE-48; \
417 NO_LOCAL_POINTERS; \
418 /* copy arguments to stack */ \
419 MOV stackArgs+16(FP), A1; \
420 MOVWU stackArgsSize+24(FP), A2; \
421 MOV X2, A3; \
422 ADD $8, A3; \
423 ADD A3, A2; \
424 BEQ A3, A2, 6(PC); \
425 MOVBU (A1), A4; \
426 ADD $1, A1; \
427 MOVB A4, (A3); \
428 ADD $1, A3; \
429 JMP -5(PC); \
430 /* call function */ \
431 MOV f+8(FP), CTXT; \
432 MOV (CTXT), A4; \
433 PCDATA $PCDATA_StackMapIndex, $0; \
434 JALR RA, A4; \
435 /* copy return values back */ \
436 MOV stackArgsType+0(FP), A5; \
437 MOV stackArgs+16(FP), A1; \
438 MOVWU stackArgsSize+24(FP), A2; \
439 MOVWU stackRetOffset+28(FP), A4; \
440 ADD $8, X2, A3; \
441 ADD A4, A3; \
442 ADD A4, A1; \
443 SUB A4, A2; \
444 CALL callRet<>(SB); \
445 RET
446
447 // callRet copies return values back at the end of call*. This is a
448 // separate function so it can allocate stack space for the arguments
449 // to reflectcallmove. It does not follow the Go ABI; it expects its
450 // arguments in registers.
451 TEXT callRet<>(SB), NOSPLIT, $40-0
452 MOV A5, 8(X2)
453 MOV A1, 16(X2)
454 MOV A3, 24(X2)
455 MOV A2, 32(X2)
456 MOV ZERO, 40(X2)
457 CALL runtime·reflectcallmove(SB)
458 RET
459
460 CALLFN(·call16, 16)
461 CALLFN(·call32, 32)
462 CALLFN(·call64, 64)
463 CALLFN(·call128, 128)
464 CALLFN(·call256, 256)
465 CALLFN(·call512, 512)
466 CALLFN(·call1024, 1024)
467 CALLFN(·call2048, 2048)
468 CALLFN(·call4096, 4096)
469 CALLFN(·call8192, 8192)
470 CALLFN(·call16384, 16384)
471 CALLFN(·call32768, 32768)
472 CALLFN(·call65536, 65536)
473 CALLFN(·call131072, 131072)
474 CALLFN(·call262144, 262144)
475 CALLFN(·call524288, 524288)
476 CALLFN(·call1048576, 1048576)
477 CALLFN(·call2097152, 2097152)
478 CALLFN(·call4194304, 4194304)
479 CALLFN(·call8388608, 8388608)
480 CALLFN(·call16777216, 16777216)
481 CALLFN(·call33554432, 33554432)
482 CALLFN(·call67108864, 67108864)
483 CALLFN(·call134217728, 134217728)
484 CALLFN(·call268435456, 268435456)
485 CALLFN(·call536870912, 536870912)
486 CALLFN(·call1073741824, 1073741824)
487
488 // Called from cgo wrappers, this function returns g->m->curg.stack.hi.
489 // Must obey the gcc calling convention.
490 TEXT _cgo_topofstack(SB),NOSPLIT,$8
491 // g (X27) and REG_TMP (X31) might be clobbered by load_g.
492 // X27 is callee-save in the gcc calling convention, so save it.
493 MOV g, savedX27-8(SP)
494
495 CALL runtime·load_g(SB)
496 MOV g_m(g), X5
497 MOV m_curg(X5), X5
498 MOV (g_stack+stack_hi)(X5), X10 // return value in X10
499
500 MOV savedX27-8(SP), g
501 RET
502
503 // func goexit(neverCallThisFunction)
504 // The top-most function running on a goroutine
505 // returns to goexit+PCQuantum.
506 TEXT runtime·goexit(SB),NOSPLIT|NOFRAME|TOPFRAME,$0-0
507 MOV ZERO, ZERO // NOP
508 JMP runtime·goexit1(SB) // does not return
509 // traceback from goexit1 must hit code range of goexit
510 MOV ZERO, ZERO // NOP
511
512 // func cgocallback(fn, frame unsafe.Pointer, ctxt uintptr)
513 // See cgocall.go for more details.
514 TEXT ·cgocallback(SB),NOSPLIT,$24-24
515 NO_LOCAL_POINTERS
516
517 // Load m and g from thread-local storage.
518 MOVBU runtime·iscgo(SB), X5
519 BEQ ZERO, X5, nocgo
520 CALL runtime·load_g(SB)
521 nocgo:
522
523 // If g is nil, Go did not create the current thread.
524 // Call needm to obtain one for temporary use.
525 // In this case, we're running on the thread stack, so there's
526 // lots of space, but the linker doesn't know. Hide the call from
527 // the linker analysis by using an indirect call.
528 BEQ ZERO, g, needm
529
530 MOV g_m(g), X5
531 MOV X5, savedm-8(SP)
532 JMP havem
533
534 needm:
535 MOV g, savedm-8(SP) // g is zero, so is m.
536 MOV $runtime·needm(SB), X6
537 JALR RA, X6
538
539 // Set m->sched.sp = SP, so that if a panic happens
540 // during the function we are about to execute, it will
541 // have a valid SP to run on the g0 stack.
542 // The next few lines (after the havem label)
543 // will save this SP onto the stack and then write
544 // the same SP back to m->sched.sp. That seems redundant,
545 // but if an unrecovered panic happens, unwindm will
546 // restore the g->sched.sp from the stack location
547 // and then systemstack will try to use it. If we don't set it here,
548 // that restored SP will be uninitialized (typically 0) and
549 // will not be usable.
550 MOV g_m(g), X5
551 MOV m_g0(X5), X6
552 MOV X2, (g_sched+gobuf_sp)(X6)
553
554 havem:
555 // Now there's a valid m, and we're running on its m->g0.
556 // Save current m->g0->sched.sp on stack and then set it to SP.
557 // Save current sp in m->g0->sched.sp in preparation for
558 // switch back to m->curg stack.
559 // NOTE: unwindm knows that the saved g->sched.sp is at 8(X2) aka savedsp-24(SP).
560 MOV m_g0(X5), X6
561 MOV (g_sched+gobuf_sp)(X6), X7
562 MOV X7, savedsp-24(SP) // must match frame size
563 MOV X2, (g_sched+gobuf_sp)(X6)
564
565 // Switch to m->curg stack and call runtime.cgocallbackg.
566 // Because we are taking over the execution of m->curg
567 // but *not* resuming what had been running, we need to
568 // save that information (m->curg->sched) so we can restore it.
569 // We can restore m->curg->sched.sp easily, because calling
570 // runtime.cgocallbackg leaves SP unchanged upon return.
571 // To save m->curg->sched.pc, we push it onto the curg stack and
572 // open a frame the same size as cgocallback's g0 frame.
573 // Once we switch to the curg stack, the pushed PC will appear
574 // to be the return PC of cgocallback, so that the traceback
575 // will seamlessly trace back into the earlier calls.
576 MOV m_curg(X5), g
577 CALL runtime·save_g(SB)
578 MOV (g_sched+gobuf_sp)(g), X6 // prepare stack as X6
579 MOV (g_sched+gobuf_pc)(g), X7
580 MOV X7, -(24+8)(X6) // "saved LR"; must match frame size
581 // Gather our arguments into registers.
582 MOV fn+0(FP), X7
583 MOV frame+8(FP), X8
584 MOV ctxt+16(FP), X9
585 MOV $-(24+8)(X6), X2 // switch stack; must match frame size
586 MOV X7, 8(X2)
587 MOV X8, 16(X2)
588 MOV X9, 24(X2)
589 CALL runtime·cgocallbackg(SB)
590
591 // Restore g->sched (== m->curg->sched) from saved values.
592 MOV 0(X2), X7
593 MOV X7, (g_sched+gobuf_pc)(g)
594 MOV $(24+8)(X2), X6 // must match frame size
595 MOV X6, (g_sched+gobuf_sp)(g)
596
597 // Switch back to m->g0's stack and restore m->g0->sched.sp.
598 // (Unlike m->curg, the g0 goroutine never uses sched.pc,
599 // so we do not have to restore it.)
600 MOV g_m(g), X5
601 MOV m_g0(X5), g
602 CALL runtime·save_g(SB)
603 MOV (g_sched+gobuf_sp)(g), X2
604 MOV savedsp-24(SP), X6 // must match frame size
605 MOV X6, (g_sched+gobuf_sp)(g)
606
607 // If the m on entry was nil, we called needm above to borrow an m
608 // for the duration of the call. Since the call is over, return it with dropm.
609 MOV savedm-8(SP), X5
610 BNE ZERO, X5, droppedm
611 MOV $runtime·dropm(SB), X6
612 JALR RA, X6
613 droppedm:
614
615 // Done!
616 RET
617
618 TEXT runtime·breakpoint(SB),NOSPLIT|NOFRAME,$0-0
619 EBREAK
620 RET
621
622 TEXT runtime·abort(SB),NOSPLIT|NOFRAME,$0-0
623 EBREAK
624 RET
625
626 // void setg(G*); set g. for use by needm.
627 TEXT runtime·setg(SB), NOSPLIT, $0-8
628 MOV gg+0(FP), g
629 // This only happens if iscgo, so jump straight to save_g
630 CALL runtime·save_g(SB)
631 RET
632
633 TEXT ·checkASM(SB),NOSPLIT,$0-1
634 MOV $1, T0
635 MOV T0, ret+0(FP)
636 RET
637
638 // gcWriteBarrier performs a heap pointer write and informs the GC.
639 //
640 // gcWriteBarrier does NOT follow the Go ABI. It takes two arguments:
641 // - T0 is the destination of the write
642 // - T1 is the value being written at T0.
643 // It clobbers R30 (the linker temp register - REG_TMP).
644 // The act of CALLing gcWriteBarrier will clobber RA (LR).
645 // It does not clobber any other general-purpose registers,
646 // but may clobber others (e.g., floating point registers).
647 TEXT runtime·gcWriteBarrier(SB),NOSPLIT,$216
648 // Save the registers clobbered by the fast path.
649 MOV A0, 25*8(X2)
650 MOV A1, 26*8(X2)
651 MOV g_m(g), A0
652 MOV m_p(A0), A0
653 MOV (p_wbBuf+wbBuf_next)(A0), A1
654 // Increment wbBuf.next position.
655 ADD $16, A1
656 MOV A1, (p_wbBuf+wbBuf_next)(A0)
657 MOV (p_wbBuf+wbBuf_end)(A0), A0
658 MOV A0, T6 // T6 is linker temp register (REG_TMP)
659 // Record the write.
660 MOV T1, -16(A1) // Record value
661 MOV (T0), A0 // TODO: This turns bad writes into bad reads.
662 MOV A0, -8(A1) // Record *slot
663 // Is the buffer full?
664 BEQ A1, T6, flush
665 ret:
666 MOV 25*8(X2), A0
667 MOV 26*8(X2), A1
668 // Do the write.
669 MOV T1, (T0)
670 RET
671
672 flush:
673 // Save all general purpose registers since these could be
674 // clobbered by wbBufFlush and were not saved by the caller.
675 MOV T0, 1*8(X2) // Also first argument to wbBufFlush
676 MOV T1, 2*8(X2) // Also second argument to wbBufFlush
677 // X0 is zero register
678 // X1 is LR, saved by prologue
679 // X2 is SP
680 MOV X3, 3*8(X2)
681 // X4 is TP
682 // X5 is first arg to wbBufFlush (T0)
683 // X6 is second arg to wbBufFlush (T1)
684 MOV X7, 4*8(X2)
685 MOV X8, 5*8(X2)
686 MOV X9, 6*8(X2)
687 // X10 already saved (A0)
688 // X11 already saved (A1)
689 MOV X12, 7*8(X2)
690 MOV X13, 8*8(X2)
691 MOV X14, 9*8(X2)
692 MOV X15, 10*8(X2)
693 MOV X16, 11*8(X2)
694 MOV X17, 12*8(X2)
695 MOV X18, 13*8(X2)
696 MOV X19, 14*8(X2)
697 MOV X20, 15*8(X2)
698 MOV X21, 16*8(X2)
699 MOV X22, 17*8(X2)
700 MOV X23, 18*8(X2)
701 MOV X24, 19*8(X2)
702 MOV X25, 20*8(X2)
703 MOV X26, 21*8(X2)
704 // X27 is g.
705 MOV X28, 22*8(X2)
706 MOV X29, 23*8(X2)
707 MOV X30, 24*8(X2)
708 // X31 is tmp register.
709
710 // This takes arguments T0 and T1.
711 CALL runtime·wbBufFlush(SB)
712
713 MOV 1*8(X2), T0
714 MOV 2*8(X2), T1
715 MOV 3*8(X2), X3
716 MOV 4*8(X2), X7
717 MOV 5*8(X2), X8
718 MOV 6*8(X2), X9
719 MOV 7*8(X2), X12
720 MOV 8*8(X2), X13
721 MOV 9*8(X2), X14
722 MOV 10*8(X2), X15
723 MOV 11*8(X2), X16
724 MOV 12*8(X2), X17
725 MOV 13*8(X2), X18
726 MOV 14*8(X2), X19
727 MOV 15*8(X2), X20
728 MOV 16*8(X2), X21
729 MOV 17*8(X2), X22
730 MOV 18*8(X2), X23
731 MOV 19*8(X2), X24
732 MOV 20*8(X2), X25
733 MOV 21*8(X2), X26
734 MOV 22*8(X2), X28
735 MOV 23*8(X2), X29
736 MOV 24*8(X2), X30
737
738 JMP ret
739
740 // Note: these functions use a special calling convention to save generated code space.
741 // Arguments are passed in registers, but the space for those arguments are allocated
742 // in the caller's stack frame. These stubs write the args into that stack space and
743 // then tail call to the corresponding runtime handler.
744 // The tail call makes these stubs disappear in backtraces.
745 TEXT runtime·panicIndex(SB),NOSPLIT,$0-16
746 MOV T0, x+0(FP)
747 MOV T1, y+8(FP)
748 JMP runtime·goPanicIndex(SB)
749 TEXT runtime·panicIndexU(SB),NOSPLIT,$0-16
750 MOV T0, x+0(FP)
751 MOV T1, y+8(FP)
752 JMP runtime·goPanicIndexU(SB)
753 TEXT runtime·panicSliceAlen(SB),NOSPLIT,$0-16
754 MOV T1, x+0(FP)
755 MOV T2, y+8(FP)
756 JMP runtime·goPanicSliceAlen(SB)
757 TEXT runtime·panicSliceAlenU(SB),NOSPLIT,$0-16
758 MOV T1, x+0(FP)
759 MOV T2, y+8(FP)
760 JMP runtime·goPanicSliceAlenU(SB)
761 TEXT runtime·panicSliceAcap(SB),NOSPLIT,$0-16
762 MOV T1, x+0(FP)
763 MOV T2, y+8(FP)
764 JMP runtime·goPanicSliceAcap(SB)
765 TEXT runtime·panicSliceAcapU(SB),NOSPLIT,$0-16
766 MOV T1, x+0(FP)
767 MOV T2, y+8(FP)
768 JMP runtime·goPanicSliceAcapU(SB)
769 TEXT runtime·panicSliceB(SB),NOSPLIT,$0-16
770 MOV T0, x+0(FP)
771 MOV T1, y+8(FP)
772 JMP runtime·goPanicSliceB(SB)
773 TEXT runtime·panicSliceBU(SB),NOSPLIT,$0-16
774 MOV T0, x+0(FP)
775 MOV T1, y+8(FP)
776 JMP runtime·goPanicSliceBU(SB)
777 TEXT runtime·panicSlice3Alen(SB),NOSPLIT,$0-16
778 MOV T2, x+0(FP)
779 MOV T3, y+8(FP)
780 JMP runtime·goPanicSlice3Alen(SB)
781 TEXT runtime·panicSlice3AlenU(SB),NOSPLIT,$0-16
782 MOV T2, x+0(FP)
783 MOV T3, y+8(FP)
784 JMP runtime·goPanicSlice3AlenU(SB)
785 TEXT runtime·panicSlice3Acap(SB),NOSPLIT,$0-16
786 MOV T2, x+0(FP)
787 MOV T3, y+8(FP)
788 JMP runtime·goPanicSlice3Acap(SB)
789 TEXT runtime·panicSlice3AcapU(SB),NOSPLIT,$0-16
790 MOV T2, x+0(FP)
791 MOV T3, y+8(FP)
792 JMP runtime·goPanicSlice3AcapU(SB)
793 TEXT runtime·panicSlice3B(SB),NOSPLIT,$0-16
794 MOV T1, x+0(FP)
795 MOV T2, y+8(FP)
796 JMP runtime·goPanicSlice3B(SB)
797 TEXT runtime·panicSlice3BU(SB),NOSPLIT,$0-16
798 MOV T1, x+0(FP)
799 MOV T2, y+8(FP)
800 JMP runtime·goPanicSlice3BU(SB)
801 TEXT runtime·panicSlice3C(SB),NOSPLIT,$0-16
802 MOV T0, x+0(FP)
803 MOV T1, y+8(FP)
804 JMP runtime·goPanicSlice3C(SB)
805 TEXT runtime·panicSlice3CU(SB),NOSPLIT,$0-16
806 MOV T0, x+0(FP)
807 MOV T1, y+8(FP)
808 JMP runtime·goPanicSlice3CU(SB)
809 TEXT runtime·panicSliceConvert(SB),NOSPLIT,$0-16
810 MOV T2, x+0(FP)
811 MOV T3, y+8(FP)
812 JMP runtime·goPanicSliceConvert(SB)
813
814 DATA runtime·mainPC+0(SB)/8,$runtime·main(SB)
815 GLOBL runtime·mainPC(SB),RODATA,$8
816
View as plain text