Source file
src/runtime/netpoll.go
Documentation: runtime
1
2
3
4
5
6
7
8 package runtime
9
10 import (
11 "runtime/internal/atomic"
12 "unsafe"
13 )
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42 const (
43 pollNoError = 0
44 pollErrClosing = 1
45 pollErrTimeout = 2
46 pollErrNotPollable = 3
47 )
48
49
50
51
52
53
54
55
56
57
58
59
60
61 const (
62 pdReady uintptr = 1
63 pdWait uintptr = 2
64 )
65
66 const pollBlockSize = 4 * 1024
67
68
69
70
71
72
73 type pollDesc struct {
74 link *pollDesc
75
76
77
78
79
80
81
82
83 lock mutex
84 fd uintptr
85 closing bool
86 everr bool
87 user uint32
88 rseq uintptr
89 rg uintptr
90 rt timer
91 rd int64
92 wseq uintptr
93 wg uintptr
94 wt timer
95 wd int64
96 self *pollDesc
97 }
98
99 type pollCache struct {
100 lock mutex
101 first *pollDesc
102
103
104
105
106
107 }
108
109 var (
110 netpollInitLock mutex
111 netpollInited uint32
112
113 pollcache pollCache
114 netpollWaiters uint32
115 )
116
117
118 func poll_runtime_pollServerInit() {
119 netpollGenericInit()
120 }
121
122 func netpollGenericInit() {
123 if atomic.Load(&netpollInited) == 0 {
124 lockInit(&netpollInitLock, lockRankNetpollInit)
125 lock(&netpollInitLock)
126 if netpollInited == 0 {
127 netpollinit()
128 atomic.Store(&netpollInited, 1)
129 }
130 unlock(&netpollInitLock)
131 }
132 }
133
134 func netpollinited() bool {
135 return atomic.Load(&netpollInited) != 0
136 }
137
138
139
140
141
142 func poll_runtime_isPollServerDescriptor(fd uintptr) bool {
143 return netpollIsPollDescriptor(fd)
144 }
145
146
147 func poll_runtime_pollOpen(fd uintptr) (*pollDesc, int) {
148 pd := pollcache.alloc()
149 lock(&pd.lock)
150 if pd.wg != 0 && pd.wg != pdReady {
151 throw("runtime: blocked write on free polldesc")
152 }
153 if pd.rg != 0 && pd.rg != pdReady {
154 throw("runtime: blocked read on free polldesc")
155 }
156 pd.fd = fd
157 pd.closing = false
158 pd.everr = false
159 pd.rseq++
160 pd.rg = 0
161 pd.rd = 0
162 pd.wseq++
163 pd.wg = 0
164 pd.wd = 0
165 pd.self = pd
166 unlock(&pd.lock)
167
168 errno := netpollopen(fd, pd)
169 if errno != 0 {
170 pollcache.free(pd)
171 return nil, int(errno)
172 }
173 return pd, 0
174 }
175
176
177 func poll_runtime_pollClose(pd *pollDesc) {
178 if !pd.closing {
179 throw("runtime: close polldesc w/o unblock")
180 }
181 if pd.wg != 0 && pd.wg != pdReady {
182 throw("runtime: blocked write on closing polldesc")
183 }
184 if pd.rg != 0 && pd.rg != pdReady {
185 throw("runtime: blocked read on closing polldesc")
186 }
187 netpollclose(pd.fd)
188 pollcache.free(pd)
189 }
190
191 func (c *pollCache) free(pd *pollDesc) {
192 lock(&c.lock)
193 pd.link = c.first
194 c.first = pd
195 unlock(&c.lock)
196 }
197
198
199
200
201
202 func poll_runtime_pollReset(pd *pollDesc, mode int) int {
203 errcode := netpollcheckerr(pd, int32(mode))
204 if errcode != pollNoError {
205 return errcode
206 }
207 if mode == 'r' {
208 pd.rg = 0
209 } else if mode == 'w' {
210 pd.wg = 0
211 }
212 return pollNoError
213 }
214
215
216
217
218
219
220 func poll_runtime_pollWait(pd *pollDesc, mode int) int {
221 errcode := netpollcheckerr(pd, int32(mode))
222 if errcode != pollNoError {
223 return errcode
224 }
225
226 if GOOS == "solaris" || GOOS == "illumos" || GOOS == "aix" {
227 netpollarm(pd, mode)
228 }
229 for !netpollblock(pd, int32(mode), false) {
230 errcode = netpollcheckerr(pd, int32(mode))
231 if errcode != pollNoError {
232 return errcode
233 }
234
235
236
237 }
238 return pollNoError
239 }
240
241
242 func poll_runtime_pollWaitCanceled(pd *pollDesc, mode int) {
243
244
245 for !netpollblock(pd, int32(mode), true) {
246 }
247 }
248
249
250 func poll_runtime_pollSetDeadline(pd *pollDesc, d int64, mode int) {
251 lock(&pd.lock)
252 if pd.closing {
253 unlock(&pd.lock)
254 return
255 }
256 rd0, wd0 := pd.rd, pd.wd
257 combo0 := rd0 > 0 && rd0 == wd0
258 if d > 0 {
259 d += nanotime()
260 if d <= 0 {
261
262
263 d = 1<<63 - 1
264 }
265 }
266 if mode == 'r' || mode == 'r'+'w' {
267 pd.rd = d
268 }
269 if mode == 'w' || mode == 'r'+'w' {
270 pd.wd = d
271 }
272 combo := pd.rd > 0 && pd.rd == pd.wd
273 rtf := netpollReadDeadline
274 if combo {
275 rtf = netpollDeadline
276 }
277 if pd.rt.f == nil {
278 if pd.rd > 0 {
279 pd.rt.f = rtf
280
281
282
283 pd.rt.arg = pd.makeArg()
284 pd.rt.seq = pd.rseq
285 resettimer(&pd.rt, pd.rd)
286 }
287 } else if pd.rd != rd0 || combo != combo0 {
288 pd.rseq++
289 if pd.rd > 0 {
290 modtimer(&pd.rt, pd.rd, 0, rtf, pd.makeArg(), pd.rseq)
291 } else {
292 deltimer(&pd.rt)
293 pd.rt.f = nil
294 }
295 }
296 if pd.wt.f == nil {
297 if pd.wd > 0 && !combo {
298 pd.wt.f = netpollWriteDeadline
299 pd.wt.arg = pd.makeArg()
300 pd.wt.seq = pd.wseq
301 resettimer(&pd.wt, pd.wd)
302 }
303 } else if pd.wd != wd0 || combo != combo0 {
304 pd.wseq++
305 if pd.wd > 0 && !combo {
306 modtimer(&pd.wt, pd.wd, 0, netpollWriteDeadline, pd.makeArg(), pd.wseq)
307 } else {
308 deltimer(&pd.wt)
309 pd.wt.f = nil
310 }
311 }
312
313 var rg, wg *g
314 if pd.rd < 0 || pd.wd < 0 {
315 atomic.StorepNoWB(noescape(unsafe.Pointer(&wg)), nil)
316 if pd.rd < 0 {
317 rg = netpollunblock(pd, 'r', false)
318 }
319 if pd.wd < 0 {
320 wg = netpollunblock(pd, 'w', false)
321 }
322 }
323 unlock(&pd.lock)
324 if rg != nil {
325 netpollgoready(rg, 3)
326 }
327 if wg != nil {
328 netpollgoready(wg, 3)
329 }
330 }
331
332
333 func poll_runtime_pollUnblock(pd *pollDesc) {
334 lock(&pd.lock)
335 if pd.closing {
336 throw("runtime: unblock on closing polldesc")
337 }
338 pd.closing = true
339 pd.rseq++
340 pd.wseq++
341 var rg, wg *g
342 atomic.StorepNoWB(noescape(unsafe.Pointer(&rg)), nil)
343 rg = netpollunblock(pd, 'r', false)
344 wg = netpollunblock(pd, 'w', false)
345 if pd.rt.f != nil {
346 deltimer(&pd.rt)
347 pd.rt.f = nil
348 }
349 if pd.wt.f != nil {
350 deltimer(&pd.wt)
351 pd.wt.f = nil
352 }
353 unlock(&pd.lock)
354 if rg != nil {
355 netpollgoready(rg, 3)
356 }
357 if wg != nil {
358 netpollgoready(wg, 3)
359 }
360 }
361
362
363
364
365
366
367
368
369
370 func netpollready(toRun *gList, pd *pollDesc, mode int32) {
371 var rg, wg *g
372 if mode == 'r' || mode == 'r'+'w' {
373 rg = netpollunblock(pd, 'r', true)
374 }
375 if mode == 'w' || mode == 'r'+'w' {
376 wg = netpollunblock(pd, 'w', true)
377 }
378 if rg != nil {
379 toRun.push(rg)
380 }
381 if wg != nil {
382 toRun.push(wg)
383 }
384 }
385
386 func netpollcheckerr(pd *pollDesc, mode int32) int {
387 if pd.closing {
388 return pollErrClosing
389 }
390 if (mode == 'r' && pd.rd < 0) || (mode == 'w' && pd.wd < 0) {
391 return pollErrTimeout
392 }
393
394
395
396 if mode == 'r' && pd.everr {
397 return pollErrNotPollable
398 }
399 return pollNoError
400 }
401
402 func netpollblockcommit(gp *g, gpp unsafe.Pointer) bool {
403 r := atomic.Casuintptr((*uintptr)(gpp), pdWait, uintptr(unsafe.Pointer(gp)))
404 if r {
405
406
407
408 atomic.Xadd(&netpollWaiters, 1)
409 }
410 return r
411 }
412
413 func netpollgoready(gp *g, traceskip int) {
414 atomic.Xadd(&netpollWaiters, -1)
415 goready(gp, traceskip+1)
416 }
417
418
419
420 func netpollblock(pd *pollDesc, mode int32, waitio bool) bool {
421 gpp := &pd.rg
422 if mode == 'w' {
423 gpp = &pd.wg
424 }
425
426
427 for {
428 old := *gpp
429 if old == pdReady {
430 *gpp = 0
431 return true
432 }
433 if old != 0 {
434 throw("runtime: double wait")
435 }
436 if atomic.Casuintptr(gpp, 0, pdWait) {
437 break
438 }
439 }
440
441
442
443
444 if waitio || netpollcheckerr(pd, mode) == 0 {
445 gopark(netpollblockcommit, unsafe.Pointer(gpp), waitReasonIOWait, traceEvGoBlockNet, 5)
446 }
447
448 old := atomic.Xchguintptr(gpp, 0)
449 if old > pdWait {
450 throw("runtime: corrupted polldesc")
451 }
452 return old == pdReady
453 }
454
455 func netpollunblock(pd *pollDesc, mode int32, ioready bool) *g {
456 gpp := &pd.rg
457 if mode == 'w' {
458 gpp = &pd.wg
459 }
460
461 for {
462 old := *gpp
463 if old == pdReady {
464 return nil
465 }
466 if old == 0 && !ioready {
467
468
469 return nil
470 }
471 var new uintptr
472 if ioready {
473 new = pdReady
474 }
475 if atomic.Casuintptr(gpp, old, new) {
476 if old == pdWait {
477 old = 0
478 }
479 return (*g)(unsafe.Pointer(old))
480 }
481 }
482 }
483
484 func netpolldeadlineimpl(pd *pollDesc, seq uintptr, read, write bool) {
485 lock(&pd.lock)
486
487
488 currentSeq := pd.rseq
489 if !read {
490 currentSeq = pd.wseq
491 }
492 if seq != currentSeq {
493
494 unlock(&pd.lock)
495 return
496 }
497 var rg *g
498 if read {
499 if pd.rd <= 0 || pd.rt.f == nil {
500 throw("runtime: inconsistent read deadline")
501 }
502 pd.rd = -1
503 atomic.StorepNoWB(unsafe.Pointer(&pd.rt.f), nil)
504 rg = netpollunblock(pd, 'r', false)
505 }
506 var wg *g
507 if write {
508 if pd.wd <= 0 || pd.wt.f == nil && !read {
509 throw("runtime: inconsistent write deadline")
510 }
511 pd.wd = -1
512 atomic.StorepNoWB(unsafe.Pointer(&pd.wt.f), nil)
513 wg = netpollunblock(pd, 'w', false)
514 }
515 unlock(&pd.lock)
516 if rg != nil {
517 netpollgoready(rg, 0)
518 }
519 if wg != nil {
520 netpollgoready(wg, 0)
521 }
522 }
523
524 func netpollDeadline(arg interface{}, seq uintptr) {
525 netpolldeadlineimpl(arg.(*pollDesc), seq, true, true)
526 }
527
528 func netpollReadDeadline(arg interface{}, seq uintptr) {
529 netpolldeadlineimpl(arg.(*pollDesc), seq, true, false)
530 }
531
532 func netpollWriteDeadline(arg interface{}, seq uintptr) {
533 netpolldeadlineimpl(arg.(*pollDesc), seq, false, true)
534 }
535
536 func (c *pollCache) alloc() *pollDesc {
537 lock(&c.lock)
538 if c.first == nil {
539 const pdSize = unsafe.Sizeof(pollDesc{})
540 n := pollBlockSize / pdSize
541 if n == 0 {
542 n = 1
543 }
544
545
546 mem := persistentalloc(n*pdSize, 0, &memstats.other_sys)
547 for i := uintptr(0); i < n; i++ {
548 pd := (*pollDesc)(add(mem, i*pdSize))
549 pd.link = c.first
550 c.first = pd
551 }
552 }
553 pd := c.first
554 c.first = pd.link
555 lockInit(&pd.lock, lockRankPollDesc)
556 unlock(&c.lock)
557 return pd
558 }
559
560
561
562
563
564
565 func (pd *pollDesc) makeArg() (i interface{}) {
566 x := (*eface)(unsafe.Pointer(&i))
567 x._type = pdType
568 x.data = unsafe.Pointer(&pd.self)
569 return
570 }
571
572 var (
573 pdEface interface{} = (*pollDesc)(nil)
574 pdType *_type = efaceOf(&pdEface)._type
575 )
576
View as plain text