1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
|
// Copyright 2024 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build race
package impl
// When running under race detector, we add a presence map of bytes, that we can access
// in the hook functions so that we trigger the race detection whenever we have concurrent
// Read-Writes or Write-Writes. The race detector does not otherwise detect invalid concurrent
// access to lazy fields as all updates of bitmaps and pointers are done using atomic operations.
type RaceDetectHookData struct {
shadowPresence *[]byte
}
// Hooks for presence bitmap operations that allocate, read and write the shadowPresence
// using non-atomic operations.
func (data *RaceDetectHookData) raceDetectHookAlloc(size presenceSize) {
sp := make([]byte, size)
atomicStoreShadowPresence(&data.shadowPresence, &sp)
}
func (p presence) raceDetectHookPresent(num uint32) {
data := p.toRaceDetectData()
if data == nil {
return
}
sp := atomicLoadShadowPresence(&data.shadowPresence)
if sp != nil {
_ = (*sp)[num]
}
}
func (p presence) raceDetectHookSetPresent(num uint32, size presenceSize) {
data := p.toRaceDetectData()
if data == nil {
return
}
sp := atomicLoadShadowPresence(&data.shadowPresence)
if sp == nil {
data.raceDetectHookAlloc(size)
sp = atomicLoadShadowPresence(&data.shadowPresence)
}
(*sp)[num] = 1
}
func (p presence) raceDetectHookClearPresent(num uint32) {
data := p.toRaceDetectData()
if data == nil {
return
}
sp := atomicLoadShadowPresence(&data.shadowPresence)
if sp != nil {
(*sp)[num] = 0
}
}
// raceDetectHookAllocAndCopy allocates a new shadowPresence slice at lazy and copies
// shadowPresence bytes from src to lazy.
func (p presence) raceDetectHookAllocAndCopy(q presence) {
sData := q.toRaceDetectData()
dData := p.toRaceDetectData()
if sData == nil {
return
}
srcSp := atomicLoadShadowPresence(&sData.shadowPresence)
if srcSp == nil {
atomicStoreShadowPresence(&dData.shadowPresence, nil)
return
}
n := len(*srcSp)
dSlice := make([]byte, n)
atomicStoreShadowPresence(&dData.shadowPresence, &dSlice)
for i := 0; i < n; i++ {
dSlice[i] = (*srcSp)[i]
}
}
// raceDetectHookPresent is called by the generated file interface
// (*proto.internalFuncs) Present to optionally read an unprotected
// shadow bitmap when race detection is enabled. In regular code it is
// a noop.
func raceDetectHookPresent(field *uint32, num uint32) {
data := findPointerToRaceDetectData(field, num)
if data == nil {
return
}
sp := atomicLoadShadowPresence(&data.shadowPresence)
if sp != nil {
_ = (*sp)[num]
}
}
// raceDetectHookSetPresent is called by the generated file interface
// (*proto.internalFuncs) SetPresent to optionally write an unprotected
// shadow bitmap when race detection is enabled. In regular code it is
// a noop.
func raceDetectHookSetPresent(field *uint32, num uint32, size presenceSize) {
data := findPointerToRaceDetectData(field, num)
if data == nil {
return
}
sp := atomicLoadShadowPresence(&data.shadowPresence)
if sp == nil {
data.raceDetectHookAlloc(size)
sp = atomicLoadShadowPresence(&data.shadowPresence)
}
(*sp)[num] = 1
}
// raceDetectHookClearPresent is called by the generated file interface
// (*proto.internalFuncs) ClearPresent to optionally write an unprotected
// shadow bitmap when race detection is enabled. In regular code it is
// a noop.
func raceDetectHookClearPresent(field *uint32, num uint32) {
data := findPointerToRaceDetectData(field, num)
if data == nil {
return
}
sp := atomicLoadShadowPresence(&data.shadowPresence)
if sp != nil {
(*sp)[num] = 0
}
}
|
