3030// indicates scanning can ignore the rest of the allocation.
3131//
3232// The 2-bit entries are split when written into the byte, so that the top half
33- // of the byte contains 4 high bits and the bottom half contains 4 low (pointer)
34- // bits.
35- // This form allows a copy from the 1-bit to the 4-bit form to keep the
36- // pointer bits contiguous, instead of having to space them out.
33+ // of the byte contains 4 high (scan) bits and the bottom half contains 4 low
34+ // (pointer) bits. This form allows a copy from the 1-bit to the 4-bit form to
35+ // keep the pointer bits contiguous, instead of having to space them out.
3736//
3837// The code makes use of the fact that the zero value for a heap
3938// bitmap means scalar/dead. This property must be preserved when
@@ -816,6 +815,12 @@ func (s *mspan) countAlloc() int {
816815func heapBitsSetType (x , size , dataSize uintptr , typ * _type ) {
817816 const doubleCheck = false // slow but helpful; enable to test modifications to this code
818817
818+ const (
819+ mask1 = bitPointer | bitScan // 00010001
820+ mask2 = bitPointer | bitScan | mask1 << heapBitsShift // 00110011
821+ mask3 = bitPointer | bitScan | mask2 << heapBitsShift // 01110111
822+ )
823+
819824 // dataSize is always size rounded up to the next malloc size class,
820825 // except in the case of allocating a defer block, in which case
821826 // size is sizeof(_defer{}) (at least 6 words) and dataSize may be
@@ -844,11 +849,12 @@ func heapBitsSetType(x, size, dataSize uintptr, typ *_type) {
844849 h := heapBitsForAddr (x )
845850 ptrmask := typ .gcdata // start of 1-bit pointer mask (or GC program, handled below)
846851
847- // Heap bitmap bits for 2-word object are only 4 bits,
848- // so also shared with objects next to it.
849- // This is called out as a special case primarily for 32-bit systems,
850- // so that on 32-bit systems the code below can assume all objects
851- // are 4-word aligned (because they're all 16-byte aligned).
852+ // 2-word objects only have 4 bitmap bits and 3-word objects only have 6 bitmap bits.
853+ // Therefore, these objects share a heap bitmap byte with the objects next to them.
854+ // These are called out as a special case primarily so the code below can assume all
855+ // objects are at least 4 words long and that their bitmaps start either at the beginning
856+ // of a bitmap byte, or half-way in (h.shift of 0 and 2 respectively).
857+
852858 if size == 2 * sys .PtrSize {
853859 if typ .size == sys .PtrSize {
854860 // We're allocating a block big enough to hold two pointers.
@@ -865,7 +871,7 @@ func heapBitsSetType(x, size, dataSize uintptr, typ *_type) {
865871 * h .bitp &^= (bitPointer | bitScan | (bitPointer | bitScan )<< heapBitsShift ) << h .shift
866872 * h .bitp |= (bitPointer | bitScan ) << h .shift
867873 } else {
868- // 2-element slice of pointer.
874+ // 2-element array of pointer.
869875 * h .bitp |= (bitPointer | bitScan | (bitPointer | bitScan )<< heapBitsShift ) << h .shift
870876 }
871877 return
@@ -886,6 +892,70 @@ func heapBitsSetType(x, size, dataSize uintptr, typ *_type) {
886892 * h .bitp &^= (bitPointer | bitScan | ((bitPointer | bitScan ) << heapBitsShift )) << h .shift
887893 * h .bitp |= uint8 (hb << h .shift )
888894 return
895+ } else if size == 3 * sys .PtrSize {
896+ b := uint8 (* ptrmask )
897+ if doubleCheck {
898+ if b == 0 {
899+ println ("runtime: invalid type " , typ .string ())
900+ throw ("heapBitsSetType: called with non-pointer type" )
901+ }
902+ if sys .PtrSize != 8 {
903+ throw ("heapBitsSetType: unexpected 3 pointer wide size class on 32 bit" )
904+ }
905+ if typ .kind & kindGCProg != 0 {
906+ throw ("heapBitsSetType: unexpected GC prog for 3 pointer wide size class" )
907+ }
908+ if typ .size == 2 * sys .PtrSize {
909+ print ("runtime: heapBitsSetType size=" , size , " but typ.size=" , typ .size , "\n " )
910+ throw ("heapBitsSetType: inconsistent object sizes" )
911+ }
912+ }
913+ if typ .size == sys .PtrSize {
914+ // The type contains a pointer otherwise heapBitsSetType wouldn't have been called.
915+ // Since the type is only 1 pointer wide and contains a pointer, its gcdata must be exactly 1.
916+ if doubleCheck && * typ .gcdata != 1 {
917+ print ("runtime: heapBitsSetType size=" , size , " typ.size=" , typ .size , "but *typ.gcdata" , * typ .gcdata , "\n " )
918+ throw ("heapBitsSetType: unexpected gcdata for 1 pointer wide type size in 3 pointer wide size class" )
919+ }
920+ // 3 element array of pointers. Unrolling ptrmask 3 times into p yields 00000111.
921+ b = 7
922+ }
923+
924+ hb := b & 7
925+ // Set bitScan bits for all pointers.
926+ hb |= hb << wordsPerBitmapByte
927+ // First bitScan bit is always set since the type contains pointers.
928+ hb |= bitScan
929+ // Second bitScan bit needs to also be set if the third bitScan bit is set.
930+ hb |= hb & (bitScan << (2 * heapBitsShift )) >> 1
931+
932+ // For h.shift > 1 heap bits cross a byte boundary and need to be written part
933+ // to h.bitp and part to the next h.bitp.
934+ switch h .shift {
935+ case 0 :
936+ * h .bitp &^= mask3 << 0
937+ * h .bitp |= hb << 0
938+ case 1 :
939+ * h .bitp &^= mask3 << 1
940+ * h .bitp |= hb << 1
941+ case 2 :
942+ * h .bitp &^= mask2 << 2
943+ * h .bitp |= (hb & mask2 ) << 2
944+ // Two words written to the first byte.
945+ // Advance two words to get to the next byte.
946+ h = h .next ().next ()
947+ * h .bitp &^= mask1
948+ * h .bitp |= (hb >> 2 ) & mask1
949+ case 3 :
950+ * h .bitp &^= mask1 << 3
951+ * h .bitp |= (hb & mask1 ) << 3
952+ // One word written to the first byte.
953+ // Advance one word to get to the next byte.
954+ h = h .next ()
955+ * h .bitp &^= mask2
956+ * h .bitp |= (hb >> 1 ) & mask2
957+ }
958+ return
889959 }
890960
891961 // Copy from 1-bit ptrmask into 2-bit bitmap.
@@ -1079,7 +1149,7 @@ func heapBitsSetType(x, size, dataSize uintptr, typ *_type) {
10791149 // word must be set to scan since there are pointers
10801150 // somewhere in the object.
10811151 // In all following words, we set the scan/dead
1082- // appropriately to indicate that the object contains
1152+ // appropriately to indicate that the object continues
10831153 // to the next 2-bit entry in the bitmap.
10841154 //
10851155 // We set four bits at a time here, but if the object
@@ -1095,12 +1165,22 @@ func heapBitsSetType(x, size, dataSize uintptr, typ *_type) {
10951165 b >>= 4
10961166 nb -= 4
10971167
1098- case sys . PtrSize == 8 && h .shift == 2 :
1168+ case h .shift == 2 :
10991169 // Ptrmask and heap bitmap are misaligned.
1170+ //
1171+ // On 32 bit architectures only the 6-word object that corresponds
1172+ // to a 24 bytes size class can start with h.shift of 2 here since
1173+ // all other non 16 byte aligned size classes have been handled by
1174+ // special code paths at the beginning of heapBitsSetType on 32 bit.
1175+ //
1176+ // Many size classes are only 16 byte aligned. On 64 bit architectures
1177+ // this results in a heap bitmap position starting with a h.shift of 2.
1178+ //
11001179 // The bits for the first two words are in a byte shared
11011180 // with another object, so we must be careful with the bits
11021181 // already there.
1103- // We took care of 1-word and 2-word objects above,
1182+ //
1183+ // We took care of 1-word, 2-word, and 3-word objects above,
11041184 // so this is at least a 6-word object.
11051185 hb = (b & (bitPointer | bitPointer << heapBitsShift )) << (2 * heapBitsShift )
11061186 hb |= bitScan << (2 * heapBitsShift )
@@ -1113,7 +1193,7 @@ func heapBitsSetType(x, size, dataSize uintptr, typ *_type) {
11131193 * hbitp |= uint8 (hb )
11141194 hbitp = add1 (hbitp )
11151195 if w += 2 ; w >= nw {
1116- // We know that there is more data, because we handled 2-word objects above.
1196+ // We know that there is more data, because we handled 2-word and 3-word objects above.
11171197 // This must be at least a 6-word object. If we're out of pointer words,
11181198 // mark no scan in next bitmap byte and finish.
11191199 hb = 0
@@ -1248,12 +1328,12 @@ Phase4:
12481328 // Handle the first byte specially if it's shared. See
12491329 // Phase 1 for why this is the only special case we need.
12501330 if doubleCheck {
1251- if ! (h .shift == 0 || ( sys . PtrSize == 8 && h .shift == 2 ) ) {
1331+ if ! (h .shift == 0 || h .shift == 2 ) {
12521332 print ("x=" , x , " size=" , size , " cnw=" , h .shift , "\n " )
12531333 throw ("bad start shift" )
12541334 }
12551335 }
1256- if sys . PtrSize == 8 && h .shift == 2 {
1336+ if h .shift == 2 {
12571337 * h .bitp = * h .bitp &^((bitPointer | bitScan | (bitPointer | bitScan )<< heapBitsShift )<< (2 * heapBitsShift )) | * src
12581338 h = h .next ().next ()
12591339 cnw -= 2
0 commit comments