debug/pe: add support for bigobj COFF format

This adds support for parsing bigobj COFF files, which use a different
header format with a 32-bit section count, and symbols with a 32 bit
section number.

This fixes linking bigobj COFF with cgo.

Fixes #24341

Author: Peter0x44

src/debug/pe/file.go

@@ -29,7 +29,11 @@ import (
 
 // A File represents an open PE file.
 type File struct {
-	FileHeader
+	// FileHeader is populated for regular COFF files
+	FileHeader *FileHeader
+	// BigObjHeader is populated for bigobj COFF files
+	BigObjHeader *BigObjHeader
+
 	OptionalHeader any // of type *OptionalHeader32 or *OptionalHeader64
 	Sections       []*Section
 	Symbols        []*Symbol    // COFF symbols with auxiliary symbol records removed
@@ -39,6 +43,69 @@ type File struct {
 	closer io.Closer
 }
 
+// IsBigObj reports whether the file is a bigobj COFF file.
+func (f *File) IsBigObj() bool {
+	return f.BigObjHeader != nil
+}
+
+// GetMachine returns the machine type from the appropriate header.
+func (f *File) GetMachine() uint16 {
+	if f.BigObjHeader != nil {
+		return f.BigObjHeader.Machine
+	}
+	return f.FileHeader.Machine
+}
+
+// GetNumberOfSections returns the number of sections from the appropriate header.
+func (f *File) GetNumberOfSections() uint32 {
+	if f.BigObjHeader != nil {
+		return f.BigObjHeader.NumberOfSections
+	}
+	return uint32(f.FileHeader.NumberOfSections)
+}
+
+// GetTimeDateStamp returns the timestamp from the appropriate header.
+func (f *File) GetTimeDateStamp() uint32 {
+	if f.BigObjHeader != nil {
+		return f.BigObjHeader.TimeDateStamp
+	}
+	return f.FileHeader.TimeDateStamp
+}
+
+// GetPointerToSymbolTable returns the symbol table pointer from the appropriate header.
+func (f *File) GetPointerToSymbolTable() uint32 {
+	if f.BigObjHeader != nil {
+		return f.BigObjHeader.PointerToSymbolTable
+	}
+	return f.FileHeader.PointerToSymbolTable
+}
+
+// GetNumberOfSymbols returns the number of symbols from the appropriate header.
+func (f *File) GetNumberOfSymbols() uint32 {
+	if f.BigObjHeader != nil {
+		return f.BigObjHeader.NumberOfSymbols
+	}
+	return f.FileHeader.NumberOfSymbols
+}
+
+// GetSizeOfOptionalHeader returns the optional header size from the appropriate header.
+// BigObj files don't have optional headers, so this returns 0 for them.
+func (f *File) GetSizeOfOptionalHeader() uint16 {
+	if f.BigObjHeader != nil {
+		return 0
+	}
+	return f.FileHeader.SizeOfOptionalHeader
+}
+
+// GetCharacteristics returns the characteristics from the appropriate header.
+// BigObj files don't have characteristics, so this returns 0 for them.
+func (f *File) GetCharacteristics() uint16 {
+	if f.BigObjHeader != nil {
+		return 0
+	}
+	return f.FileHeader.Characteristics
+}
+
 // Open opens the named file using [os.Open] and prepares it for use as a PE binary.
 func Open(name string) (*File, error) {
 	f, err := os.Open(name)
@@ -68,6 +135,69 @@ func (f *File) Close() error {
 
 // TODO(brainman): add Load function, as a replacement for NewFile, that does not call removeAuxSymbols (for performance)
 
+// isBigObjFormat detects if the reader contains a bigobj COFF file by checking
+// the signature and GUID. The reader position should be at the start of the COFF header.
+func isBigObjFormat(r io.ReadSeeker) (bool, error) {
+	currentPos, err := r.Seek(0, io.SeekCurrent)
+	if err != nil {
+		return false, err
+	}
+	defer r.Seek(currentPos, io.SeekStart)
+
+	// Read the first part of what could be a BigObjHeader
+	var sig struct {
+		Sig1          uint16
+		Sig2          uint16
+		Version       uint16
+		Machine       uint16
+		TimeDateStamp uint32
+		ClassID       [16]uint8
+	}
+
+	err = binary.Read(r, binary.LittleEndian, &sig)
+	if err != nil {
+		return false, err
+	}
+
+	if sig.Sig1 != BigObjSig1 || sig.Sig2 != BigObjSig2 {
+		return false, nil
+	}
+
+	if sig.ClassID != BigObjClassID {
+		return false, nil
+	}
+
+	return true, nil
+}
+
+// readCOFFHeader reads the appropriate COFF header type ("regular" or bigobj).
+// The unused header type will be nil
+func readCOFFHeader(sr *io.SectionReader, base int64) (*FileHeader, *BigObjHeader, error) {
+	_, err := sr.Seek(base, io.SeekStart)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	isBigObj, err := isBigObjFormat(sr)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	if isBigObj {
+		bigObjHeader := new(BigObjHeader)
+		if err := binary.Read(sr, binary.LittleEndian, bigObjHeader); err != nil {
+			return nil, nil, err
+		}
+		return nil, bigObjHeader, nil
+	} else {
+		fileHeader := new(FileHeader)
+		if err := binary.Read(sr, binary.LittleEndian, fileHeader); err != nil {
+			return nil, nil, err
+		}
+		return fileHeader, nil, nil
+	}
+}
+
 // NewFile creates a new [File] for accessing a PE binary in an underlying reader.
 func NewFile(r io.ReaderAt) (*File, error) {
 	f := new(File)
@@ -89,11 +219,24 @@ func NewFile(r io.ReaderAt) (*File, error) {
 	} else {
 		base = int64(0)
 	}
-	sr.Seek(base, io.SeekStart)
-	if err := binary.Read(sr, binary.LittleEndian, &f.FileHeader); err != nil {
+	// Read appropriate header type - unused header will be nil
+	fileHeader, bigObjHeader, err := readCOFFHeader(sr, base)
+	if err != nil {
 		return nil, err
 	}
-	switch f.FileHeader.Machine {
+	f.FileHeader = fileHeader
+	f.BigObjHeader = bigObjHeader
+
+	// Calculate header size based on actual type
+	var headerSize int
+	if f.BigObjHeader != nil {
+		headerSize = binary.Size(*f.BigObjHeader)
+	} else {
+		headerSize = binary.Size(*f.FileHeader)
+	}
+
+	// Validate machine type
+	switch f.GetMachine() {
 	case IMAGE_FILE_MACHINE_AMD64,
 		IMAGE_FILE_MACHINE_ARM64,
 		IMAGE_FILE_MACHINE_ARMNT,
@@ -104,19 +247,17 @@ func NewFile(r io.ReaderAt) (*File, error) {
 		IMAGE_FILE_MACHINE_UNKNOWN:
 		// ok
 	default:
-		return nil, fmt.Errorf("unrecognized PE machine: %#x", f.FileHeader.Machine)
+		return nil, fmt.Errorf("unrecognized PE machine: %#x", f.GetMachine())
 	}
 
-	var err error
-
 	// Read string table.
-	f.StringTable, err = readStringTable(&f.FileHeader, sr)
+	f.StringTable, err = readStringTableFromFile(f, sr)
 	if err != nil {
 		return nil, err
 	}
 
 	// Read symbol table.
-	f.COFFSymbols, err = readCOFFSymbols(&f.FileHeader, sr)
+	f.COFFSymbols, err = readCOFFSymbols(f, sr)
 	if err != nil {
 		return nil, err
 	}
@@ -126,20 +267,23 @@ func NewFile(r io.ReaderAt) (*File, error) {
 	}
 
 	// Seek past file header.
-	_, err = sr.Seek(base+int64(binary.Size(f.FileHeader)), io.SeekStart)
+	_, err = sr.Seek(base+int64(headerSize), io.SeekStart)
 	if err != nil {
 		return nil, err
 	}
 
-	// Read optional header.
-	f.OptionalHeader, err = readOptionalHeader(sr, f.FileHeader.SizeOfOptionalHeader)
-	if err != nil {
-		return nil, err
+	// Read optional header (only for regular COFF files).
+	if !f.IsBigObj() {
+		f.OptionalHeader, err = readOptionalHeader(sr, f.GetSizeOfOptionalHeader())
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	// Process sections.
-	f.Sections = make([]*Section, f.FileHeader.NumberOfSections)
-	for i := 0; i < int(f.FileHeader.NumberOfSections); i++ {
+	numSections := f.GetNumberOfSections()
+	f.Sections = make([]*Section, numSections)
+	for i := uint32(0); i < numSections; i++ {
 		sh := new(SectionHeader32)
 		if err := binary.Read(sr, binary.LittleEndian, sh); err != nil {
 			return nil, err

src/debug/pe/pe.go

@@ -14,6 +14,37 @@ type FileHeader struct {
 	Characteristics      uint16
 }
 
+// BigObjHeader represents the ANON_OBJECT_HEADER_BIGOBJ structure
+// used in bigobj COFF format. This format allows for more than 65535 sections.
+type BigObjHeader struct {
+	Sig1                 uint16 // Must be 0x0
+	Sig2                 uint16 // Must be 0xFFFF
+	Version              uint16 // Currently 2
+	Machine              uint16
+	TimeDateStamp        uint32
+	ClassID              [16]uint8 // GUID that identifies this as bigobj format
+	SizeOfData           uint32
+	Flags                uint32
+	MetaDataSize         uint32
+	MetaDataOffset       uint32
+	NumberOfSections     uint32 // 32-bit field (vs 16-bit in regular COFF)
+	PointerToSymbolTable uint32
+	NumberOfSymbols      uint32
+}
+
+// BigObj signature constants
+const (
+	BigObjSig1    = 0x0
+	BigObjSig2    = 0xFFFF
+	BigObjVersion = 2
+)
+
+// The GUID that identifies a file as bigobj format
+var BigObjClassID = [16]uint8{
+	0xC7, 0xA1, 0xBA, 0xD1, 0xEE, 0xBA, 0xA9, 0x4B,
+	0xAF, 0x20, 0xFA, 0xF6, 0x6A, 0xA4, 0xDC, 0xB8,
+}
+
 type DataDirectory struct {
 	VirtualAddress uint32
 	Size           uint32

src/debug/pe/string.go

@@ -25,12 +25,19 @@ func cstring(b []byte) string {
 // StringTable is a COFF string table.
 type StringTable []byte
 
-func readStringTable(fh *FileHeader, r io.ReadSeeker) (StringTable, error) {
+// readStringTableFromFile reads string table using the File struct
+func readStringTableFromFile(f *File, r io.ReadSeeker) (StringTable, error) {
 	// COFF string table is located right after COFF symbol table.
-	if fh.PointerToSymbolTable <= 0 {
+	if f.GetPointerToSymbolTable() <= 0 {
 		return nil, nil
 	}
-	offset := fh.PointerToSymbolTable + COFFSymbolSize*fh.NumberOfSymbols
+
+	symbolSize := COFFSymbolSize
+	if f.IsBigObj() {
+		symbolSize = BigObjSymbolSize
+	}
+
+	offset := f.GetPointerToSymbolTable() + symbolSize*f.GetNumberOfSymbols()
 	_, err := r.Seek(int64(offset), io.SeekStart)
 	if err != nil {
 		return nil, fmt.Errorf("fail to seek to string table: %v", err)