1 files changed, 188 insertions, 83 deletions

diff --git a/vendor/github.com/cilium/ebpf/linker.go b/vendor/github.com/cilium/ebpf/linker.go
index f843bb25e..e6276b182 100644
--- a/vendor/github.com/cilium/ebpf/linker.go
+++ b/vendor/github.com/cilium/ebpf/linker.go
@@ -1,133 +1,238 @@
 package ebpf
 
 import (
+	"errors"
 	"fmt"
+	"sync"
 
 	"github.com/cilium/ebpf/asm"
-	"github.com/cilium/ebpf/internal/btf"
+	"github.com/cilium/ebpf/btf"
 )
 
-// link resolves bpf-to-bpf calls.
+// splitSymbols splits insns into subsections delimited by Symbol Instructions.
+// insns cannot be empty and must start with a Symbol Instruction.
 //
-// Each library may contain multiple functions / labels, and is only linked
-// if prog references one of these functions.
-//
-// Libraries also linked.
-func link(prog *ProgramSpec, libs []*ProgramSpec) error {
-	var (
-		linked  = make(map[*ProgramSpec]bool)
-		pending = []asm.Instructions{prog.Instructions}
-		insns   asm.Instructions
-	)
-	for len(pending) > 0 {
-		insns, pending = pending[0], pending[1:]
-		for _, lib := range libs {
-			if linked[lib] {
-				continue
-			}
+// The resulting map is indexed by Symbol name.
+func splitSymbols(insns asm.Instructions) (map[string]asm.Instructions, error) {
+	if len(insns) == 0 {
+		return nil, errors.New("insns is empty")
+	}
 
-			needed, err := needSection(insns, lib.Instructions)
-			if err != nil {
-				return fmt.Errorf("linking %s: %w", lib.Name, err)
-			}
+	if insns[0].Symbol() == "" {
+		return nil, errors.New("insns must start with a Symbol")
+	}
 
-			if !needed {
-				continue
+	var name string
+	progs := make(map[string]asm.Instructions)
+	for _, ins := range insns {
+		if sym := ins.Symbol(); sym != "" {
+			if progs[sym] != nil {
+				return nil, fmt.Errorf("insns contains duplicate Symbol %s", sym)
 			}
+			name = sym
+		}
+
+		progs[name] = append(progs[name], ins)
+	}
 
-			linked[lib] = true
-			prog.Instructions = append(prog.Instructions, lib.Instructions...)
-			pending = append(pending, lib.Instructions)
+	return progs, nil
+}
 
-			if prog.BTF != nil && lib.BTF != nil {
-				if err := btf.ProgramAppend(prog.BTF, lib.BTF); err != nil {
-					return fmt.Errorf("linking BTF of %s: %w", lib.Name, err)
-				}
-			}
+// The linker is responsible for resolving bpf-to-bpf calls between programs
+// within an ELF. Each BPF program must be a self-contained binary blob,
+// so when an instruction in one ELF program section wants to jump to
+// a function in another, the linker needs to pull in the bytecode
+// (and BTF info) of the target function and concatenate the instruction
+// streams.
+//
+// Later on in the pipeline, all call sites are fixed up with relative jumps
+// within this newly-created instruction stream to then finally hand off to
+// the kernel with BPF_PROG_LOAD.
+//
+// Each function is denoted by an ELF symbol and the compiler takes care of
+// register setup before each jump instruction.
+
+// hasFunctionReferences returns true if insns contains one or more bpf2bpf
+// function references.
+func hasFunctionReferences(insns asm.Instructions) bool {
+	for _, i := range insns {
+		if i.IsFunctionReference() {
+			return true
 		}
 	}
-
-	return nil
+	return false
 }
 
-func needSection(insns, section asm.Instructions) (bool, error) {
-	// A map of symbols to the libraries which contain them.
-	symbols, err := section.SymbolOffsets()
+// applyRelocations collects and applies any CO-RE relocations in insns.
+//
+// Passing a nil target will relocate against the running kernel. insns are
+// modified in place.
+func applyRelocations(insns asm.Instructions, local, target *btf.Spec) error {
+	var relos []*btf.CORERelocation
+	var reloInsns []*asm.Instruction
+	iter := insns.Iterate()
+	for iter.Next() {
+		if relo := btf.CORERelocationMetadata(iter.Ins); relo != nil {
+			relos = append(relos, relo)
+			reloInsns = append(reloInsns, iter.Ins)
+		}
+	}
+
+	if len(relos) == 0 {
+		return nil
+	}
+
+	target, err := maybeLoadKernelBTF(target)
 	if err != nil {
-		return false, err
+		return err
 	}
 
-	for _, ins := range insns {
-		if ins.Reference == "" {
-			continue
-		}
+	fixups, err := btf.CORERelocate(local, target, relos)
+	if err != nil {
+		return err
+	}
 
-		if ins.OpCode.JumpOp() != asm.Call || ins.Src != asm.PseudoCall {
-			continue
+	for i, fixup := range fixups {
+		if err := fixup.Apply(reloInsns[i]); err != nil {
+			return fmt.Errorf("apply fixup %s: %w", &fixup, err)
 		}
+	}
+
+	return nil
+}
+
+// flattenPrograms resolves bpf-to-bpf calls for a set of programs.
+//
+// Links all programs in names by modifying their ProgramSpec in progs.
+func flattenPrograms(progs map[string]*ProgramSpec, names []string) {
+	// Pre-calculate all function references.
+	refs := make(map[*ProgramSpec][]string)
+	for _, prog := range progs {
+		refs[prog] = prog.Instructions.FunctionReferences()
+	}
+
+	// Create a flattened instruction stream, but don't modify progs yet to
+	// avoid linking multiple times.
+	flattened := make([]asm.Instructions, 0, len(names))
+	for _, name := range names {
+		flattened = append(flattened, flattenInstructions(name, progs, refs))
+	}
+
+	// Finally, assign the flattened instructions.
+	for i, name := range names {
+		progs[name].Instructions = flattened[i]
+	}
+}
+
+// flattenInstructions resolves bpf-to-bpf calls for a single program.
+//
+// Flattens the instructions of prog by concatenating the instructions of all
+// direct and indirect dependencies.
+//
+// progs contains all referenceable programs, while refs contain the direct
+// dependencies of each program.
+func flattenInstructions(name string, progs map[string]*ProgramSpec, refs map[*ProgramSpec][]string) asm.Instructions {
+	prog := progs[name]
+
+	insns := make(asm.Instructions, len(prog.Instructions))
+	copy(insns, prog.Instructions)
+
+	// Add all direct references of prog to the list of to be linked programs.
+	pending := make([]string, len(refs[prog]))
+	copy(pending, refs[prog])
 
-		if ins.Constant != -1 {
-			// This is already a valid call, no need to link again.
+	// All references for which we've appended instructions.
+	linked := make(map[string]bool)
+
+	// Iterate all pending references. We can't use a range since pending is
+	// modified in the body below.
+	for len(pending) > 0 {
+		var ref string
+		ref, pending = pending[0], pending[1:]
+
+		if linked[ref] {
+			// We've already linked this ref, don't append instructions again.
 			continue
 		}
 
-		if _, ok := symbols[ins.Reference]; !ok {
-			// Symbol isn't available in this section
+		progRef := progs[ref]
+		if progRef == nil {
+			// We don't have instructions that go with this reference. This
+			// happens when calling extern functions.
 			continue
 		}
 
-		// At this point we know that at least one function in the
-		// library is called from insns, so we have to link it.
-		return true, nil
+		insns = append(insns, progRef.Instructions...)
+		linked[ref] = true
+
+		// Make sure we link indirect references.
+		pending = append(pending, refs[progRef]...)
 	}
 
-	// None of the functions in the section are called.
-	return false, nil
+	return insns
 }
 
-func fixupJumpsAndCalls(insns asm.Instructions) error {
-	symbolOffsets := make(map[string]asm.RawInstructionOffset)
+// fixupAndValidate is called by the ELF reader right before marshaling the
+// instruction stream. It performs last-minute adjustments to the program and
+// runs some sanity checks before sending it off to the kernel.
+func fixupAndValidate(insns asm.Instructions) error {
 	iter := insns.Iterate()
 	for iter.Next() {
 		ins := iter.Ins
 
-		if ins.Symbol == "" {
-			continue
+		// Map load was tagged with a Reference, but does not contain a Map pointer.
+		if ins.IsLoadFromMap() && ins.Reference() != "" && ins.Map() == nil {
+			return fmt.Errorf("instruction %d: map %s: %w", iter.Index, ins.Reference(), asm.ErrUnsatisfiedMapReference)
 		}
 
-		if _, ok := symbolOffsets[ins.Symbol]; ok {
-			return fmt.Errorf("duplicate symbol %s", ins.Symbol)
-		}
+		fixupProbeReadKernel(ins)
+	}
 
-		symbolOffsets[ins.Symbol] = iter.Offset
+	return nil
+}
+
+// fixupProbeReadKernel replaces calls to bpf_probe_read_{kernel,user}(_str)
+// with bpf_probe_read(_str) on kernels that don't support it yet.
+func fixupProbeReadKernel(ins *asm.Instruction) {
+	if !ins.IsBuiltinCall() {
+		return
 	}
 
-	iter = insns.Iterate()
-	for iter.Next() {
-		i := iter.Index
-		offset := iter.Offset
-		ins := iter.Ins
+	// Kernel supports bpf_probe_read_kernel, nothing to do.
+	if haveProbeReadKernel() == nil {
+		return
+	}
 
-		switch {
-		case ins.IsFunctionCall() && ins.Constant == -1:
-			// Rewrite bpf to bpf call
-			callOffset, ok := symbolOffsets[ins.Reference]
-			if !ok {
-				return fmt.Errorf("instruction %d: reference to missing symbol %q", i, ins.Reference)
-			}
+	switch asm.BuiltinFunc(ins.Constant) {
+	case asm.FnProbeReadKernel, asm.FnProbeReadUser:
+		ins.Constant = int64(asm.FnProbeRead)
+	case asm.FnProbeReadKernelStr, asm.FnProbeReadUserStr:
+		ins.Constant = int64(asm.FnProbeReadStr)
+	}
+}
 
-			ins.Constant = int64(callOffset - offset - 1)
+var kernelBTF struct {
+	sync.Mutex
+	spec *btf.Spec
+}
 
-		case ins.OpCode.Class() == asm.JumpClass && ins.Offset == -1:
-			// Rewrite jump to label
-			jumpOffset, ok := symbolOffsets[ins.Reference]
-			if !ok {
-				return fmt.Errorf("instruction %d: reference to missing symbol %q", i, ins.Reference)
-			}
+// maybeLoadKernelBTF loads the current kernel's BTF if spec is nil, otherwise
+// it returns spec unchanged.
+//
+// The kernel BTF is cached for the lifetime of the process.
+func maybeLoadKernelBTF(spec *btf.Spec) (*btf.Spec, error) {
+	if spec != nil {
+		return spec, nil
+	}
 
-			ins.Offset = int16(jumpOffset - offset - 1)
-		}
+	kernelBTF.Lock()
+	defer kernelBTF.Unlock()
+
+	if kernelBTF.spec != nil {
+		return kernelBTF.spec, nil
 	}
 
-	return nil
+	var err error
+	kernelBTF.spec, err = btf.LoadKernelSpec()
+	return kernelBTF.spec, err
 }