diff --git a/ebpf-poc/bpf_bpfeb.go b/ebpf-poc/bpf_bpfeb.go index 92442e0..d18708b 100644 --- a/ebpf-poc/bpf_bpfeb.go +++ b/ebpf-poc/bpf_bpfeb.go @@ -14,23 +14,33 @@ import ( ) type bpfEvent struct { - _ structs.HostLayout - Pid uint32 - Uid uint32 - Daddr uint32 - Dport uint16 - Proto uint8 - Comm [16]uint8 - _ [1]byte + _ structs.HostLayout + Pid uint32 + Uid uint32 + Daddr uint32 + Dport uint16 + Proto uint8 + Action uint8 + Comm [16]uint8 +} + +type bpfTarget struct { + _ structs.HostLayout + Ip uint32 + Port uint16 + _ [2]byte } // Names of all BPF objects in the ELF. // // Used for safe lookups in a Collection or CollectionSpec. const ( - bpfMapEvents = "events" - bpfProgConnect4 = "connect4" - bpfVarUnusedEvent = "unused_event" + bpfMapEvents = "events" + bpfMapExemptMap = "exempt_map" + bpfMapTargetMap = "target_map" + bpfProgConnect4 = "connect4" + bpfVarUnusedEvent = "unused_event" + bpfVarUnusedTarget = "unused_target" ) // loadBpf returns the embedded CollectionSpec for bpf. @@ -82,14 +92,17 @@ type bpfProgramSpecs struct { // // It can be passed ebpf.CollectionSpec.Assign. type bpfMapSpecs struct { - Events *ebpf.MapSpec `ebpf:"events"` + Events *ebpf.MapSpec `ebpf:"events"` + ExemptMap *ebpf.MapSpec `ebpf:"exempt_map"` + TargetMap *ebpf.MapSpec `ebpf:"target_map"` } // bpfVariableSpecs contains global variables before they are loaded into the kernel. // // It can be passed ebpf.CollectionSpec.Assign. type bpfVariableSpecs struct { - UnusedEvent *ebpf.VariableSpec `ebpf:"unused_event"` + UnusedEvent *ebpf.VariableSpec `ebpf:"unused_event"` + UnusedTarget *ebpf.VariableSpec `ebpf:"unused_target"` } // bpfObjects contains all objects after they have been loaded into the kernel. @@ -112,12 +125,16 @@ func (o *bpfObjects) Close() error { // // It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign. type bpfMaps struct { - Events *ebpf.Map `ebpf:"events"` + Events *ebpf.Map `ebpf:"events"` + ExemptMap *ebpf.Map `ebpf:"exempt_map"` + TargetMap *ebpf.Map `ebpf:"target_map"` } func (m *bpfMaps) Close() error { return _BpfClose( m.Events, + m.ExemptMap, + m.TargetMap, ) } @@ -125,7 +142,8 @@ func (m *bpfMaps) Close() error { // // It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign. type bpfVariables struct { - UnusedEvent *ebpf.Variable `ebpf:"unused_event"` + UnusedEvent *ebpf.Variable `ebpf:"unused_event"` + UnusedTarget *ebpf.Variable `ebpf:"unused_target"` } // bpfPrograms contains all programs after they have been loaded into the kernel. diff --git a/ebpf-poc/bpf_bpfeb.o b/ebpf-poc/bpf_bpfeb.o index 8cc7507..72c57f3 100644 Binary files a/ebpf-poc/bpf_bpfeb.o and b/ebpf-poc/bpf_bpfeb.o differ diff --git a/ebpf-poc/bpf_bpfel.go b/ebpf-poc/bpf_bpfel.go index 1f8c396..ff1b666 100644 --- a/ebpf-poc/bpf_bpfel.go +++ b/ebpf-poc/bpf_bpfel.go @@ -14,23 +14,33 @@ import ( ) type bpfEvent struct { - _ structs.HostLayout - Pid uint32 - Uid uint32 - Daddr uint32 - Dport uint16 - Proto uint8 - Comm [16]uint8 - _ [1]byte + _ structs.HostLayout + Pid uint32 + Uid uint32 + Daddr uint32 + Dport uint16 + Proto uint8 + Action uint8 + Comm [16]uint8 +} + +type bpfTarget struct { + _ structs.HostLayout + Ip uint32 + Port uint16 + _ [2]byte } // Names of all BPF objects in the ELF. // // Used for safe lookups in a Collection or CollectionSpec. const ( - bpfMapEvents = "events" - bpfProgConnect4 = "connect4" - bpfVarUnusedEvent = "unused_event" + bpfMapEvents = "events" + bpfMapExemptMap = "exempt_map" + bpfMapTargetMap = "target_map" + bpfProgConnect4 = "connect4" + bpfVarUnusedEvent = "unused_event" + bpfVarUnusedTarget = "unused_target" ) // loadBpf returns the embedded CollectionSpec for bpf. @@ -82,14 +92,17 @@ type bpfProgramSpecs struct { // // It can be passed ebpf.CollectionSpec.Assign. type bpfMapSpecs struct { - Events *ebpf.MapSpec `ebpf:"events"` + Events *ebpf.MapSpec `ebpf:"events"` + ExemptMap *ebpf.MapSpec `ebpf:"exempt_map"` + TargetMap *ebpf.MapSpec `ebpf:"target_map"` } // bpfVariableSpecs contains global variables before they are loaded into the kernel. // // It can be passed ebpf.CollectionSpec.Assign. type bpfVariableSpecs struct { - UnusedEvent *ebpf.VariableSpec `ebpf:"unused_event"` + UnusedEvent *ebpf.VariableSpec `ebpf:"unused_event"` + UnusedTarget *ebpf.VariableSpec `ebpf:"unused_target"` } // bpfObjects contains all objects after they have been loaded into the kernel. @@ -112,12 +125,16 @@ func (o *bpfObjects) Close() error { // // It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign. type bpfMaps struct { - Events *ebpf.Map `ebpf:"events"` + Events *ebpf.Map `ebpf:"events"` + ExemptMap *ebpf.Map `ebpf:"exempt_map"` + TargetMap *ebpf.Map `ebpf:"target_map"` } func (m *bpfMaps) Close() error { return _BpfClose( m.Events, + m.ExemptMap, + m.TargetMap, ) } @@ -125,7 +142,8 @@ func (m *bpfMaps) Close() error { // // It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign. type bpfVariables struct { - UnusedEvent *ebpf.Variable `ebpf:"unused_event"` + UnusedEvent *ebpf.Variable `ebpf:"unused_event"` + UnusedTarget *ebpf.Variable `ebpf:"unused_target"` } // bpfPrograms contains all programs after they have been loaded into the kernel. diff --git a/ebpf-poc/bpf_bpfel.o b/ebpf-poc/bpf_bpfel.o index 419d18d..1339f12 100644 Binary files a/ebpf-poc/bpf_bpfel.o and b/ebpf-poc/bpf_bpfel.o differ diff --git a/ebpf-poc/connect.c b/ebpf-poc/connect.c index 1a4f878..b32256f 100644 --- a/ebpf-poc/connect.c +++ b/ebpf-poc/connect.c @@ -5,35 +5,108 @@ char __license[] SEC("license") = "Dual MIT/GPL"; +#define IPPROTO_TCP 6 + +// Only HTTPS is a candidate for redirect. Every other destination port is left +// alone, otherwise ssh, dns, postgres and everything else on the host would be +// steered into a proxy that cannot speak their protocol. +#define REDIRECT_DPORT 443 + +// What the hook decided. Recorded on every event so the ladder can be checked +// against real traffic before any rewrite is switched on. +#define ACTION_REDIRECT 0 +#define ACTION_SKIP_PROTO 1 +#define ACTION_SKIP_LOOPBACK 2 +#define ACTION_SKIP_EXEMPT 3 +#define ACTION_SKIP_DPORT 4 +#define ACTION_SKIP_NOTARGET 5 + struct event { __u32 pid; __u32 uid; __u32 daddr; __u16 dport; __u8 proto; + __u8 action; __u8 comm[16]; // command }; struct event *unused_event __attribute__((unused)); +// target is the redirect destination, written from userspace once the proxy is +// listening. Keeping it in a map means a proxy restart on a different port is +// one map update rather than a reload of the program. +struct target { + __u32 ip; // network byte order, same layout as ctx->user_ip4 + __u16 port; // host byte order, converted when written back to ctx +}; + +struct target *unused_target __attribute__((unused)); + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, struct target); +} target_map SEC(".maps"); + +// Uids whose traffic is never redirected. The proxy's own upstream fetches live +// here. Without this the rewrite sends the proxy back into itself forever. +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(max_entries, 8); + __type(key, __u32); + __type(value, __u8); +} exempt_map SEC(".maps"); + struct { __uint(type, BPF_MAP_TYPE_RINGBUF); __uint(max_entries, 1 << 24); } events SEC(".maps"); +static __always_inline __u8 decide(struct bpf_sock_addr *ctx, __u32 uid, __u16 dport) { + if (ctx->protocol != IPPROTO_TCP) + return ACTION_SKIP_PROTO; + + // Loopback is never registry traffic, and this also covers clients already + // configured to reach the proxy directly. + if ((bpf_ntohl(ctx->user_ip4) >> 24) == 127) + return ACTION_SKIP_LOOPBACK; + + if (bpf_map_lookup_elem(&exempt_map, &uid)) + return ACTION_SKIP_EXEMPT; + + if (dport != REDIRECT_DPORT) + return ACTION_SKIP_DPORT; + + __u32 key = 0; + struct target *t = bpf_map_lookup_elem(&target_map, &key); + if (!t || t->port == 0) + return ACTION_SKIP_NOTARGET; + + return ACTION_REDIRECT; +} + SEC("cgroup/connect4") int connect4(struct bpf_sock_addr *ctx) { + __u32 uid = bpf_get_current_uid_gid(); + __u16 dport = bpf_ntohs(ctx->user_port); + __u8 action = decide(ctx, uid, dport); + + // A full ring buffer costs an event, never a change in behaviour. struct event *e = bpf_ringbuf_reserve(&events, sizeof(*e), 0); - if (!e) - return 1; + if (e) { + e->pid = bpf_get_current_pid_tgid() >> 32; + e->uid = uid; + e->daddr = ctx->user_ip4; + e->dport = dport; + e->proto = ctx->protocol; + e->action = action; + bpf_get_current_comm(&e->comm, sizeof(e->comm)); + bpf_ringbuf_submit(e, 0); + } - e->pid = bpf_get_current_pid_tgid() >> 32; - e->uid = bpf_get_current_uid_gid(); - e->daddr = ctx->user_ip4; - e->dport = bpf_ntohs(ctx->user_port); - e-> proto = ctx->protocol; - bpf_get_current_comm(&e->comm, sizeof(e->comm)); - - bpf_ringbuf_submit(e, 0); + // Shadow mode. The decision is recorded and never acted on. Turning + // ACTION_REDIRECT into a rewrite of user_ip4 and user_port is the next step. return 1; } diff --git a/ebpf-poc/go.mod b/ebpf-poc/go.mod index 11a65f5..5f87e06 100644 --- a/ebpf-poc/go.mod +++ b/ebpf-poc/go.mod @@ -2,7 +2,6 @@ module ebpf-poc go 1.25.0 -require ( - github.com/cilium/ebpf v0.22.0 // indirect - golang.org/x/sys v0.43.0 // indirect -) +require github.com/cilium/ebpf v0.22.0 + +require golang.org/x/sys v0.43.0 // indirect diff --git a/ebpf-poc/go.sum b/ebpf-poc/go.sum index 02d7cd8..bc23b6c 100644 --- a/ebpf-poc/go.sum +++ b/ebpf-poc/go.sum @@ -1,4 +1,26 @@ github.com/cilium/ebpf v0.22.0 h1:v2ktp0roffpMOj2MMf3idtCQZOsAoC4BJbAJN+ke2bY= github.com/cilium/ebpf v0.22.0/go.mod h1:CDzZbe2hC5JjlDC+CY3KFCzlYwN4gbxppYM+Z10bQt4= +github.com/go-quicktest/qt v1.101.1-0.20240301121107-c6c8733fa1e6 h1:teYtXy9B7y5lHTp8V9KPxpYRAVA7dozigQcMiBust1s= +github.com/go-quicktest/qt v1.101.1-0.20240301121107-c6c8733fa1e6/go.mod h1:p4lGIVX+8Wa6ZPNDvqcxq36XpUDLh42FLetFU7odllI= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/josharian/native v1.1.0 h1:uuaP0hAbW7Y4l0ZRQ6C9zfb7Mg1mbFKry/xzDAfmtLA= +github.com/josharian/native v1.1.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= +github.com/jsimonetti/rtnetlink/v2 v2.0.1 h1:xda7qaHDSVOsADNouv7ukSuicKZO7GgVUCXxpaIEIlM= +github.com/jsimonetti/rtnetlink/v2 v2.0.1/go.mod h1:7MoNYNbb3UaDHtF8udiJo/RH6VsTKP1pqKLUTVCvToE= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mdlayher/netlink v1.7.2 h1:/UtM3ofJap7Vl4QWCPDGXY8d3GIY2UGSDbK+QWmY8/g= +github.com/mdlayher/netlink v1.7.2/go.mod h1:xraEF7uJbxLhc5fpHL4cPe221LI2bdttWlU+ZGLfQSw= +github.com/mdlayher/socket v0.5.1 h1:VZaqt6RkGkt2OE9l3GcC6nZkqD3xKeQLyfleW/uBcos= +github.com/mdlayher/socket v0.5.1/go.mod h1:TjPLHI1UgwEv5J1B5q0zTZq12A/6H7nKmtTanQE37IQ= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= +golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= diff --git a/ebpf-poc/main.go b/ebpf-poc/main.go index 922edf2..a661ae4 100644 --- a/ebpf-poc/main.go +++ b/ebpf-poc/main.go @@ -1,4 +1,4 @@ -//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -type event bpf connect.c -- -I/usr/include/aarch64-linux-gnu +//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -type event -type target bpf connect.c -- -I/usr/include/aarch64-linux-gnu package main @@ -7,10 +7,13 @@ import ( "context" "encoding/binary" "errors" + "flag" "fmt" "net" "os" "os/signal" + "strconv" + "strings" "syscall" "github.com/cilium/ebpf" @@ -19,46 +22,65 @@ import ( "github.com/cilium/ebpf/rlimit" ) +// Mirrors the ACTION_* constants in connect.c. +var actionNames = map[uint8]string{ + 0: "REDIRECT", + 1: "skip/proto", + 2: "skip/loopback", + 3: "skip/exempt", + 4: "skip/dport", + 5: "skip/no-target", +} + func main() { - err := rlimit.RemoveMemlock() - if err != nil { - fmt.Printf("Failed to remove mem lock: %v\n", err) - return + proxyFlag := flag.String("proxy", "", "redirect target as host:port, e.g. 127.0.0.1:8443") + exemptFlag := flag.String("exempt-uid", "", "comma separated uids that are never redirected") + tcpOnlyFlag := flag.Bool("tcp-only", false, "only print TCP events, hiding DNS and route probe noise") + flag.Parse() + + if err := run(*proxyFlag, *exemptFlag, *tcpOnlyFlag); err != nil { + fmt.Fprintf(os.Stderr, "pmgwatch: %v\n", err) + os.Exit(1) + } +} + +func run(proxyAddr, exemptUIDs string, tcpOnly bool) error { + if err := rlimit.RemoveMemlock(); err != nil { + return fmt.Errorf("remove memlock: %w", err) } var objs bpfObjects - err = loadBpfObjects(&objs, nil) - if err != nil { - fmt.Printf("Failed to load bpf objects: %v\n", err) - return + if err := loadBpfObjects(&objs, nil); err != nil { + return fmt.Errorf("load bpf objects: %w", err) } - defer objs.Close() - rootCgroup := "/sys/fs/cgroup" + if err := configureTarget(&objs, proxyAddr); err != nil { + return err + } + + if err := configureExempt(&objs, exemptUIDs); err != nil { + return err + } l, err := link.AttachCgroup(link.CgroupOptions{ - Path: rootCgroup, + Path: "/sys/fs/cgroup", Attach: ebpf.AttachCGroupInet4Connect, Program: objs.Connect4, }) if err != nil { - fmt.Printf("Failed to attach cgroup: %v\n", err) - return + return fmt.Errorf("attach cgroup: %w", err) } defer l.Close() rd, err := ringbuf.NewReader(objs.Events) if err != nil { - fmt.Printf("Failed to create ringbuf reader: %v\n", err) - return + return fmt.Errorf("open ringbuf: %w", err) } defer rd.Close() ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) - defer stop() // Cleans up resources allocated by the signal package - - fmt.Println("Application started. Press Ctrl+C to exit.") + defer stop() go func() { <-ctx.Done() @@ -66,41 +88,139 @@ func main() { rd.Close() }() + fmt.Println("Shadow mode: decisions are recorded, nothing is redirected. Ctrl+C to exit.") + fmt.Printf("%-15s %-17s %-7s %-8s %-22s %s\n", "ACTION", "COMMAND", "UID", "PID", "DESTINATION", "PROTO") + + return drain(rd, tcpOnly) +} + +func drain(rd *ringbuf.Reader, tcpOnly bool) error { var e bpfEvent + for { rec, err := rd.Read() if err != nil { if errors.Is(err, ringbuf.ErrClosed) { - return + return nil } - fmt.Printf("failed to read raw event: %v\n", err) + + fmt.Fprintf(os.Stderr, "read event: %v\n", err) continue } - err = binary.Read(bytes.NewReader(rec.RawSample), binary.LittleEndian, &e) - if err != nil { - fmt.Printf("failed to read event: %v\n", err) + if err := binary.Read(bytes.NewReader(rec.RawSample), binary.LittleEndian, &e); err != nil { + fmt.Fprintf(os.Stderr, "decode event: %v\n", err) continue } - ip := make(net.IP, 4) - binary.LittleEndian.PutUint32(ip, e.Daddr) - - name := e.Comm[:] - if i := bytes.IndexByte(name, 0); i != -1 { - name = name[:i] // keep bytes before the first NUL - } - comm := string(name) - - proto := "" - switch e.Proto { - case syscall.IPPROTO_TCP: - proto = "TCP" - case syscall.IPPROTO_UDP: - proto = "UDP" + if tcpOnly && e.Proto != syscall.IPPROTO_TCP { + continue } - fmt.Println() - fmt.Printf("PID: %v\nUID: %v\ndAddr: %v\ndPort: %v\nProto: %v\nCommand: %v\n", e.Pid, e.Uid, ip, e.Dport, proto, comm) + printEvent(&e) } } + +func printEvent(e *bpfEvent) { + ip := make(net.IP, 4) + binary.LittleEndian.PutUint32(ip, e.Daddr) + + action, ok := actionNames[e.Action] + if !ok { + action = fmt.Sprintf("action/%d", e.Action) + } + + fmt.Printf("%-15s %-17s %-7d %-8d %-22s %s\n", + action, comm(e.Comm[:]), e.Uid, e.Pid, + net.JoinHostPort(ip.String(), strconv.Itoa(int(e.Dport))), proto(e.Proto)) +} + +// comm is a fixed 16 byte buffer, NUL terminated when the name is shorter. +func comm(raw []byte) string { + if i := bytes.IndexByte(raw, 0); i != -1 { + raw = raw[:i] + } + + return string(raw) +} + +func proto(p uint8) string { + switch p { + case syscall.IPPROTO_TCP: + return "TCP" + case syscall.IPPROTO_UDP: + return "UDP" + default: + return strconv.Itoa(int(p)) + } +} + +// configureTarget writes the redirect destination the hook will use. Leaving it +// unset is valid: every otherwise eligible connection then reports +// skip/no-target, which is how shadow mode runs before a proxy exists. +func configureTarget(objs *bpfObjects, addr string) error { + if addr == "" { + fmt.Println("No redirect target set, eligible connections will report skip/no-target") + return nil + } + + host, portStr, err := net.SplitHostPort(addr) + if err != nil { + return fmt.Errorf("parse proxy address %q: %w", addr, err) + } + + ip := net.ParseIP(host) + if ip == nil || ip.To4() == nil { + return fmt.Errorf("proxy address %q must be an IPv4 address", addr) + } + + port, err := strconv.ParseUint(portStr, 10, 16) + if err != nil { + return fmt.Errorf("parse proxy port %q: %w", portStr, err) + } + + // The kernel holds the address in network byte order, which is what reading + // the four octets little endian reproduces. + target := bpfTarget{ + Ip: binary.LittleEndian.Uint32(ip.To4()), + Port: uint16(port), + } + + if err := objs.TargetMap.Put(uint32(0), target); err != nil { + return fmt.Errorf("write redirect target: %w", err) + } + + fmt.Printf("Redirect target: %s\n", addr) + + return nil +} + +func configureExempt(objs *bpfObjects, list string) error { + if list == "" { + fmt.Println("No exempt uids set, nothing bypasses the ladder") + return nil + } + + var applied []string + for _, field := range strings.Split(list, ",") { + field = strings.TrimSpace(field) + if field == "" { + continue + } + + uid, err := strconv.ParseUint(field, 10, 32) + if err != nil { + return fmt.Errorf("parse exempt uid %q: %w", field, err) + } + + if err := objs.ExemptMap.Put(uint32(uid), uint8(1)); err != nil { + return fmt.Errorf("write exempt uid %d: %w", uid, err) + } + + applied = append(applied, field) + } + + fmt.Printf("Exempt uids: %s\n", strings.Join(applied, ", ")) + + return nil +}