From 2aa5c051cd537b95f95021a685827a452e4e4ee1 Mon Sep 17 00:00:00 2001 From: Marc Sanmiquel Date: Fri, 20 Dec 2024 11:55:08 +0100 Subject: [PATCH 1/5] feat(symbolization): Add DWARF symbolization POC with debuginfod support --- cmd/symbolization/main.go | 104 +++++ go.mod | 1 + go.sum | 2 + go.work.sum | 22 ++ pkg/experiment/symbolization/addrmapper.go | 190 +++++++++ .../symbolization/debuginfod_client.go | 54 +++ pkg/experiment/symbolization/dwarf.go | 373 ++++++++++++++++++ pkg/experiment/symbolization/symbolizer.go | 139 +++++++ pkg/experiment/symbolization/types.go | 45 +++ 9 files changed, 930 insertions(+) create mode 100644 cmd/symbolization/main.go create mode 100644 pkg/experiment/symbolization/addrmapper.go create mode 100644 pkg/experiment/symbolization/debuginfod_client.go create mode 100644 pkg/experiment/symbolization/dwarf.go create mode 100644 pkg/experiment/symbolization/symbolizer.go create mode 100644 pkg/experiment/symbolization/types.go diff --git a/cmd/symbolization/main.go b/cmd/symbolization/main.go new file mode 100644 index 0000000000..9c37fdf61b --- /dev/null +++ b/cmd/symbolization/main.go @@ -0,0 +1,104 @@ +package main + +import ( + "context" + "fmt" + "log" + + pprof "github.com/google/pprof/profile" + "github.com/grafana/pyroscope/pkg/experiment/symbolization" +) + +const ( + debuginfodBaseURL = "https://debuginfod.elfutils.org" + buildID = "2fa2055ef20fabc972d5751147e093275514b142" +) + +func main() { + client := symbolization.NewDebuginfodClient(debuginfodBaseURL) + + // Alternatively, use a local debug info file: + //client := &localDebuginfodClient{debugFilePath: "/path/to/your/debug/file"} + + symbolizer := symbolization.NewSymbolizer(client) + ctx := context.Background() + + _, err := client.FetchDebuginfo(buildID) + if err != nil { + log.Fatalf("Failed to fetch debug info: %v", err) + } + //defer os.Remove(debugFilePath) + + // Create a request to symbolize specific addresses + req := symbolization.Request{ + BuildID: buildID, + Mappings: []symbolization.RequestMapping{ + { + Locations: []*symbolization.Location{ + { + Address: 0x1500, + Mapping: &pprof.Mapping{}, + }, + { + Address: 0x3c5a, + Mapping: &pprof.Mapping{}, + }, + { + Address: 0x2745, + Mapping: &pprof.Mapping{}, + }, + }, + }, + }, + } + + if err := symbolizer.Symbolize(ctx, req); err != nil { + log.Fatalf("Failed to symbolize: %v", err) + } + + fmt.Println("Symbolization Results:") + fmt.Printf("Build ID: %s\n", buildID) + fmt.Println("----------------------------------------") + + for i, mapping := range req.Mappings { + fmt.Printf("Mapping #%d:\n", i+1) + for _, loc := range mapping.Locations { + fmt.Printf("\nAddress: 0x%x\n", loc.Address) + if len(loc.Lines) == 0 { + fmt.Println(" No symbolization information found") + continue + } + + for j, line := range loc.Lines { + fmt.Printf(" Line %d:\n", j+1) + if line.Function != nil { + fmt.Printf(" Function: %s\n", line.Function.Name) + fmt.Printf(" File: %s\n", line.Function.Filename) + fmt.Printf(" Line: %d\n", line.Line) + fmt.Printf(" StartLine: %d\n", line.Function.StartLine) + } else { + fmt.Println(" No function information available") + } + } + fmt.Println("----------------------------------------") + } + } + + // Alternatively: Symbolize all addresses in the binary + // Note: Comment out the above specific symbolization when using this + // as it's a different approach meant for exploring all available symbols + //if err := symbolizer.SymbolizeAll(ctx, buildID); err != nil { + // log.Fatalf("Failed to symbolize all addresses: %v", err) + //} + + fmt.Println("\nSymbolization completed successfully.") +} + +// localDebuginfodClient provides a way to use local debug info files instead of fetching from a server +type localDebuginfodClient struct { + debugFilePath string +} + +func (c *localDebuginfodClient) FetchDebuginfo(buildID string) (string, error) { + return c.debugFilePath, nil +} diff --git a/go.mod b/go.mod index 9aa176ed9d..1f37dad4bf 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,7 @@ require ( github.com/felixge/fgprof v0.9.4-0.20221116204635-ececf7638e93 github.com/felixge/httpsnoop v1.0.4 github.com/fsnotify/fsnotify v1.7.0 + github.com/go-delve/delve v1.23.1 github.com/go-kit/log v0.2.1 github.com/gogo/protobuf v1.3.2 github.com/gogo/status v1.1.1 diff --git a/go.sum b/go.sum index eaf480caa4..2354768b56 100644 --- a/go.sum +++ b/go.sum @@ -241,6 +241,8 @@ github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nos github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/fullstorydev/emulators/storage v0.0.0-20240401123056-edc69752f474 h1:TufioMBjkJ6/Oqmlye/ReuxHFS35HyLmypj/BNy/8GY= github.com/fullstorydev/emulators/storage v0.0.0-20240401123056-edc69752f474/go.mod h1:PQwxF4UU8wuL+srGxr3BOhIW5zXqgucwVlO/nPZLsxw= +github.com/go-delve/delve v1.23.1 h1:MtZ13ppptttkqSuvVnwJ5CPhIAzDiOwRrYuCk3ES7fU= +github.com/go-delve/delve v1.23.1/go.mod h1:S3SLuEE2mn7wipKilTvk1p9HdTMnXXElcEpiZ+VcuqU= github.com/go-fonts/dejavu v0.3.4 h1:Qqyx9IOs5CQFxyWTdvddeWzrX0VNwUAvbmAzL0fpjbc= github.com/go-fonts/dejavu v0.3.4/go.mod h1:D1z0DglIz+lmpeNYMYlxW4r22IhcdOYnt+R3PShU/Kg= github.com/go-fonts/latin-modern v0.3.3 h1:g2xNgI8yzdNzIVm+qvbMryB6yGPe0pSMss8QT3QwlJ0= diff --git a/go.work.sum b/go.work.sum index 865979150f..bac225dc4a 100644 --- a/go.work.sum +++ b/go.work.sum @@ -628,13 +628,21 @@ github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3 github.com/coreos/go-oidc/v3 v3.5.0 h1:VxKtbccHZxs8juq7RdJntSqtXFtde9YpNpGn0yqgEHw= github.com/coreos/go-oidc/v3 v3.5.0/go.mod h1:ecXRtV4romGPeO6ieExAsUK9cb/3fp9hXNz1tlv8PIM= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/cosiner/argv v0.1.0 h1:BVDiEL32lwHukgJKP87btEPenzrrHUjajs/8yzaqcXg= +github.com/cosiner/argv v0.1.0/go.mod h1:EusR6TucWKX+zFgtdUsKT2Cvg45K5rtpCcWz4hK06d8= +github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9 h1:uDmaGzcdjhF4i/plgjmEsriH11Y0o7RKapEf/LDaM3w= github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/creack/pty v1.1.20 h1:VIPb/a2s17qNeQgDnkfZC35RScx+blkKF8GV68n80J4= +github.com/creack/pty v1.1.20/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/cristalhq/hedgedhttp v0.9.1 h1:g68L9cf8uUyQKQJwciD0A1Vgbsz+QgCjuB1I8FAsCDs= github.com/cristalhq/hedgedhttp v0.9.1/go.mod h1:XkqWU6qVMutbhW68NnzjWrGtH8NUx1UfYqGYtHVKIsI= github.com/davecgh/go-xdr v0.0.0-20161123171359-e6a2ba005892 h1:qg9VbHo1TlL0KDM0vYvBG9EY0X0Yku5WYIPoFWt8f6o= github.com/davecgh/go-xdr v0.0.0-20161123171359-e6a2ba005892/go.mod h1:CTDl0pzVzE5DEzZhPfvhY/9sPFMQIxaJ9VAMs9AagrE= +github.com/derekparker/trie v0.0.0-20230829180723-39f4de51ef7d h1:hUWoLdw5kvo2xCsqlsIBMvWUc1QCSsCYD2J2+Fg6YoU= +github.com/derekparker/trie v0.0.0-20230829180723-39f4de51ef7d/go.mod h1:C7Es+DLenIpPc9J6IYw4jrK0h7S9bKj4DNl8+KxGEXU= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/digitalocean/godo v1.104.1/go.mod h1:VAI/L5YDzMuPRU01lEEUSQ/sp5Z//1HnnFv/RBTEdbg= @@ -658,6 +666,8 @@ github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2 github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 h1:Mn26/9ZMNWSw9C9ERFA1PUxfmGpolnw2v0bKOREu5ew= github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32/go.mod h1:GIjDIg/heH5DOkXY3YJ/wNhfHsQHoXGjl8G8amsYQ1I= +github.com/go-delve/liner v1.2.3-0.20231231155935-4726ab1d7f62 h1:IGtvsNyIuRjl04XAOFGACozgUD7A82UffYxZt4DWbvA= +github.com/go-delve/liner v1.2.3-0.20231231155935-4726ab1d7f62/go.mod h1:biJCRbqp51wS+I92HMqn5H8/A0PAhxn2vyOT+JqhiGI= github.com/go-fonts/stix v0.2.2 h1:v9krocr13J1llaOHLEol1eaHsv8S43UuFX/1bFgEJJ4= github.com/go-fonts/stix v0.2.2/go.mod h1:SUxggC9dxd/Q+rb5PkJuvfvTbOPtNc2Qaua00fIp9iU= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1 h1:QbL/5oDUmRBzO9/Z7Seo6zf912W/a6Sr4Eu0G/3Jho0= @@ -739,6 +749,8 @@ github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9 github.com/google/flatbuffers v2.0.8+incompatible h1:ivUb1cGomAB101ZM1T0nOiWz9pSrTMoa9+EiY7igmkM= github.com/google/flatbuffers v2.0.8+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-dap v0.12.0 h1:rVcjv3SyMIrpaOoTAdFDyHs99CwVOItIJGKLQFQhNeM= +github.com/google/go-dap v0.12.0/go.mod h1:tNjCASCm5cqePi/RVXXWEVqtnNLV1KTWtYOqu6rZNzc= github.com/google/go-pkcs11 v0.2.1-0.20230907215043-c6f79328ddf9 h1:OF1IPgv+F4NmqmJ98KTjdN97Vs1JxDPB3vbmYzV2dpk= github.com/google/go-pkcs11 v0.2.1-0.20230907215043-c6f79328ddf9/go.mod h1:6eQoGcuNJpa7jnd5pMGdkSaQpNDYvPlXWMcjXXThLlY= github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= @@ -793,6 +805,8 @@ github.com/hudl/fargo v1.4.0/go.mod h1:9Ai6uvFy5fQNq6VPKtg+Ceq1+eTY4nKUlR2JElEOc github.com/iancoleman/strcase v0.2.0 h1:05I4QRnGpI0m37iZQRuskXh+w77mr6Z41lwQzuHLwW0= github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/influxdata/influxdb1-client v0.0.0-20200827194710-b269163b24ab h1:HqW4xhhynfjrtEiiSGcQUd6vrK23iMam1FO8rI7mwig= github.com/influxdata/influxdb1-client v0.0.0-20200827194710-b269163b24ab/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/ionos-cloud/sdk-go/v6 v6.1.9/go.mod h1:EzEgRIDxBELvfoa/uBN0kOQaqovLjUWEB7iW4/Q+t4k= @@ -963,6 +977,8 @@ github.com/rs/cors v1.10.1 h1:L0uuZVXIKlI1SShY2nhFfo44TYvDPQ1w4oFkUJNfhyo= github.com/rs/cors v1.10.1/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/russross/blackfriday v1.6.0 h1:KqfZb0pUVN2lYqZUYRddxF4OR8ZMURnJIG5Y3VRLtww= github.com/russross/blackfriday v1.6.0/go.mod h1:ti0ldHuxg49ri4ksnFxlkCfN+hvslNlmVHqNRXXJNAY= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ruudk/golang-pdf417 v0.0.0-20201230142125-a7e3863a1245 h1:K1Xf3bKttbF+koVGaX5xngRIZ5bVjbmPnaxE/dR08uY= github.com/ruudk/golang-pdf417 v0.0.0-20201230142125-a7e3863a1245/go.mod h1:pQAZKsJ8yyVxGRWYNEm9oFB8ieLgKFnamEyDmSA0BRk= github.com/ryanuber/columnize v2.1.0+incompatible h1:j1Wcmh8OrK4Q7GXY+V7SVSY8nUWQxHW5TkBe7YUl+2s= @@ -993,6 +1009,8 @@ github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkU github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8= +github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= +github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= @@ -1082,9 +1100,13 @@ go.opentelemetry.io/otel/trace v1.17.0/go.mod h1:I/4vKTgFclIsXRVucpH25X0mpFSczM7 go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ= go.opentelemetry.io/otel/trace v1.22.0/go.mod h1:RbbHXVqKES9QhzZq/fE5UnOSILqRt40a21sPw2He1xo= go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= +go.starlark.net v0.0.0-20231101134539-556fd59b42f6 h1:+eC0F/k4aBLC4szgOcjd7bDTEnpxADJyWJE0yowgM3E= +go.starlark.net v0.0.0-20231101134539-556fd59b42f6/go.mod h1:LcLNIzVOMp4oV+uusnpk+VU+SzXaJakUuBjoCSWH5dM= go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= +golang.org/x/arch v0.6.0 h1:S0JTfE48HbRj80+4tbvZDYsJ3tGv6BUU3XxyZ7CirAc= +golang.org/x/arch v0.6.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/crypto v0.0.0-20200414173820-0848c9571904/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= diff --git a/pkg/experiment/symbolization/addrmapper.go b/pkg/experiment/symbolization/addrmapper.go new file mode 100644 index 0000000000..3fc020e9ac --- /dev/null +++ b/pkg/experiment/symbolization/addrmapper.go @@ -0,0 +1,190 @@ +package symbolization + +import ( + "debug/elf" + "fmt" +) + +// BinaryLayout contains the information needed to translate between runtime addresses +// and the addresses in the debug information. This is necessary because: +// 1. Executables (ET_EXEC) use fixed addresses but may need segment offset adjustments +// 2. Shared libraries (ET_DYN) can be loaded at any address, requiring base address calculations +// 3. Relocatable files (ET_REL) need special handling for their relocations +type BinaryLayout struct { + ElfType uint16 + ProgramHeaders []MemoryRegion +} + +// MemoryRegion represents a loadable segment in the ELF file. +// These segments define how the program should be loaded into memory: +// - Off: where the segment data starts in the file +// - Vaddr: the virtual address where the segment should be loaded +// - Memsz: how much memory the segment occupies when loaded +type MemoryRegion struct { + Off uint64 // File offset + Vaddr uint64 // Virtual address + Filesz uint64 // Size in file + Memsz uint64 // Size in memory (may be larger than Filesz due to .bss) + Type uint32 +} + +func ExecutableInfoFromELF(f *elf.File) (*BinaryLayout, error) { + loadableSegments := make([]MemoryRegion, 0, len(f.Progs)) + for _, segment := range f.Progs { + if segment.Type == elf.PT_LOAD { + loadableSegments = append(loadableSegments, MemoryRegion{ + Off: segment.Off, + Vaddr: segment.Vaddr, + Filesz: segment.Filesz, + Memsz: segment.Memsz, + Type: uint32(segment.Type), + }) + } + } + + return &BinaryLayout{ + ElfType: uint16(f.Type), + ProgramHeaders: loadableSegments, + }, nil +} + +// MapRuntimeAddress translates a runtime address to its corresponding address +// in the debug information. This translation is necessary because: +// - The program might be loaded at a different address than it was linked for +// - Different segments might need different adjustments +// - Various ELF types (EXEC, DYN, REL) handle addressing differently +func MapRuntimeAddress(runtimeAddr uint64, ei *BinaryLayout, m Mapping) (uint64, error) { + baseOffset, err := CalculateBase(ei, m, runtimeAddr) + if err != nil { + return runtimeAddr, fmt.Errorf("calculate base offset: %w", err) + } + + return runtimeAddr - baseOffset, nil +} + +// CalculateBase determines the base address adjustment needed for address translation. +// The calculation varies depending on the ELF type: +// - ET_EXEC: Uses fixed addresses with potential segment adjustments +// - ET_DYN: Can be loaded anywhere, needs runtime base address adjustment +// - ET_REL: Requires relocation processing +func CalculateBase(ei *BinaryLayout, m Mapping, addr uint64) (uint64, error) { + segment, err := ei.FindProgramHeader(m, addr) + if err != nil { + return 0, fmt.Errorf("find program segment: %w", err) + } + + if segment == nil { + return 0, nil + } + + // Handle special case where mapping spans entire address space + if m.Start == 0 && m.Offset == 0 && (m.Limit == ^uint64(0) || m.Limit == 0) { + return 0, nil + } + + switch elf.Type(ei.ElfType) { + case elf.ET_EXEC: + return calculateExecBase(m, segment) + case elf.ET_REL: + return calculateRelocatableBase(m) + case elf.ET_DYN: + return calculateDynamicBase(m, segment) + } + + return 0, fmt.Errorf("unsupported ELF type: %v", elf.Type(ei.ElfType)) +} + +// FindProgramHeader finds the program header containing the given address. +// It returns nil if no header is found. +func (ei *BinaryLayout) FindProgramHeader(m Mapping, addr uint64) (*MemoryRegion, error) { + // Special case: if mapping is empty (all zeros), just look for any header containing the address + if m.Start == 0 && m.Limit == 0 { + for i := range ei.ProgramHeaders { + h := &ei.ProgramHeaders[i] + if h.Type == uint32(elf.PT_LOAD) { + if h.Vaddr <= addr && addr < h.Vaddr+h.Memsz { + return h, nil + } + } + } + return nil, nil + } + + // Fast path: if address is invalid or outside reasonable range + if m.Start >= m.Limit { + return nil, fmt.Errorf("invalid mapping range: start %x >= limit %x", m.Start, m.Limit) + } + + // Special case: kernel addresses or very high addresses + if m.Limit >= (1 << 63) { + return nil, nil + } + + // No loadable segments + if len(ei.ProgramHeaders) == 0 { + return nil, nil + } + + // Calculate file offset from the address + fileOffset := addr - m.Start + m.Offset + + // Find all headers that could contain this address + var candidateHeaders []*MemoryRegion + for i := range ei.ProgramHeaders { + h := &ei.ProgramHeaders[i] + if h.Type != uint32(elf.PT_LOAD) { + continue + } + + // Check if the file offset falls within this segment + if fileOffset >= h.Off && fileOffset < h.Off+h.Memsz { + candidateHeaders = append(candidateHeaders, h) + } + } + + // No matching headers found + if len(candidateHeaders) == 0 { + return nil, nil + } + + // If only one header matches, return it + if len(candidateHeaders) == 1 { + return candidateHeaders[0], nil + } + + // Multiple headers - need to select the most appropriate one + // Choose the one with the closest starting address to our target + var bestHeader *MemoryRegion + bestDistance := uint64(^uint64(0)) // Max uint64 as initial distance + + for _, h := range candidateHeaders { + distance := addr - h.Vaddr + if distance < bestDistance { + bestDistance = distance + bestHeader = h + } + } + + return bestHeader, nil +} + +func calculateExecBase(m Mapping, h *MemoryRegion) (uint64, error) { + if h == nil { + return 0, nil + } + return m.Start - m.Offset + h.Off - h.Vaddr, nil +} + +func calculateRelocatableBase(m Mapping) (uint64, error) { + if m.Offset != 0 { + return 0, fmt.Errorf("relocatable files with non-zero offset not supported") + } + return m.Start, nil +} + +func calculateDynamicBase(m Mapping, h *MemoryRegion) (uint64, error) { + if h == nil { + return m.Start - m.Offset, nil + } + return m.Start - m.Offset + h.Off - h.Vaddr, nil +} diff --git a/pkg/experiment/symbolization/debuginfod_client.go b/pkg/experiment/symbolization/debuginfod_client.go new file mode 100644 index 0000000000..5318db1488 --- /dev/null +++ b/pkg/experiment/symbolization/debuginfod_client.go @@ -0,0 +1,54 @@ +package symbolization + +import ( + "fmt" + "io" + "net/http" + "os" + "path/filepath" +) + +type DebuginfodClient interface { + FetchDebuginfo(buildID string) (string, error) +} + +type debuginfodClient struct { + baseURL string +} + +func NewDebuginfodClient(baseURL string) DebuginfodClient { + return &debuginfodClient{ + baseURL: baseURL, + } +} + +// FetchDebuginfo fetches the debuginfo file for a specific build ID. +func (c *debuginfodClient) FetchDebuginfo(buildID string) (string, error) { + url := fmt.Sprintf("%s/buildid/%s/debuginfo", c.baseURL, buildID) + + resp, err := http.Get(url) + if err != nil { + return "", fmt.Errorf("failed to fetch debuginfod: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("unexpected HTTP status: %s", resp.Status) + } + + // Save the debuginfo to a temporary file + tempDir := os.TempDir() + filePath := filepath.Join(tempDir, fmt.Sprintf("%s.elf", buildID)) + outFile, err := os.Create(filePath) + if err != nil { + return "", fmt.Errorf("failed to create temp file: %w", err) + } + defer outFile.Close() + + _, err = io.Copy(outFile, resp.Body) + if err != nil { + return "", fmt.Errorf("failed to write debuginfod to file: %w", err) + } + + return filePath, nil +} diff --git a/pkg/experiment/symbolization/dwarf.go b/pkg/experiment/symbolization/dwarf.go new file mode 100644 index 0000000000..e2fb172c8e --- /dev/null +++ b/pkg/experiment/symbolization/dwarf.go @@ -0,0 +1,373 @@ +package symbolization + +import ( + "context" + "debug/dwarf" + "errors" + "fmt" + "io" + "sort" + + "github.com/go-delve/delve/pkg/dwarf/godwarf" + "github.com/go-delve/delve/pkg/dwarf/reader" + pprof "github.com/google/pprof/profile" +) + +// DWARFInfo implements the liner interface +type DWARFInfo struct { + debugData *dwarf.Data + lineEntries map[dwarf.Offset][]dwarf.LineEntry + subprograms map[dwarf.Offset][]*godwarf.Tree + abstractSubprograms map[dwarf.Offset]*dwarf.Entry +} + +// NewDWARFInfo creates a new liner using DWARF debug info +func NewDWARFInfo(debugData *dwarf.Data) *DWARFInfo { + return &DWARFInfo{ + debugData: debugData, + lineEntries: make(map[dwarf.Offset][]dwarf.LineEntry), + subprograms: make(map[dwarf.Offset][]*godwarf.Tree), + abstractSubprograms: make(map[dwarf.Offset]*dwarf.Entry), + } +} + +func (d *DWARFInfo) ResolveAddress(_ context.Context, addr uint64) ([]SymbolLocation, error) { + er := reader.New(d.debugData) + cu, err := er.SeekPC(addr) + if err != nil { + return nil, fmt.Errorf("no symbol information found for address 0x%x", addr) + } + if cu == nil { + return nil, errors.New("no symbol information found for address") + } + + if err := d.buildLookupTables(cu); err != nil { + return nil, err + } + + var lines []SymbolLocation + var targetTree *godwarf.Tree + for _, tree := range d.subprograms[cu.Offset] { + if tree.ContainsPC(addr) { + targetTree = tree + break + } + } + + if targetTree == nil { + return lines, nil + } + + functionName, ok := targetTree.Entry.Val(dwarf.AttrName).(string) + if !ok { + functionName = "" + } + + declLine, ok := targetTree.Entry.Val(dwarf.AttrDeclLine).(int64) + if !ok { + declLine = 0 + } + + file, line := d.findLineInfo(d.lineEntries[cu.Offset], targetTree.Ranges) + lines = append(lines, SymbolLocation{ + Function: &pprof.Function{ + Name: functionName, + Filename: file, + StartLine: declLine, + }, + Line: line, + }) + + // Enhanced inline function processing + for _, tr := range reader.InlineStack(targetTree, addr) { + + var functionName string + if tr.Tag == dwarf.TagSubprogram { + functionName, ok = targetTree.Entry.Val(dwarf.AttrName).(string) + if !ok { + functionName = "" + } + } else { + if abstractOffset, ok := tr.Entry.Val(dwarf.AttrAbstractOrigin).(dwarf.Offset); ok { + if abstractOrigin, exists := d.abstractSubprograms[abstractOffset]; exists { + functionName = d.getFunctionName(abstractOrigin) + } else { + functionName = "?" + } + } else { + functionName = "?" + } + } + + declLine, ok := tr.Entry.Val(dwarf.AttrDeclLine).(int64) + if !ok { + declLine = 0 + } + + file, line := d.findLineInfo(d.lineEntries[cu.Offset], tr.Ranges) + + lines = append(lines, SymbolLocation{ + Function: &pprof.Function{ + Name: functionName, + Filename: file, + StartLine: declLine, + }, + Line: line, + }) + } + + return lines, nil +} + +func (d *DWARFInfo) resolveFunctionName(entry *dwarf.Entry) string { + if entry == nil { + return "?" + } + + if name, ok := entry.Val(dwarf.AttrName).(string); ok { + return name + } + if name, ok := entry.Val(dwarf.AttrLinkageName).(string); ok { + return name + } + + return "?" +} + +func (d *DWARFInfo) buildLookupTables(cu *dwarf.Entry) error { + // Check if we already processed this compilation unit + if _, exists := d.lineEntries[cu.Offset]; exists { + return nil + } + + // TODO: not 100% sure about it. Review it. + // Scan all DWARF entries for abstract subprograms before processing this compilation unit. + // This scan is necessary because DWARF debug info can contain cross-compilation unit + // references, particularly for inlined functions. When a function is inlined, its + // definition (the abstract entry) may be in one compilation unit while its usage + // (via AttrAbstractOrigin) can be in another. By scanning all entries upfront, + // we ensure we can resolve these cross-unit references when they occur. + // + // For example, when a C++ standard library function is inlined (like printf from stdio.h), + // its abstract entry might be in the compilation unit for stdio.h, but we need to + // resolve its name when we find it inlined in our program's compilation unit. + if len(d.abstractSubprograms) == 0 { + if err := d.scanAbstractSubprograms(); err != nil { + return fmt.Errorf("scan abstract subprograms: %w", err) + } + } + + // Process line entries first + if err := d.processLineEntries(cu); err != nil { + return fmt.Errorf("process line entries: %w", err) + } + + // Process subprograms and their trees + if err := d.processSubprogramEntries(cu); err != nil { + return fmt.Errorf("process subprogram entries: %w", err) + } + + return nil +} + +func (d *DWARFInfo) processLineEntries(cu *dwarf.Entry) error { + lr, err := d.debugData.LineReader(cu) + if err != nil { + return fmt.Errorf("create line reader: %w", err) + } + if lr == nil { + return errors.New("no line reader available") + } + + entries := make([]dwarf.LineEntry, 0) + for { + var entry dwarf.LineEntry + err := lr.Next(&entry) + if err != nil { + if err == io.EOF { + break + } + return fmt.Errorf("read line entry: %w", err) + } + + // Only store statement entries + if entry.IsStmt { + entries = append(entries, entry) + } + } + + d.lineEntries[cu.Offset] = entries + return nil +} + +func (d *DWARFInfo) processSubprogramEntries(cu *dwarf.Entry) error { + reader := d.debugData.Reader() + reader.Seek(cu.Offset) + + entry, err := reader.Next() + if err != nil { + return fmt.Errorf("read initial entry: %w", err) + } + if entry == nil || entry.Tag != dwarf.TagCompileUnit { + return fmt.Errorf("unexpected entry type at CU offset: %v", cu.Offset) + } + + subprograms := make([]*godwarf.Tree, 0) + for { + entry, err := reader.Next() + if err != nil { + if err == io.EOF { + break + } + return fmt.Errorf("read entry: %w", err) + } + if entry == nil || entry.Tag == dwarf.TagCompileUnit { + break + } + + if entry.Tag != dwarf.TagSubprogram { + continue + } + + // Check for abstract entries first + isAbstract := false + for _, field := range entry.Field { + if field.Attr == dwarf.AttrInline { + d.abstractSubprograms[entry.Offset] = entry + isAbstract = true + break + } + } + + //Skip if this was an abstract entry + if isAbstract { + continue + } + + // Extract the subprogram tree + tree, err := godwarf.LoadTree(entry.Offset, d.debugData, 0) + if err != nil { + return fmt.Errorf("load subprogram tree: %w", err) + } + + subprograms = append(subprograms, tree) + } + + d.subprograms[cu.Offset] = subprograms + return nil +} + +func (d *DWARFInfo) findLineInfo(entries []dwarf.LineEntry, ranges [][2]uint64) (string, int64) { + sort.Slice(entries, func(i, j int) bool { + return entries[i].Address < entries[j].Address + }) + + // Try to find an entry that contains our target address + targetAddr := ranges[0][0] + for _, entry := range entries { + if entry.Address >= targetAddr && entry.Address < ranges[0][1] { + if entry.File != nil { + return entry.File.Name, int64(entry.Line) + } + } + } + + // Find the closest entry before our target address + var lastEntry *dwarf.LineEntry + for i := range entries { + if entries[i].Address > targetAddr { + break + } + lastEntry = &entries[i] + } + + if lastEntry != nil && lastEntry.File != nil { + return lastEntry.File.Name, int64(lastEntry.Line) + } + + return "?", 0 +} + +func (d *DWARFInfo) getFunctionName(entry *dwarf.Entry) string { + name := "?" + ok := false + if entry != nil { + for _, field := range entry.Field { + if field.Attr == dwarf.AttrName { + name, ok = field.Val.(string) + if !ok { + name = "?" + } + } + } + } + return name +} + +func (d *DWARFInfo) SymbolizeAllAddresses() map[uint64][]SymbolLocation { + results := make(map[uint64][]SymbolLocation) + + // Get all compilation units + reader := d.debugData.Reader() + for { + entry, err := reader.Next() + if err != nil || entry == nil { + break + } + + if entry.Tag != dwarf.TagCompileUnit { + continue + } + + // Get ranges for this compilation unit + ranges, err := d.debugData.Ranges(entry) + if err != nil { + fmt.Printf("Warning: Failed to get ranges for CU: %v\n", err) + continue + } + + for _, rng := range ranges { + // Skip invalid ranges + if rng[0] >= rng[1] { + continue + } + + // Sample multiple points in this range + addresses := []uint64{ + rng[0], // start + rng[0] + (rng[1]-rng[0])/2, // middle + rng[1] - 1, // end (exclusive) + } + + for _, addr := range addresses { + lines, err := d.ResolveAddress(context.Background(), addr) + if err != nil { + continue + } + + if len(lines) > 0 { + results[addr] = lines + } + } + } + } + + return results +} + +func (d *DWARFInfo) scanAbstractSubprograms() error { + reader := d.debugData.Reader() + // Scan from the start, don't stop at first CU + for { + entry, err := reader.Next() + if err != nil || entry == nil { + break + } + + if entry.Tag == dwarf.TagSubprogram { + // Store ALL subprograms, not just inline ones + d.abstractSubprograms[entry.Offset] = entry + } + } + return nil +} diff --git a/pkg/experiment/symbolization/symbolizer.go b/pkg/experiment/symbolization/symbolizer.go new file mode 100644 index 0000000000..7f9d4fab32 --- /dev/null +++ b/pkg/experiment/symbolization/symbolizer.go @@ -0,0 +1,139 @@ +package symbolization + +import ( + "context" + "debug/dwarf" + "debug/elf" + "fmt" +) + +// DwarfResolver implements the liner interface +type DwarfResolver struct { + debugData *dwarf.Data + dbgFile *DWARFInfo + file *elf.File +} + +func NewDwarfResolver(f *elf.File) (SymbolResolver, error) { + debugData, err := f.DWARF() + if err != nil { + return nil, fmt.Errorf("read DWARF data: %w", err) + } + + debugInfo := NewDWARFInfo(debugData) + + return &DwarfResolver{ + debugData: debugData, + dbgFile: debugInfo, + file: f, + }, nil +} + +func (d *DwarfResolver) ResolveAddress(ctx context.Context, pc uint64) ([]SymbolLocation, error) { + return d.dbgFile.ResolveAddress(ctx, pc) +} + +func (d *DwarfResolver) Close() error { + return d.file.Close() +} + +type Symbolizer struct { + client DebuginfodClient +} + +func NewSymbolizer(client DebuginfodClient) *Symbolizer { + return &Symbolizer{ + client: client, + } +} + +func (s *Symbolizer) Symbolize(ctx context.Context, req Request) error { + // Fetch debug info file + debugFilePath, err := s.client.FetchDebuginfo(req.BuildID) + if err != nil { + return fmt.Errorf("fetch debuginfo: %w", err) + } + + // Open ELF file + f, err := elf.Open(debugFilePath) + if err != nil { + return fmt.Errorf("open ELF file: %w", err) + } + defer f.Close() + + // Get executable info for address normalization + ei, err := ExecutableInfoFromELF(f) + if err != nil { + return fmt.Errorf("executable info from ELF: %w", err) + } + + // Create liner + liner, err := NewDwarfResolver(f) + if err != nil { + return fmt.Errorf("create liner: %w", err) + } + //defer liner.Close() + + // Process each mapping's locations + for _, mapping := range req.Mappings { + for _, loc := range mapping.Locations { + addr, err := MapRuntimeAddress(loc.Address, ei, Mapping{ + Start: loc.Mapping.Start, + Limit: loc.Mapping.Limit, + Offset: loc.Mapping.Offset, + }) + if err != nil { + return fmt.Errorf("normalize address: %w", err) + } + + // Get source lines for the address + lines, err := liner.ResolveAddress(ctx, addr) + if err != nil { + continue // Skip errors for individual addresses + } + + // Update the location directly (this is why Parca modifies the request - it's updating the shared locations) + loc.Lines = lines + } + } + + return nil +} + +func (s *Symbolizer) SymbolizeAll(ctx context.Context, buildID string) error { + // Reuse the existing debuginfo file + debugFilePath, err := s.client.FetchDebuginfo(buildID) + if err != nil { + return fmt.Errorf("fetch debuginfo: %w", err) + } + + f, err := elf.Open(debugFilePath) + if err != nil { + return fmt.Errorf("open ELF file: %w", err) + } + defer f.Close() + + debugData, err := f.DWARF() + if err != nil { + return fmt.Errorf("get DWARF data: %w", err) + } + + debugInfo := NewDWARFInfo(debugData) + allSymbols := debugInfo.SymbolizeAllAddresses() + + fmt.Println("\nSymbolizing all addresses in DWARF file:") + fmt.Println("----------------------------------------") + + for addr, lines := range allSymbols { + fmt.Printf("\nAddress: 0x%x\n", addr) + for _, line := range lines { + fmt.Printf(" Function: %s\n", line.Function.Name) + fmt.Printf(" File: %s\n", line.Function.Filename) + fmt.Printf(" Line: %d\n", line.Line) + fmt.Printf(" StartLine: %d\n", line.Function.StartLine) + fmt.Println("----------------------------------------") + } + } + + return nil +} diff --git a/pkg/experiment/symbolization/types.go b/pkg/experiment/symbolization/types.go new file mode 100644 index 0000000000..d81471d0ec --- /dev/null +++ b/pkg/experiment/symbolization/types.go @@ -0,0 +1,45 @@ +package symbolization + +import ( + "context" + + pprof "github.com/google/pprof/profile" +) + +// SymbolLocation represents a resolved source code location with function information +type SymbolLocation struct { + Function *pprof.Function + Line int64 +} + +// Location represents a memory address to be symbolized +type Location struct { + ID string + Address uint64 + Lines []SymbolLocation + Mapping *pprof.Mapping +} + +// Request represents a symbolization request for multiple addresses +type Request struct { + BuildID string + Mappings []RequestMapping +} + +type RequestMapping struct { + Locations []*Location +} + +// Mapping describes how a binary section is mapped in memory +type Mapping struct { + Start uint64 + End uint64 + Limit uint64 + Offset uint64 +} + +// SymbolResolver converts memory addresses to source code locations +type SymbolResolver interface { + ResolveAddress(ctx context.Context, addr uint64) ([]SymbolLocation, error) + //Close() error +} From ed6005f28ac3a334239a7251b1fdaec44124c05a Mon Sep 17 00:00:00 2001 From: Marc Sanmiquel Date: Fri, 20 Dec 2024 12:20:03 +0100 Subject: [PATCH 2/5] fix lint errors --- cmd/symbolization/main.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmd/symbolization/main.go b/cmd/symbolization/main.go index 9c37fdf61b..a8cdf65bd0 100644 --- a/cmd/symbolization/main.go +++ b/cmd/symbolization/main.go @@ -6,6 +6,7 @@ import ( "log" pprof "github.com/google/pprof/profile" + "github.com/grafana/pyroscope/pkg/experiment/symbolization" ) @@ -95,10 +96,13 @@ func main() { } // localDebuginfodClient provides a way to use local debug info files instead of fetching from a server +// +//nolint:all type localDebuginfodClient struct { debugFilePath string } +//nolint:all func (c *localDebuginfodClient) FetchDebuginfo(buildID string) (string, error) { return c.debugFilePath, nil } From 6b009d36c3a8f7b575ca92c4feeceaf91dab67db Mon Sep 17 00:00:00 2001 From: Marc Sanmiquel Date: Thu, 16 Jan 2025 13:14:01 +0100 Subject: [PATCH 3/5] Add symbolization inside the read path --- cmd/symbolization/main.go | 14 +- pkg/experiment/query_backend/backend.go | 19 ++ pkg/experiment/query_backend/block_reader.go | 5 +- pkg/experiment/query_backend/query.go | 26 ++- pkg/experiment/query_backend/query_tree.go | 4 +- .../addrmapper.go | 2 +- .../debuginfod_client.go | 23 +- .../{symbolization => symbolizer}/dwarf.go | 2 +- .../symbolizer.go | 4 +- .../{symbolization => symbolizer}/types.go | 2 +- pkg/phlaredb/symdb/resolver.go | 28 ++- pkg/phlaredb/symdb/resolver_tree.go | 91 ++++++++ pkg/phlaredb/symdb/resolver_tree_test.go | 197 ++++++++++++++++++ pkg/phlaredb/symdb/symdb.go | 3 + .../symdb/testdata/unsymbolized.debug | Bin 0 -> 27624 bytes 15 files changed, 391 insertions(+), 29 deletions(-) rename pkg/experiment/{symbolization => symbolizer}/addrmapper.go (99%) rename pkg/experiment/{symbolization => symbolizer}/debuginfod_client.go (59%) rename pkg/experiment/{symbolization => symbolizer}/dwarf.go (99%) rename pkg/experiment/{symbolization => symbolizer}/symbolizer.go (95%) rename pkg/experiment/{symbolization => symbolizer}/types.go (97%) create mode 100644 pkg/phlaredb/symdb/testdata/unsymbolized.debug diff --git a/cmd/symbolization/main.go b/cmd/symbolization/main.go index a8cdf65bd0..6675a3f3ed 100644 --- a/cmd/symbolization/main.go +++ b/cmd/symbolization/main.go @@ -7,7 +7,7 @@ import ( pprof "github.com/google/pprof/profile" - "github.com/grafana/pyroscope/pkg/experiment/symbolization" + "github.com/grafana/pyroscope/pkg/experiment/symbolizer" ) const ( @@ -16,12 +16,12 @@ const ( ) func main() { - client := symbolization.NewDebuginfodClient(debuginfodBaseURL) + client := symbolizer.NewDebuginfodClient(debuginfodBaseURL) // Alternatively, use a local debug info file: //client := &localDebuginfodClient{debugFilePath: "/path/to/your/debug/file"} - symbolizer := symbolization.NewSymbolizer(client) + s := symbolizer.NewSymbolizer(client) ctx := context.Background() _, err := client.FetchDebuginfo(buildID) @@ -31,11 +31,11 @@ func main() { //defer os.Remove(debugFilePath) // Create a request to symbolize specific addresses - req := symbolization.Request{ + req := symbolizer.Request{ BuildID: buildID, - Mappings: []symbolization.RequestMapping{ + Mappings: []symbolizer.RequestMapping{ { - Locations: []*symbolization.Location{ + Locations: []*symbolizer.Location{ { Address: 0x1500, Mapping: &pprof.Mapping{}, @@ -53,7 +53,7 @@ func main() { }, } - if err := symbolizer.Symbolize(ctx, req); err != nil { + if err := s.Symbolize(ctx, req); err != nil { log.Fatalf("Failed to symbolize: %v", err) } diff --git a/pkg/experiment/query_backend/backend.go b/pkg/experiment/query_backend/backend.go index e8e7a9f43c..36593d4988 100644 --- a/pkg/experiment/query_backend/backend.go +++ b/pkg/experiment/query_backend/backend.go @@ -14,16 +14,19 @@ import ( metastorev1 "github.com/grafana/pyroscope/api/gen/proto/go/metastore/v1" queryv1 "github.com/grafana/pyroscope/api/gen/proto/go/query/v1" + "github.com/grafana/pyroscope/pkg/experiment/symbolizer" "github.com/grafana/pyroscope/pkg/util" ) type Config struct { Address string `yaml:"address"` GRPCClientConfig grpcclient.Config `yaml:"grpc_client_config" doc:"description=Configures the gRPC client used to communicate between the query-frontends and the query-schedulers."` + DebuginfodURL string `yaml:"debuginfod_url"` } func (cfg *Config) RegisterFlags(f *flag.FlagSet) { f.StringVar(&cfg.Address, "query-backend.address", "localhost:9095", "") + f.StringVar(&cfg.DebuginfodURL, "query-backend.debuginfod-url", "https://debuginfod.elfutils.org", "URL of the debuginfod server") cfg.GRPCClientConfig.RegisterFlagsWithPrefix("query-backend.grpc-client-config", f) } @@ -48,6 +51,8 @@ type QueryBackend struct { backendClient QueryHandler blockReader QueryHandler + + symbolizer *symbolizer.Symbolizer } func New( @@ -57,13 +62,27 @@ func New( backendClient QueryHandler, blockReader QueryHandler, ) (*QueryBackend, error) { + var sym *symbolizer.Symbolizer + if config.DebuginfodURL != "" { + sym = symbolizer.NewSymbolizer( + symbolizer.NewDebuginfodClient(config.DebuginfodURL), + ) + } + q := QueryBackend{ config: config, logger: logger, reg: reg, backendClient: backendClient, blockReader: blockReader, + symbolizer: sym, } + + // Pass symbolizer to BlockReader if it's the right type + if br, ok := blockReader.(*BlockReader); ok { + br.symbolizer = sym + } + q.service = services.NewIdleService(q.starting, q.stopping) return &q, nil } diff --git a/pkg/experiment/query_backend/block_reader.go b/pkg/experiment/query_backend/block_reader.go index 059038ecf9..9d67567461 100644 --- a/pkg/experiment/query_backend/block_reader.go +++ b/pkg/experiment/query_backend/block_reader.go @@ -16,6 +16,7 @@ import ( queryv1 "github.com/grafana/pyroscope/api/gen/proto/go/query/v1" "github.com/grafana/pyroscope/pkg/experiment/block" + "github.com/grafana/pyroscope/pkg/experiment/symbolizer" "github.com/grafana/pyroscope/pkg/objstore" "github.com/grafana/pyroscope/pkg/util" ) @@ -48,6 +49,8 @@ type BlockReader struct { log log.Logger storage objstore.Bucket + symbolizer *symbolizer.Symbolizer + // TODO: // - Use a worker pool instead of the errgroup. // - Reusable query context. @@ -83,7 +86,7 @@ func (b *BlockReader) Invoke( object := block.NewObject(b.storage, md) for _, ds := range md.Datasets { dataset := block.NewDataset(ds, object) - qcs = append(qcs, newQueryContext(ctx, b.log, r, agg, dataset)) + qcs = append(qcs, newQueryContext(ctx, b.log, r, agg, dataset, b.symbolizer)) } } diff --git a/pkg/experiment/query_backend/query.go b/pkg/experiment/query_backend/query.go index 1243e9f31e..7618fdd944 100644 --- a/pkg/experiment/query_backend/query.go +++ b/pkg/experiment/query_backend/query.go @@ -11,6 +11,7 @@ import ( queryv1 "github.com/grafana/pyroscope/api/gen/proto/go/query/v1" "github.com/grafana/pyroscope/pkg/experiment/block" + "github.com/grafana/pyroscope/pkg/experiment/symbolizer" ) // TODO(kolesnikovae): We have a procedural definition of our queries, @@ -71,12 +72,13 @@ func registerQueryType( } type queryContext struct { - ctx context.Context - log log.Logger - req *request - agg *reportAggregator - ds *block.Dataset - err error + ctx context.Context + log log.Logger + req *request + agg *reportAggregator + ds *block.Dataset + err error + symbolizer *symbolizer.Symbolizer } func newQueryContext( @@ -85,13 +87,15 @@ func newQueryContext( req *request, agg *reportAggregator, ds *block.Dataset, + symbolizer *symbolizer.Symbolizer, ) *queryContext { return &queryContext{ - ctx: ctx, - log: log, - req: req, - agg: agg, - ds: ds, + ctx: ctx, + log: log, + req: req, + agg: agg, + ds: ds, + symbolizer: symbolizer, } } diff --git a/pkg/experiment/query_backend/query_tree.go b/pkg/experiment/query_backend/query_tree.go index 21ad773290..0de88417bf 100644 --- a/pkg/experiment/query_backend/query_tree.go +++ b/pkg/experiment/query_backend/query_tree.go @@ -56,7 +56,9 @@ func queryTree(q *queryContext, query *queryv1.Query) (*queryv1.Report, error) { defer runutil.CloseWithErrCapture(&err, profiles, "failed to close profile stream") resolver := symdb.NewResolver(q.ctx, q.ds.Symbols(), - symdb.WithResolverMaxNodes(query.Tree.GetMaxNodes())) + symdb.WithResolverMaxNodes(query.Tree.GetMaxNodes()), + symdb.WithSymbolizer(q.symbolizer)) + defer resolver.Release() if len(spanSelector) > 0 { diff --git a/pkg/experiment/symbolization/addrmapper.go b/pkg/experiment/symbolizer/addrmapper.go similarity index 99% rename from pkg/experiment/symbolization/addrmapper.go rename to pkg/experiment/symbolizer/addrmapper.go index 3fc020e9ac..1e4e7f9637 100644 --- a/pkg/experiment/symbolization/addrmapper.go +++ b/pkg/experiment/symbolizer/addrmapper.go @@ -1,4 +1,4 @@ -package symbolization +package symbolizer import ( "debug/elf" diff --git a/pkg/experiment/symbolization/debuginfod_client.go b/pkg/experiment/symbolizer/debuginfod_client.go similarity index 59% rename from pkg/experiment/symbolization/debuginfod_client.go rename to pkg/experiment/symbolizer/debuginfod_client.go index 5318db1488..9b6054e27f 100644 --- a/pkg/experiment/symbolization/debuginfod_client.go +++ b/pkg/experiment/symbolizer/debuginfod_client.go @@ -1,4 +1,4 @@ -package symbolization +package symbolizer import ( "fmt" @@ -6,6 +6,7 @@ import ( "net/http" "os" "path/filepath" + "regexp" ) type DebuginfodClient interface { @@ -24,7 +25,12 @@ func NewDebuginfodClient(baseURL string) DebuginfodClient { // FetchDebuginfo fetches the debuginfo file for a specific build ID. func (c *debuginfodClient) FetchDebuginfo(buildID string) (string, error) { - url := fmt.Sprintf("%s/buildid/%s/debuginfo", c.baseURL, buildID) + sanitizedBuildID, err := sanitizeBuildID(buildID) + if err != nil { + return "", err + } + + url := fmt.Sprintf("%s/buildid/%s/debuginfo", c.baseURL, sanitizedBuildID) resp, err := http.Get(url) if err != nil { @@ -36,9 +42,10 @@ func (c *debuginfodClient) FetchDebuginfo(buildID string) (string, error) { return "", fmt.Errorf("unexpected HTTP status: %s", resp.Status) } + // TODO: Avoid file operations and handle debuginfo in memory. // Save the debuginfo to a temporary file tempDir := os.TempDir() - filePath := filepath.Join(tempDir, fmt.Sprintf("%s.elf", buildID)) + filePath := filepath.Join(tempDir, fmt.Sprintf("%s.elf", sanitizedBuildID)) outFile, err := os.Create(filePath) if err != nil { return "", fmt.Errorf("failed to create temp file: %w", err) @@ -52,3 +59,13 @@ func (c *debuginfodClient) FetchDebuginfo(buildID string) (string, error) { return filePath, nil } + +// sanitizeBuildID ensures that the buildID is a safe and valid string for use in file paths. +func sanitizeBuildID(buildID string) (string, error) { + // Allow only alphanumeric characters, dashes, and underscores. + validBuildID := regexp.MustCompile(`^[a-zA-Z0-9_-]+$`) + if !validBuildID.MatchString(buildID) { + return "", fmt.Errorf("invalid build ID: %s", buildID) + } + return buildID, nil +} diff --git a/pkg/experiment/symbolization/dwarf.go b/pkg/experiment/symbolizer/dwarf.go similarity index 99% rename from pkg/experiment/symbolization/dwarf.go rename to pkg/experiment/symbolizer/dwarf.go index e2fb172c8e..2274163c5d 100644 --- a/pkg/experiment/symbolization/dwarf.go +++ b/pkg/experiment/symbolizer/dwarf.go @@ -1,4 +1,4 @@ -package symbolization +package symbolizer import ( "context" diff --git a/pkg/experiment/symbolization/symbolizer.go b/pkg/experiment/symbolizer/symbolizer.go similarity index 95% rename from pkg/experiment/symbolization/symbolizer.go rename to pkg/experiment/symbolizer/symbolizer.go index 7f9d4fab32..34e748c49c 100644 --- a/pkg/experiment/symbolization/symbolizer.go +++ b/pkg/experiment/symbolizer/symbolizer.go @@ -1,4 +1,4 @@ -package symbolization +package symbolizer import ( "context" @@ -92,7 +92,7 @@ func (s *Symbolizer) Symbolize(ctx context.Context, req Request) error { continue // Skip errors for individual addresses } - // Update the location directly (this is why Parca modifies the request - it's updating the shared locations) + // Update the location directly loc.Lines = lines } } diff --git a/pkg/experiment/symbolization/types.go b/pkg/experiment/symbolizer/types.go similarity index 97% rename from pkg/experiment/symbolization/types.go rename to pkg/experiment/symbolizer/types.go index d81471d0ec..87ec145567 100644 --- a/pkg/experiment/symbolization/types.go +++ b/pkg/experiment/symbolizer/types.go @@ -1,4 +1,4 @@ -package symbolization +package symbolizer import ( "context" diff --git a/pkg/phlaredb/symdb/resolver.go b/pkg/phlaredb/symdb/resolver.go index aa5438577a..1331ffb308 100644 --- a/pkg/phlaredb/symdb/resolver.go +++ b/pkg/phlaredb/symdb/resolver.go @@ -11,6 +11,7 @@ import ( googlev1 "github.com/grafana/pyroscope/api/gen/proto/go/google/v1" typesv1 "github.com/grafana/pyroscope/api/gen/proto/go/types/v1" + "github.com/grafana/pyroscope/pkg/experiment/symbolizer" "github.com/grafana/pyroscope/pkg/model" schemav1 "github.com/grafana/pyroscope/pkg/phlaredb/schemas/v1" "github.com/grafana/pyroscope/pkg/pprof" @@ -37,6 +38,8 @@ type Resolver struct { maxNodes int64 sts *typesv1.StackTraceSelector + + symbolizer *symbolizer.Symbolizer } type ResolverOption func(*Resolver) @@ -57,6 +60,12 @@ func WithResolverMaxNodes(n int64) ResolverOption { } } +func WithSymbolizer(s *symbolizer.Symbolizer) ResolverOption { + return func(r *Resolver) { + r.symbolizer = s + } +} + // WithResolverStackTraceSelector specifies the stack trace selector. // Only stack traces that belong to the callSite (have the prefix provided) // will be selected. If empty, the filter is ignored. @@ -273,7 +282,9 @@ func (r *Resolver) withSymbols(ctx context.Context, fn func(*Symbols, *SampleApp if err := p.fetch(ctx); err != nil { return err } - return fn(p.reader.Symbols(), p.samples) + symbols := p.reader.Symbols() + symbols.SetSymbolizer(r.symbolizer) + return fn(symbols, p.samples) })) } return g.Wait() @@ -295,3 +306,18 @@ func (r *Symbols) Tree( ) (*model.Tree, error) { return buildTree(ctx, r, appender, maxNodes) } + +func (r *Symbols) SetSymbolizer(sym *symbolizer.Symbolizer) { + r.Symbolizer = sym +} + +func (r *Symbols) needsDebuginfodSymbolization(loc *schemav1.InMemoryLocation, mapping *schemav1.InMemoryMapping) bool { + if r.Symbolizer == nil { + return false + } + if len(loc.Line) == 0 { + // Must have mapping with build ID + return mapping != nil && mapping.BuildId != 0 + } + return false +} diff --git a/pkg/phlaredb/symdb/resolver_tree.go b/pkg/phlaredb/symdb/resolver_tree.go index 5397e1761b..dd75ec8184 100644 --- a/pkg/phlaredb/symdb/resolver_tree.go +++ b/pkg/phlaredb/symdb/resolver_tree.go @@ -4,8 +4,10 @@ import ( "context" "sync" + pprof "github.com/google/pprof/profile" "golang.org/x/sync/errgroup" + "github.com/grafana/pyroscope/pkg/experiment/symbolizer" "github.com/grafana/pyroscope/pkg/iter" "github.com/grafana/pyroscope/pkg/model" schemav1 "github.com/grafana/pyroscope/pkg/phlaredb/schemas/v1" @@ -19,6 +21,13 @@ func buildTree( appender *SampleAppender, maxNodes int64, ) (*model.Tree, error) { + // Try debuginfod symbolization first + if symbols != nil && symbols.Symbolizer != nil { + if err := symbolizeLocations(ctx, symbols); err != nil { + // TODO: Log error but continue? partial symbolization is better than none + } + } + // If the number of samples is large (> 128K) and the StacktraceResolver // implements the range iterator, we will be building the tree based on // the parent pointer tree of the partition (a copy of). The only exception @@ -239,3 +248,85 @@ func minValue(nodes []Node, maxNodes int64) int64 { } return h[0] } + +func symbolizeLocations(ctx context.Context, symbols *Symbols) error { + type locToSymbolize struct { + idx int32 + loc *schemav1.InMemoryLocation + mapping *schemav1.InMemoryMapping + } + locsByBuildId := make(map[string][]locToSymbolize) + + // Find all locations needing symbolization + for i, loc := range symbols.Locations { + if mapping := &symbols.Mappings[loc.MappingId]; symbols.needsDebuginfodSymbolization(&loc, mapping) { + buildIDStr := symbols.Strings[mapping.BuildId] + locsByBuildId[buildIDStr] = append(locsByBuildId[buildIDStr], locToSymbolize{ + idx: int32(i), + loc: &loc, + mapping: mapping, + }) + } + } + + for buildID, locs := range locsByBuildId { + req := symbolizer.Request{ + BuildID: buildID, + Mappings: []symbolizer.RequestMapping{{ + Locations: make([]*symbolizer.Location, len(locs)), + }}, + } + + for i, loc := range locs { + req.Mappings[0].Locations[i] = &symbolizer.Location{ + Address: loc.loc.Address, + Mapping: &pprof.Mapping{ + Start: loc.mapping.MemoryStart, + Limit: loc.mapping.MemoryLimit, + Offset: loc.mapping.FileOffset, + BuildID: buildID, + }, + } + } + + if err := symbols.Symbolizer.Symbolize(ctx, req); err != nil { + // TODO: log/process errors but continue with other build IDs + continue + } + + // Store symbolization results back + for i, symLoc := range req.Mappings[0].Locations { + if len(symLoc.Lines) > 0 { + // Get the original location we're updating + locIdx := locs[i].idx + + // Clear the existing lines for the location + symbols.Locations[locIdx].Line = nil + + for _, line := range symLoc.Lines { + // Create string entries first + nameIdx := uint32(len(symbols.Strings)) + symbols.Strings = append(symbols.Strings, line.Function.Name) + + filenameIdx := uint32(len(symbols.Strings)) + symbols.Strings = append(symbols.Strings, line.Function.Filename) + + // Create function entry + funcId := uint32(len(symbols.Functions)) + symbols.Functions = append(symbols.Functions, schemav1.InMemoryFunction{ + Id: uint64(funcId), + Name: nameIdx, + Filename: filenameIdx, + StartLine: uint32(line.Function.StartLine), + }) + + symbols.Locations[locIdx].Line = append(symbols.Locations[locIdx].Line, schemav1.InMemoryLine{ + FunctionId: funcId, + Line: int32(line.Line), + }) + } + } + } + } + return nil +} diff --git a/pkg/phlaredb/symdb/resolver_tree_test.go b/pkg/phlaredb/symdb/resolver_tree_test.go index 16d5e17af5..afc4f8fd26 100644 --- a/pkg/phlaredb/symdb/resolver_tree_test.go +++ b/pkg/phlaredb/symdb/resolver_tree_test.go @@ -2,12 +2,14 @@ package symdb import ( "context" + "fmt" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" profilev1 "github.com/grafana/pyroscope/api/gen/proto/go/google/v1" + "github.com/grafana/pyroscope/pkg/experiment/symbolizer" v1 "github.com/grafana/pyroscope/pkg/phlaredb/schemas/v1" ) @@ -167,3 +169,198 @@ func Test_buildTreeFromParentPointerTrees(t *testing.T) { require.Equal(t, expectedTree, resolved.String()) } + +func Test_buildTree_Symbolization(t *testing.T) { + t.Run("no symbolizer configured", func(t *testing.T) { + stringTable := []string{ + "", + "test", + "hex_address", + } + + p := &profilev1.Profile{ + Sample: []*profilev1.Sample{ + {LocationId: []uint64{1}, Value: []int64{100}}, + }, + Location: []*profilev1.Location{ + { + Id: 1, + MappingId: 1, + Address: 0x1234, + Line: []*profilev1.Line{ + { + FunctionId: 1, + Line: 0, + }, + }, + }, + }, + Function: []*profilev1.Function{ + { + Id: 1, + Name: 2, + SystemName: 2, + Filename: 1, + }, + }, + Mapping: []*profilev1.Mapping{ + { + Id: 1, + MemoryStart: 0x1000, + MemoryLimit: 0x2000, + BuildId: int64(1), + Filename: 1, + }, + }, + StringTable: stringTable, + SampleType: []*profilev1.ValueType{ + {Type: 1, Unit: 1}, + }, + } + + s := newMemSuite(t, nil) // Start with empty suite + const partition = 0 + indexed := s.db.WriteProfileSymbols(partition, p) + + // Get symbols through PartitionWriter + partitionWriter := s.db.PartitionWriter(partition) + symbols := partitionWriter.Symbols() + symbols.Symbolizer = nil + + appender := NewSampleAppender() + appender.AppendMany(indexed[partition].Samples.StacktraceIDs, indexed[partition].Samples.Values) + + tree, err := buildTree(context.Background(), symbols, appender, 0) + require.NoError(t, err) + + require.NotEmpty(t, tree.String()) + require.Contains(t, tree.String(), "hex_address") + require.Equal(t, uint64(4660), symbols.Locations[0].Address) + require.Len(t, symbols.Locations[0].Line, 1) + }) + + t.Run("with symbolizer configured", func(t *testing.T) { + stringTable := []string{ + "", + "test", + "hex_address", + } + + p := &profilev1.Profile{ + Sample: []*profilev1.Sample{ + {LocationId: []uint64{1}, Value: []int64{100}}, + }, + Location: []*profilev1.Location{{ + Id: 1, + MappingId: 1, + Address: 0x3c5a, + // No Line info - this is what should get symbolized + }, + }, + Mapping: []*profilev1.Mapping{ + { + Id: 1, + MemoryStart: 0x1000, + MemoryLimit: 0x2000, + BuildId: int64(1), + Filename: 1, + }, + }, + StringTable: stringTable, + SampleType: []*profilev1.ValueType{ + {Type: 1, Unit: 1}, + }, + } + + s := newMemSuite(t, nil) + const partition = 0 + indexed := s.db.WriteProfileSymbols(partition, p) + + partitionWriter := s.db.PartitionWriter(partition) + symbols := partitionWriter.Symbols() + + mockClient := &mockDebuginfodClient{ + fetchFunc: func(buildID string) (string, error) { + return "testdata/unsymbolized.debug", nil + }, + } + sym := symbolizer.NewSymbolizer(mockClient) + symbols.SetSymbolizer(sym) + + appender := NewSampleAppender() + appender.AppendMany(indexed[partition].Samples.StacktraceIDs, indexed[partition].Samples.Values) + + tree, err := buildTree(context.Background(), symbols, appender, 0) + require.NoError(t, err) + + require.NotEmpty(t, tree.String()) + require.Contains(t, tree.String(), "fprintf") + require.NotEmpty(t, symbols.Locations[0].Line) + }) + + t.Run("with symbolizer configured", func(t *testing.T) { + stringTable := []string{ + "", + "test", + "hex_address", + } + + p := &profilev1.Profile{ + Sample: []*profilev1.Sample{ + {LocationId: []uint64{1}, Value: []int64{100}}, + }, + Location: []*profilev1.Location{{ + Id: 1, + MappingId: 1, + Address: 0x3c5a, + // No Line info - this is what should get symbolized + }, + }, + Mapping: []*profilev1.Mapping{ + { + Id: 1, + MemoryStart: 0x1000, + MemoryLimit: 0x2000, + BuildId: int64(1), + Filename: 1, + }, + }, + StringTable: stringTable, + SampleType: []*profilev1.ValueType{ + {Type: 1, Unit: 1}, + }, + } + + s := newMemSuite(t, nil) + const partition = 0 + indexed := s.db.WriteProfileSymbols(partition, p) + + partitionWriter := s.db.PartitionWriter(partition) + symbols := partitionWriter.Symbols() + + mockClient := &mockDebuginfodClient{ + fetchFunc: func(buildID string) (string, error) { + return "", fmt.Errorf("symbolization failed") + }, + } + sym := symbolizer.NewSymbolizer(mockClient) + symbols.SetSymbolizer(sym) + + appender := NewSampleAppender() + appender.AppendMany(indexed[partition].Samples.StacktraceIDs, indexed[partition].Samples.Values) + + _, err := buildTree(context.Background(), symbols, appender, 0) + require.NoError(t, err) + }) +} + +type mockDebuginfodClient struct { + fetchFunc func(buildID string) (string, error) +} + +func (m *mockDebuginfodClient) FetchDebuginfo(buildID string) (string, error) { + if m.fetchFunc != nil { + return m.fetchFunc(buildID) + } + return "", nil +} diff --git a/pkg/phlaredb/symdb/symdb.go b/pkg/phlaredb/symdb/symdb.go index def005ebf0..5a8aef7cd7 100644 --- a/pkg/phlaredb/symdb/symdb.go +++ b/pkg/phlaredb/symdb/symdb.go @@ -8,6 +8,7 @@ import ( "time" profilev1 "github.com/grafana/pyroscope/api/gen/proto/go/google/v1" + "github.com/grafana/pyroscope/pkg/experiment/symbolizer" "github.com/grafana/pyroscope/pkg/iter" "github.com/grafana/pyroscope/pkg/phlaredb/block" schemav1 "github.com/grafana/pyroscope/pkg/phlaredb/schemas/v1" @@ -30,6 +31,8 @@ type Symbols struct { Mappings []schemav1.InMemoryMapping Functions []schemav1.InMemoryFunction Strings []string + + Symbolizer *symbolizer.Symbolizer } type PartitionStats struct { diff --git a/pkg/phlaredb/symdb/testdata/unsymbolized.debug b/pkg/phlaredb/symdb/testdata/unsymbolized.debug new file mode 100644 index 0000000000000000000000000000000000000000..61cdfba1aa139b60addb30680b100881ea6987d0 GIT binary patch literal 27624 zcmbTc1CV7)*EU$TyNoW|Mt9k^ty8vLUAAr8)n#|twr$(f{oTGX-}}Z){1cNW_I}pN z%qMf@O6e38!}m9V!r-s)8~A`YfZ%_>$$`j#Ab#s$ z{}cC5I^RF=JN2Ki1oEE<^*0{$H~!&a{t^Bk@Sj*9Al$#gzZU+%KfM39Q1Q1sK7_wH zkp96xJn;Wb`Wp}QFaAwSr2Q-WFW!ID$M-k=U;L|nNBk@NFWw*D_p&LfOE{+R=%Ck%^9pj)9hu#p!=^M#{f2 z|HJyHj6WjZf9b9n_4;~xdImti2n?xu@I{z&1b%P`L`WL)k-*eIK*DCgh5y6&*KT!* ze+u|lcg={*fc8XHUg$3`n?1fD$t)NacwiwRgeE*DaZDpY!p8PrIwJ^!t+OG5AqqPi z$$^7{Kn4Q~(8t6Oas&gm(}zHkM@GfaLgJ5=7m77Dn!j9gchYsr3HJP`7;&qzkUQ&p2UR9-gIH`GP5d~K=vGO`zGilj`6}S|*Lz|!@>BSzy z8qx5%ahGnb>cI+iI@Xo(j3cbnEbAaK$Oe9eH=GZU!S@~)b|ZH#DOD7gCYEt3w5wi_ z4+=GAtPi+8dp~?ekS>Iu%C8(W$8ppP1uyWU7zM^C)U2Y3MT(1{by1~l1kNZ8IYq#u z)XPkwWeSDBopRm0kf(}4pD=cQTR#}CaK;7}2DYnR%BIIhXeKR|n9A-h7*9NtyhQMT z5JsJtn#kP~cMj?1S9U(b2`y+h+=AdVCKT&5ik^~FfA9|TeS+6FIz(2S8x9!)7w(HT zgw4!&a|4*o*xHBzP4Oo|>P{E%9AXhifRj2C+C(Mh%AnQk#B&GMx``dPf*`Nha)qCh zuK5S*9Yqn?&>pq>Bo#%im?h4k2G3=5Crq+A?I_B|OL)npxvTjMlX(=iDV~|~OZv(Q z7>v*Ec~is+@@pxcO*wLr`QK3DZ3tizIo=c*o<;JmNc%LEFvA6Wg7fZr_B@aRxk5xs zu8?3pgO;QMyYgai3n`N2O46eJjJ?-cHWflJLpVQAf;;xWyf`2_fGGbz#Ezt$L z{0(>BDOve)tsJ?bEOxqOGhoSYn3?7NWa8cX+rW_$f_$O9|4UVlH^|VS)I0bNdENw^ zpfpTM@R(6Q#~8bttFmL!xT^Mr7@mc6w~jbNO)AoH%J)r=ssVQr^kB|RSEotbQi(*E zCnI2bi1Lu_1|&Qv?`0^K38`}~X4MMudW`KDos^-D&ucL|C3n1mMrVYIBB~WOUUuH7 zVRFTN#$lXr4GYRe_cToVaJ9>phc%)?FUE_v)CnAc-s$%;3x2I;|(+2%x+PYZ0Sy{LI#{-#A1LLVx97xmwaJWa6qS1hG_+QAlr`-h|-0@ z5Tx(D4|8?rq+^WS0hB`*!~@9H|Dd_23e<$~6B(i^K#jkG z*r=J*l<3xxuVu|7hpaL94Q9d~0lLCf0#qD#tMCZZo3%>Dr5npm& z2VVbEf&)^ALpR|fDbbPhG{Y(}_0SNsj5@3x)*JhrZQAbyO)<*YC4=Di+I$=OCrwD3 zNcT#Z<`Tg*n$67tsw$9eE@n||(_Bv^99;}c8<_sYWRt6;@GJXB%4IZisxam-1jCW= z=9EIv@`ypXjm^L4oSrGm^jq^HR9jG#*I5oZf7n23XbwCEHiusMk5mmTwt1coBoDht zUX1#!a*G^B0Rb=_iUp-eFU}a&=mmmuDc4A|l9#Ny7tAM2z#9<~+#f zEFVpV*U6i8VJ-lAI))cKA&5BjiGeyY&*+rBti`jX#h&3IHXQr7eLj@^J_8)vJ7Kkw zu`w1Pq%v@%d5CV8Z05R9S{Mp#YbDzFm=~!2=xoAWeCKA7&$qH##jerAG2j5PHB@2U zjFgnDL`E4iiR3d(P(w}IGj!$!eJtWtjc6*iJT|Zg4!456TwueU_n6{O~;mG2j@mH4l?HBQ+Pi}c;vLK?digC0>3stSZOg*9NmH1dBve}1SxD0 zj6cj;vIAIu&bh>04A!2cGdxe^)}R=$p|NN|X8Usp22!BH*e)lenB#s?KW?vF1dh*b z;Tj0nlmeS7ENbo#P>UL#C4E0RFLQAPTx~B4>u9Z0G*g4Sa5J2yw80vX+XV-2PcTlB zEtJ9_K>2@{=lTj#C+(1dvNs9*k}%vQkU?-$CRgWurPblDDqdZo@Z%x58-t2&Nt?^Q_qc!dBGouT_5dS_rZImF7!1ZZ+ zB$Gs#Gdz%2Y6@~p3qkrCgPZRt2yJ8;>v;+~i<|n3rJTCxYXjXRg*aM>LX37EiENh1 zhDxOhKO)8FYQ5#mF+G@t@P9=yfkeL6#;dMJ@8ei^C0Vm~SLU?@!O-AU3u1kPH-OQ{ z%k^h(t8{Ufd%BCE6}`j*X;<^bhcuz{&kj@*giLeMPlNJixiTPvw=tEuRL{St?RGA! z&x7T~%6)nc2vR)36x>PRnyz?ns<_V@Mz{CwjuTQ_Q+axJ72fj$N$pyz;5J`yW)W?2 zvv<6ud#tI|3=OLcqTRTJ!Xvkld)G~Gs6E(9B7tPjZ}>?4Re2AsL9OHDnO)GdKxR22 zVqMvR+0 z0Fx};C`5EV%Nt8Iy%HOX!*J!>XEZ2cQgd+8=F2Ey?0 zP#_sO?LqJw4n>wo*A=koAHL`H*bH1hA`xQNw0f-K%N$;TX46~p2}O`yTrn@u#?38$ z39CF;DldNVg29uXe(~|DkG5xc2YRUOv%sKg17@t&0O}}L6=$9CtXlHF@ z&2W8^WHLpT!7Gu#_;5=AaAg z8BZ4~W1bkya&00lGJmGBOj7xzoPCY}Ay5)f2N$yTK0U z=AvgBWT$(wu0IzE`#q({pA!+Af7B3bXFvEe4%#w-?1z~doToHMdJ*FlH^hr(;qX}% zz!^W)O9b#3zx*II@j;*sz!y(m<}Gw^b|W*%mN-0%VoDFDW|<-oG*H4f6I-jJn}`{I z$izxdpfqG=GPXRF!Wu0$@2g@qWKXHxOs(}C5waYl!CGaN9Eci@tM#lBIO`^>d$6u| zZc=|uRo7fmTCgVzZ zA}@HYuZp)rN+NyLexk<(1NvxFJn3o;dewXZRrbl+DV-1Es1C@za9urqU=zVc_gv7K zt>v^2%W3+rSrfGfXb&#vYi73>3rtzWA)Tp3eJtwqHAR;~ey?J>(F0#RQ~&j=FQZ6ifK;A>}{*=M~gRkus?eqkdJm> z6TieP!)|H<4U{);;w$V;JJ(elywb57077&wY$R9qeu=!xK0ET9HY)EW5jJ*&^DoK_ z52Vbc{V&R&O7xJJ!LwcKYcl84GUZy2`zM+Tu)BD!k}v#gBAPeluW922%c@wxUm(jA zNGt3s(T=7GA`R}OLa&o~p;G>( zp<2k-1wEr8;saBwn+84fcn+!$$hXKE;t2D-v5a4UnK=wO`(}-bj<65O*2jo?rDDGe)`scs zagj`&@@b47Wg@K>rpnxavEl;`+O(eC2a2JVCb7r~qDCWxnGDgGlD-9pM~p%>4`GSB znZinCox?BdS}$$Vo!3jKU1$+bxe@H|VI5(IEVx-7FuLXWfCI9evXHT~k*89bf6opw zX3Z)u_xZC)Y(V2sJ#~42Nv1G3m-~0IT}*BagD95UNS0gr6u188m?5<&-(pTf;=Xbt zvX_-Zsqv36eKL(KCUw&Goi&=NZ~QA0NB&5<;>;NPSZ~BBZmVdTB_^DvGmV$kDoj*LR`%xqsO^q7P70p+)5vGV@D3mMapdF-if zIfTrb@XUx}%oQk&T)gN!C53 zPOvzD8tIwi1IAG#LC44uH_~M~?XDqn8Cr0LMg9;+h;KE=E=`|Vhp{Jo%JL-bZal4F zNx*wmHuy?fbY|A%;w94YbDePk7RK0_6GT{FKPN~SM?QL2B?D<3EE_>#OjEMaV#AV4rTF^GL}iaT4KEI4x4P|EwqVbS|~cS-_>B`LFzIWaGF= z0QX;GHWRPH1EydRI}xc!%tKO^%Af#m=6}Y%&4d$32T}oegbrk_N}8E^KLFy_m~e5S zkX8|a9<_-lOl|fUoaxY*aPJt9S&;IYut@{o>}m8RrWe*u9!tG;8Wt?`ptciHZ)kbR zkwB-^kD;DD&gGcY-JsO?8MyMN-70MmZD0jS5tqOttswxma(!yB-a!iUEHRxTk}Zx& zL@=f(;BEeQ0|5Utz;FTw+?`B}8M6a-$Bb!LJA(w|6>`8sfNt~^%eB8v2Z9C$9+L=k zBNjMa05oXS*Pm&N&$L?!%T@#;0}(!j6f`fMlc17?L@x@?lnPWM7=k7SSea?-VaivS zDJK{%1qHiC0Rk==m|6f-#lhF73YmdqYu!{Vdq02X)2Ui9*aOeIz+W?!X!n}0vj0^M zy?R&&J!Rc7r2&bD+kV%hNYVnbYjTYDHnK{ ziGV8|lm`x8ED>1RxVt(RSe+?H0-`TD7>s2b$mi2?dFKoKvetFNt-p+Squ+wn!GRd> zCQ;sk4#lkl4Cm+9)X7I1)MG|sZrm0j5s0x%50msusZWM3_!FazIXb3&!vpDa_M)}p z#M76R(bKxdJtu7z8=q$;1pCy4ja{@g%f%)*vql`F`W8=naBv=-9)=c>PB5af$%2=Z z*6#^i3k8i#rIjf!LE=`%3Yw{NDpK5d5|!~@eC2z-LN;RE3Rp2zpj5y8Gvcd&Wvgh~ zZEp5fBw^<1NQ#4hX}tTrwf9M1@n#5uvq|FMvm@-5edO&NK$3m;tXV@mz!;6g4F`fg zAFEDyfjE7=a8v3|-sg-Bj;pU@_es!&wTOSnu@~;3(vrhNRgw)-mtFdM4g1o-0e8>K zFHf#pbrk6=+4tsfhb42WMWX}W>+q)R#($kKsVYEh<^b0eIPj%hcP~7@sqzlkV#aMd z58uz)VuBdAk3%YJFQZ&r>v|_dP^mPJkM=gS2R;T%N*m#tg-pM|j_2?^FqYMP#x=GM z0oj7n__t9^3L2XsJ(!8TDhU49?P)(n8F++8ya5@$95NB4`fNoTyG@7BgD{eNY_%gG z?d^%p{>2AIt_sKlH$?8(C=eet&-`>{C!=(%b9zFe=oPL_Ek`Ox*CqgbUw@RhOS>@N3)dgn#t!2 z8WVS{@e}MKGGQ5mYr^euOs~k7nSXyoje$Egx{qg2SW*;RwK6ES94Nbrn`gxGRgVZrR{MP>2 z5e>IBIwMF9Y$;Ehg-77S0D26%Ee^JZ3gPMz?A{$sT0`%PqqL|CW=nT-$Y2ZKXAVL& zj(sGI<0@)fK=s>@O_(Q<8Xe=OuOW}nS?CZts?``+SzhByw@|J+fy`PK*I*zC)tDaW zJ1qK_Q2re|?QX#=qf`iUv3};XBBGQB!H7u5{gy^)EKI;6=;fuKD3tvn6gvqTAqXko zPqEG0``?sDA>+PD+Tt`IM}PG_9$%YA$`5Jdg?kKo@px8+*1|2#-hzbPGTM_h+KNY# zMT!abzoIPyrM_)>!u72#pQr?Ra@qTg{9(h_F4L{A?|a|)=CpB>8rP<7YiRCx0omO? zxOPbxh4fgbC{OxL}4ywDDuf4X#JI?PSbj04V& zmokUqzlxF<(fk;=oz^zgHr$M)hPtHPyZP~H&>2(BG!Uf`o-L|AXU%NE%VJUQ(6XRu zFjNf6X?5&n-hf&4&f0=|er`FG_~XIK%&cJtpD_7lGlAyw6<{Tjr6nl;wOSw0!K!pP z%<4F@%UW^33YR4L9uBumyrpYB!~^`v)&lW42(Xl4Rv#q?nv-GP=S>%1N&=ufG^9S1 zV~xIJ*ei8PwXy3K(T*jXC69UFUP#Auq>uj8{QXdlb4okLE7tLTLay%-&tEs_UXlUd zg*mTDTL&IVu9MK-r5|4V$ffSWeav+O+2D9n(> zCxOST}w}YPI*DhwXK)HXmr|4 z@@9U@1-(z4@)$oHV&9$~6pv+j{FGa;XWl_CKWpW4+Il(Rd!!*FNDvxd2 z;i}x-+?+EYP;-Z`n_`2U*sXw zax@e-a5Bx49&=YB5{YT2VgLLPFfi7TJ%!WwNmWkLZ_OQmd%tpotsY|&?B&=_+Z~8kw|kkL}5HF$3SJz z2z=v~*99tr-?SHbv+Q)&#x#XZ>nrr0Q@F)#%`3}H>uOg>%64Z!PL`c05uoLGP`rr) za-l^zWEJ}pwRR~Je6y)ID@^TJCFez-d$Q%&gx-Y%P)6T&8I=7Y@6a%QK~Z7vOWQoI z9bCr$P>=i#)q;0Ucu4`ntad$zMz|LI&cC1pkvI?CY#4Tq7}*>=>{1Bz4cJ9qlo)2F zHmu|evf2Y=5-G((@0@Zg+4nM$4rFBJ(KK{K8N3g=9*Kj=x;*CbypZ^%lA#ij-KlG? zFXr|0*0jMjPGOtYk?w>#Mn=-TNvb!zC`#S`sOWhvEDc+abzj%9+-Ej?@p9U z?1|KINJT=+L<`9nv7}L&=V5<}_7NrQvS}cf3ITdW=PHv@ws?ssDNW#Qr`uTAKfs!V z2Ifn{L_Rab9Sa+&nXJi5s$(j>%oT=BV(myJ_T)6wO8O_1qJ>eWZsHRg*LKEU(1;G~ z?vbrpX)lnahd*)7gt|b#sh?Kdc@mQ~_n~80<{DKFvJEEngXz}wkM}I}Yb(fe`m?!9 zNnO!uu4US0-+%S9^un>Jy%1DbLobg*1DDXceENpCyo0cuP@+fjp2T{hS+P|>Yh8!( zjE?5geU#0HIld`eiEJF&rTEQX$f9bWvaPcRvSgXP)Z?1`2}ODP6J28FP)*nKF=PCp zWOKek(3ESiE@kVhVN0BHB*1}HxE}wfA`Yt?@bqd>}J|-&kB>{AH78UfIJ396h@)i{n$afRsuoq53 z?PcXDdI#`o=Vr9n7LIItFb^=R0U&@O*+fECa;OpgmE8eD$bt#=6Bts=f=ws@#!lgLE_CPVHi?5qqzG5LtdD~i<#DKb@i zb?Qg?8$J!FoWg(o?|$}Q_X}Jzj1`vcXZeuMYU>D6)EbRZ)ky}RP{_5IAo%GFAq55Q zBobktLOp?$yNR>GvIo!f@ecIM0Ukvj0QY1C)aig9Dk9;1uoF*OnMLQlMoDsDu2>QY ztWcJeP%$sqlm$ZP-KaGk8)e5(`5_fk=eSwb>Q!U-pw0CVnRDw19QRIrg)o|Em9<`Q znHFss+l4IdPFwzI+dsgVV?;dpzoGAK_=-L zVv9e`K5gs~pG!m``LjEwFG_Ggqfe_lyuK)%yowUFkV`;Hz3^PbV=$=l;ITb6SmWLY z7W$J1RrwZIRXj*^GvBU4bUSdmZlhoSuoRoUFnMWH;)eOAtN1dYnQ3;TV zc!N`Tlh5Xyvg3*f`$CQ?@?50ctt49 zqxra8WVA7N(|j8nqa%=x4XX@|nz@T6lbI{Bnvr?#Wn)FCHLHnBY(&bJ5pr=hv2!j5 zE5E2B$YEHn5|bA>Q|aNNQc^Tl3#XV)q$pgVGc`7Jto_Mv`rVZAKwQ0PWTu?4w$)OEu;o!^oR$+3xoLyJQFcjD|+8I*DE%;{($N#SE7A~bcD z6m0r|*{OfUv6!maUE!*T8T4R|fMx4esNM%Om9fJq%bF`2#+RNj1~KXReDP%9g+PMb zL8KozP6a>XLh@@eCRoEY?dFSRv{lv`?5+u{%fTw=<3jwI?P zPAX-o1Wz%>hMjG*qBUseGrWQr=)H$e2pn@ z2Ui##o~}uR+k=D{)ugP0EOTo4Q^}#utg=AOZ{e(nI=0ta!oB_-&yAue;EujoPsOb94u|dLS-9n)u)Kj2Za7o+w)nCU4ZztB0W=Q$4o*6@tN5&G2^2ZWU zHjN~}j`wG=GDLgQ`Nwnn;;QIE;*Il?nOw+a+&Gy1N^8!$2~~dB;=1i5s>GfLf(J;1-iYOZT{EZkZthW&?7=1_pSEt||XpPXSq zaaT6TK)n7*a$zkyY8;7m;F>gPRb>M>q+(xL&_A=`HLpNL2El?WZ$KkjQ zmWPAJi-$r(?fSh1pQs61WMfN1)#lhW*hX>?Uhp9z|S^8=5)Pg=NbkcmpVz|ZMiz>POr6{mM$)XbP1a_esaj#~TVZ~1X+vu+Lh9-y z_g2GawIVoSuEIy!4G$f#fAh3tr(zW*Ei_fr@GDm*k=A$Q%}+d^^<;|dSq=Gi*{tOO znEJI!>w+IzPVj40!w#d$P+XYQ`aoID`0PE_%vJ#FEl>YS2Ok2XT=3ae56&TblZOWJ z8+2LN`b;B$ksulp+AaWdmi1k8-Mj$hAxfUO>AAm>DH*4q5fMYrK?dSVC}E5Jp@3=U#yvRuo^98-H{!79;A)f(fp@pkEIapjxs6hL-P0ghYF(-D?D1ntEpmlkwUeqx?Kv!qS~oG(;6vLS!@SY* zMkgW~%k=vhM&yZH|3Y|u$FJJAWG#i$nC*8N{4$Xt%AQARR<^pfwRZW9OZIlRfh7&4 z(y^SRNN>{A`jf!+_UxQ|ST@29k9u|~^ZH+2-iKViPP4fJjo0u-9IQ}m@#EYi`HJR>9giVGD0Yhj%9M0=pxHii+UsyGHn!G_b(zA&N4opr_ z8@;nR70oL(eJSC1nL*B)@Dy8r39%-0Q&ZiKgwZ8la}&$GCMLy6eCEq&zIM!hmgt{$ z2i)GX9jl=KHfYi z+w)d2rdAMyKFms|;R~z4fG9iE3`s43@;uQ?`=MZV2IuDSAm+*Ya`k285fcwelZnbh zW}k&*sYzsg^ZN6kuJQ$fjU~@xq*YL76$Urx76}6jt)`eX$rX;tnnuRL^U{10qLXbj z^_PBLxV5ohuCo~mRcgeIb$B-afPdWOZ3b_dK)M1k{<9;%JFJ)1tN~{-?+TWByP_VeI<0 z&R&cbAtF^0_6uTfIqgch4_8N-Sal2yeV>lCOKHq49vYLpDq;3*RSZrC^`2e60MFj7 zYlw9OWaI^HX5r(I|~}SDg@>E z=m_@!91;BtJRzjfnf7?8@xJUx@wI&NYtlz?A*Ozeh0<@|#DoRWCLlUhdzW(}%uCMM zu%8l@S#X|QhN2@d)R={S$DhPP3C*h#yd)_uZBz*Sq!a;srQanG=> zxNx~L#oCyA^vn}JFFMS;k=r^6HE;61VKj9xkh|k{`-<%a=QI4&35bqcA36DDAT-Gc z(PxoFOz^RVhg7ZZEc8}8Ya#~3OUR$Dh*GE$`gr{NxU7v|dN;N4Wht>~g?ujgr>B?9 zy7`xHw0Z2qB&pg8%&v9gbP|ugX%#+A!+sU(-_IN!(wowQ;n|yBofJ>qHcL(Fo3b9^ zxrMvjr+N_`IrnhJJfRG`KC}C_m+B4VReB-Bi!#63QRE zHcGETV_~>?6#4VHPlVbgan9vI&z%%A<`reQE!yuhnC>d>%Zuf&>e18h*q%=jjt@B>f^X68RH%Xb!p2f6&>qi}YmKu7>v6KZci3 z8P!kw_Lyy?7O+z?MqURCCI|YSD7#5EP2YU-k89ZNSK;8zjz4zPZdfj{@AUHIn_4>xc&*`_ zL_*BrzXZCo=??oWQ->sDH}#$`v^D?UT4(5FY`^qz{}#stY2@{KtZ zL5#)%aFX;IHjFG5r`20f@>_z%0D?tT@s(MH6d!Y+#V$&p0P}~N-kV;XpZBN9vTZKG z?7{3dF0T#urk4O5t7~jYI`@`upPp@99n0s3*N?VKAKFr{307LSqu(sRTB;xMujj0@ zCCydXvv)F23H9pP76wnfsa{tk!hoEjR(V)*qo)}dY*y*Lp)s@`{gY-ML3 z*wCKEt|4x9l7qbm-Kft?F-tO_&o|Ot3#+$m`R-KLn&j?R!wYbj{UxRs+OjzXyeVcTU z;2Aa9EbKAr3pV6WnSm2U#s0=GkiPenIjyJ>V?S4l4r0gky$t4sZ8~reiYAh)x;oT#CsiNIKP~D4jL)A1M7}@uOju^pd)j@+|2Q@f~ zx2Ki}1|ymht*wQJ(pOO!A#&=WOIe31_pHNlFs;_awOjUsD!`59zS2SrQ-K317TJ+) zRH9J2$MpQJgL?BT0F~cy;ua_hu4aD0O$%v?@e=Z7|6{aRza;u{$1(JNY(C(X-{;kmcuoDZ^Ulv?*=wRj}3Pg?kRrDclxtl?T6tqB!GT7sSj zb1Jvf3?G6fouug`;Kq{;~f}J zmQZv?pv(QZS*l>w0pik;F)K0drj&{mo5^Mp@Y4d1!1-ienwE?YfE!zk3Dk(pQOw@$2bKp-GQ88+(K zik@pfO&j8?S*_qYi4|xA(%Fjob6haS@F=J0uu=v&q|7I&2uW$icY%Hbwi5xecdFiM z3s8=tLe?AEFR!^owK<=RK*#Kg`;)>GGPvOb;&4<@^}ps>@ZSmcf6ueP=pJ)WMmV8& zK^>Rmm;)qt3&I#5wRVj`WD1!YqqH}#Ld!1{fe!`zV-Z;byV%VFdZnSukL!|;CmxzD zRuz7bU=eo*ept0Y4V;0e0PZT)teT(YlZAoMf$Iv>Kf10 zDMUkrDqb#OJ!e5vBuN^IG)C3Qw-XfQ30)=fJz?=&{6vde1t6Z*ml0+2ROFZ??PY>R z`?n6k?&vc*@b{d>&v=Nj2Mhhw5G>n%QVwX(K`?orJzNG^U}fqG9|xBG=eY(0{eSAN z84Vg9YN)D7mnT$VMf`kxIr~`qq(l@`>lAskVTiUmz%DoABxK9)OuC(x1{v3*Wa;8a z_*kJR{g1GO7iIFypqT8r(gU#(B=|ciIc~>7@BkJ@YXP}MWI>kSXB>y<%B=HP@13b1 z`IDSoJdbOeoR4c&n)8|QCiURUxNPOgFMN!usG4ysqT(wg;)x?VlDrGDFMbC}#P8uZ zONv(yj+J1SqlMBGjbe-ZOGbCG1wD43rIEMGgKU(Y*p50Rv~g1!B$Wwgyp{?o64Ym= z)lIs_H)URhaMt+4#r)lR!5&1>OaqkEJXLIjIufp7UcfMeU2)uNl+g>X&}+ip@W4Cg zjQ!t;1~~V@I0UZID_$FmWvfzFS1+_dms_ z-;cIA_qMwUWiddMR;Z|~3Ms76k{%q9fr`V`%*tziuJ;2uVjvug(5{NSAqn0qn%7(q1FAppeI`k+i5SB=%;lLqKvl1sF^MSU_6 z^h(Q3k`q#6WO0|Py4tjQi_fFq4QxWSB6a)+YklrFYAc1DDm(Zu+78iPTJiRiT*7JP z?S+=`AlKGI%ScGYc?gK0*Y`D(JAAm4Z%nYlJ^KPk$?Kb~{GL0@hv0VhZ-wFnDl!)| z$p;&5knF8s$@*9KJTAcPR!M23NaG{=;C(YW^ZW!!CHGX`MHd3oMb0^{q)&1TAuj&1 z`A7DidqCGMyPt9yuw9r4I|i;M0qw2T8pjc?>>ysm8pNsV*vsglu|(_$#t#Fzk%B$e zxbBIo@bdKA;D?1JD`3aog>3o`AZk7m+&;;UyebbeouF<-(H0)8$n>OGw^v@Nj)a)| zfm=8-Zui6H`$6?=s6SSwq>)=iL_V=GI8P8s}DFDYkV(aMilO`Q+#(Uv>u?iiG&vl9>*A@0}Y*hIF+4V+P%wJ~*XB-%0fFGDh~OndY0&PyR(8sD%$ z5=cK>uVlP{fXSfRCT+IyOq#;g%4&&UJ>1z__U+HGLThZL{p3v9|0s0ut)AD-_b^CR zz-a30PuQq9tZz|hp6N|U+!${!+H~LCZfIxIPmQLef2C8($)iqKNuU(db2v!;rD=wx z<<1jdJ({DN98DE;8nu!K9|Fw&_~q)MPeHs@Q=O`^zudxQDUEOd`LQn?x$dQ+a)k{f zN@aV-OTtxa2xOJH5g9J^HfMqZX%?AyJy^5^NE^ST38P+c*5H+nV$JRBoHSRe2LS-% zOs7~Q*aFfqgnGtbq<6^(?h!?~Q58r^Wze>OaHLVra&Y|sa~$- ztODA2gzqj;t*9c(BGN>{!m|=K>D_!`#1f!Ot97bBqG6MtOc@@wL2WR?{^i8eGvZPK zoJ#Dpv5U~!|G3|LU+T)3d}=$OrwZDO5bo^a>W3*(22XeSuDP~UsnfZX(cNRT!NZ6A zpa?%C9!}t~ZwFDa6P))%9f(JhNf)GL7Vc8y20H!HchMTRsGKWYdKDj8-Zgpa23W!- zj9)1;cslq!3CYAV!c6;?BxEY#r*-79l%D)@l$K7+h4btC1D8JRMIaB>vf;^;8J@Is zQw^a~Ze3vYC1}*jZAc;Sf~sZNDrIph|MRX^BCS;FQY-b=D3ur~$5#b-A*@&$zt|Z^ zZI^~sOYv4pk&-Ubs}$j@9>7+4Zfi1uSEnj641Sn58@_hp-m6n~rkbT-B*$Q>x>{$> zC!kV*PLCR<)_r#tr5S4=egJ85)5y8h!G}Oif_i`BQYjeH)mA?ZTEbet3F-X9$-CR`D;Xx&XB?W zw-`g9bLn1YMfkZyyC|w@O~-hiPs^Tc)6&w{CVO?!OWVYa%(Hd(YCHPw47WlP&a-Au z`?bc6$?NC-fr5+|5&3uFmxn}VW8LRzdy1E7dm4A$hH*DY&8mz>t(GPG{gtHK@S9iR z=XWCbE{ZPQgiZR9ON{+T*fQ?|cLxYX>xm_fFD2F)p-S)b0Q0V?3CM6`4)ls7RceBX zQYU=^HbgJ1C=dz3m2nm%NPCyPAEv_=kjPu!f?jgrgA)*;g|<|w>k|vr+8Sj34gA(} zR}uMDz3H!bxxsQQ2=9_CA?Ie0GkkA^F0x-kK8HQ1WpVfBka{ZxwJ!#=9*l+KuCOxrKbv-LGZCDq zdFG!9elo)enJkquJe?^m`M~#pM{!|+56S7;~u^*wISa^p19{W}WwTddc1-*yC6SUgfBfI$m z;u5-z7=veAHtPIKu$1l!}Z-ggJryp za*<0%v5Iknur5B+K8h!+JAP=AJ8ISEw%au$Wbigcl&xZr(n}jGi_hBwq|d|uYJ2aI zba~3h>ctgX#`_eu#j45ybUSat!tB&2^`tKFO6iq(>@w1lHu^c}V-($ufj5*c=>$K> zLAZAN7MU zfTWLufiw!2XF>}kzEwxcyE$9PX0%QyqL{lq-C=k0C}(xF{^U=Rba%mUYDnc(^d%7r zeKW0RkMAQ_SenaN-`dWbceHRu5*l(NbDw}`^nr+3qP`daDHy=S8Au>r0b|hUGI>#_ zWuHdNFeLe~Eknz=1$r-o^JKhbk*%&BghOFiwPad0=QW&Gn}T3;eH$C9i+b#(F>H4; z>RnW9@8kx{67yaK>08wgTZ}Edel6V|$uQny=k9L&;Q#GY!xoM9%okS>jBAnThNxuT z11x3MJJ%HKl{A+SL#*d_+Uc7~_>c+^O@joN=``OR*P zHvMAgu7nR61ilXS>~8mEuBge<=k7U{af`PLg0S+1g=~&{@JZgWVpyT6QKhz5BERyfZWP zc&$INiu# zSTu>m$0p(uf*^WC;E4ufjHn@^l7LD;TrnYGktl*b0dZlKS3T1;&UEN=-uw4{(C08! z_pAHeTXpMpO)s39!ohss&@Xr2YyGIpYeTc~q#V0U$Ms$lzO1RfpEKL;XjA*O^dx?Z zac*AH(Y*MszTw_A3;%E_>TS4Iwb#QecuB5)_{RQ&rd*z2D=seG)L|?|h640Ecis@Z z5>9_JS1&y#GX2=C8~d8(4(#>l4KXO(zApQsh51kEFJ`nqJ-o#tUYzt|QR855WcjO< z^7N)jE+>9bj+L1enwceQQn7wzZ-aI)6-oLsy7i1NYa9N)8jsHwim@O93{2=kKs zI!>qC>6+o|M(5657+!dHY2&~bDbM<{pZXQX`F9q!{JD1V^`(u$%S~H{k|mETzwRF@ z0$8@lyyU|hoX&2Sw5GUtm&xoxM#b5l7cOeU-|iJq`)w zw+F;deKm6$;%*OFT=_ZZbrKT)eS>{xOWYF|`<%g!2`5?|KH#*PTb@b>K4TWUm$`+M z)p#PXXtk2+Mw<$RU3P5M?GJ_k(-xD$`&?` z5mZD!$~pvm$Is+8zFVM-7gW57sP8lcfhc#huweAd_my*$%@)U^RJ!=+@Jgz;PH!F z!JG9vZ`6++uDtxT$?4wGx@CjM#=qLT@nB3U2xJmu)J z<(HPa9Z$3bL44h5UmKVA>W*J)*}-c`8U8jrfK3z<&0bQ3EkAPHJENtxv0xooE&3Olbv_zo$|zhwzGZxedAk>U;V1q z^bv6=&~S{`wUYgnmxsE3>|a`%{UFz`^qcDW?UgPP(+x9U_PdPbJ@3kpJ}b|h#4kH9 z@IqZr><0ZRwXMiTbFCWl?9RlA+I!x2`eQiJe0w7r)RJs>ve3HPIAPGB_k5uAw|PTn z^b5ry-<_;TyE*gabMdU|v$dYh#?L-3IIy!Npxw7%;qOt~1n(Xk%)fXQq{)miW(hGy z-?&bnGsB=QA>L(2O2zri$XwY+E$`XO-F~eb`|iv3%})J8%f{iE0r0t$&AZI!Ht~1- z14C9Bk87SWSW*(dIpqxZ*i0s+RIBk%MGz#^@U{1 z=abtcCX&ab;kJyEz3bDej$F1am>sgpzA}mbI63j-u(HM9AK+y<F%8akAaT#zHq?#y)Az&4y~_m6Gocc7O8YL{7QQPd4LDPb+X#5?z{ap$@2y(@W)Y&TW7a+ z5Jzs-cY4X21CXPapM5;1IH{OpV||2JSz>Ct&o-gRdu>R&c<-0}r<>ERI2RS=KdaAq za(?CGK96sCX#19upqFu3|4wZ85L_a}mz}4Hsm zh9_hH9h)lpzUurA;)^wHZ^nLDVgKgNiWIqJ;U<%OnX!RJ= z^zzfoiC6AKPwCj#e4)t5B-CWugU^Di{Cck?Bc+Z~w(g6e-*`AzlO zP0nYA%^q^}1AjYj*Y+gi-kTq4>pOl$ImNXP=;yg?1$F+6MY@NX?e76V545Dx|M$n$ zQW!~p8fe!CyZ?kas}*0;C*}+w>LI3xQJ5O1_6VX5OYleGoG?m45V)U{6R4}~xT;{V zLznvS2E|Bzg8@$IAEx*FqScXAG+xKR1vG9%_Y>-G0~I}FI>HT8BXC_wQsEyUMf5&a z>Q)3f2`ZG|l?*%QRr>EEaK9|5&({C@Dt%+bi5dvw7-_s}6?5dMKSNn1j^d;7(awv} zd>GA#(R|R3z-T^bM`Lt-7|n;#e9(@-|6@LkbRGBzbW{&9L74lhcB%$24Pov_st+z` z^$-qnYLo70IRKxk3#lY47TlKw=d<8j0nQ!4ekQ=DjDUN9%|f~0z5of+Z9GQV(f{ju zr3CjUBlW)&=#PY72e=Wi53lo~mz$Kn{uE>vecu7`dKNn!EciR1|0DS|VZp6f@MQp} z`pJw#Ad5cAf-3;dg@e&5cL02fz9}+3ObZ0nbPA_N;I78u{aP--M;ccN3x0_OZ(zZn zvfx83xDm^|Td?4+05<~nca~x5alua?m3<@4e$nuL)05WcGW1ud>>DBO)MmKg_!~_K zQQ>M#`%}n*$FShZ0H@}K8>VdqK@PC!p8_~nS5+llXVGt9!PkO`9%+8NSoHOP{_GL_ zx&dseKG-xGZeUF7n6cQmX2IQ9@K0E9F~CP!PmmD4f?UXxBPfc6i%}92As8u_NYHRZ zB$0<>5>$YPM~k2krI3qcm=s51mqy5h$|E_IVzF!F;Xml z+eR^?WfHM0QlqXi=D#Kg1Qy0>O^_EAsWn)*1V;!`Ade<>)Cp3769kXX6D)FcTx@5% z1QFq6lvtpngyI5>#1K`~fDwfR*X;&M^_?Gy@*^TqAtsh+$o2>(!4y)BOmz^<9HBdY zX(X6cY1CqS9X&9$z^OValvBPmN~fI=gLHZ8N*zd%R4$_f38n@W5OOpElL^4sQU74j zAP~#YXaW}?Q3^h(OO6yvbO97qMv2gHjKHb1)bUM<82}M0TkIe*C#lpy#Rc+?ED!Vv*!DF6cqCkQ$MR9+xaAQ$8cjT$v(Tq)L_QDKZiOzL(Q zML`;SsS?5(@_KsAf;I&%!6*V72T6qRz?vupOC7+dMHgsdG7(rnX;CZ_%3&P~4_Dw@ zp#&Bf9M(YuVU>bGX1d{w&-VKUkOZHHLwPZS&-|T(IZ%-a`~2LBQ*Q+=dj2glE$4Tu^1Ec$&P^k2>hI739X zF!<}J7aZV#{FQ1^_1hIx70weZ69A^&{~`uIgXY5+!~WASlE&aWO;EQ$|KsTnGvf!x z{}qGpK2evC(D9@A6bJf;{3A35?Q5>j=<75pzqJo${^Wp0$|mG<;q@I|8o@x(Mi|XV zA%kxVFE$~c=D~u=KgHm?Gx(YbqYW_m)O#82{=@5c7zbY~=viGtK19nIe9e6#tvDGf z(j_x~Khu0onA~Yan%~bb0Oya^LTVU%&HX*i_lAme$@JgA;JdG&6>0t;gU|H;kml2A zrg^^wf0u-*4`o<(Gx+d*+JDO*V({T}56Fk}PcI9a{HKS96f^nsa}da1$yhL8zhOBJ zv{Q9Bek2SC;DCItN_zVNc@UYz;B&*&Es)RDXLJXd0ssoLg$C&ot{?LlIOM}}I?dMv zhSrDcFyl2QQw^b>2dQi^I?ACF&105s2N-II0}DUQLd_dg8Gmbo$@gO8&sB5&N&ij} zw7}#C(|lV-L)Stz1Jg{bT83({TN_yT>m1bj9ndjdGWnZX`tL`5v4i6r Date: Fri, 17 Jan 2025 16:27:52 +0100 Subject: [PATCH 4/5] Add cache for debug files --- cmd/symbolization/main.go | 2 +- pkg/experiment/query_backend/backend.go | 17 +-- pkg/experiment/symbolizer/cache.go | 91 ++++++++++++++++ pkg/experiment/symbolizer/symbolizer.go | 127 +++++++++++++++-------- pkg/phlaredb/symdb/resolver_tree.go | 16 ++- pkg/phlaredb/symdb/resolver_tree_test.go | 4 +- 6 files changed, 199 insertions(+), 58 deletions(-) create mode 100644 pkg/experiment/symbolizer/cache.go diff --git a/cmd/symbolization/main.go b/cmd/symbolization/main.go index 6675a3f3ed..487add7a0e 100644 --- a/cmd/symbolization/main.go +++ b/cmd/symbolization/main.go @@ -21,7 +21,7 @@ func main() { // Alternatively, use a local debug info file: //client := &localDebuginfodClient{debugFilePath: "/path/to/your/debug/file"} - s := symbolizer.NewSymbolizer(client) + s := symbolizer.NewSymbolizer(client, nil) ctx := context.Background() _, err := client.FetchDebuginfo(buildID) diff --git a/pkg/experiment/query_backend/backend.go b/pkg/experiment/query_backend/backend.go index 36593d4988..1f2ea83769 100644 --- a/pkg/experiment/query_backend/backend.go +++ b/pkg/experiment/query_backend/backend.go @@ -4,10 +4,10 @@ import ( "context" "flag" "fmt" - "github.com/go-kit/log" "github.com/grafana/dskit/grpcclient" "github.com/grafana/dskit/services" + "github.com/grafana/pyroscope/pkg/objstore/client" "github.com/opentracing/opentracing-go" "github.com/prometheus/client_golang/prometheus" "golang.org/x/sync/errgroup" @@ -21,13 +21,14 @@ import ( type Config struct { Address string `yaml:"address"` GRPCClientConfig grpcclient.Config `yaml:"grpc_client_config" doc:"description=Configures the gRPC client used to communicate between the query-frontends and the query-schedulers."` - DebuginfodURL string `yaml:"debuginfod_url"` + Symbolizer symbolizer.Config `yaml:"symbolizer"` + DebugStorage client.Config `yaml:"debug_storage"` } func (cfg *Config) RegisterFlags(f *flag.FlagSet) { f.StringVar(&cfg.Address, "query-backend.address", "localhost:9095", "") - f.StringVar(&cfg.DebuginfodURL, "query-backend.debuginfod-url", "https://debuginfod.elfutils.org", "URL of the debuginfod server") cfg.GRPCClientConfig.RegisterFlagsWithPrefix("query-backend.grpc-client-config", f) + cfg.Symbolizer.RegisterFlagsWithPrefix("query-backend.symbolizer", f) } func (cfg *Config) Validate() error { @@ -63,10 +64,12 @@ func New( blockReader QueryHandler, ) (*QueryBackend, error) { var sym *symbolizer.Symbolizer - if config.DebuginfodURL != "" { - sym = symbolizer.NewSymbolizer( - symbolizer.NewDebuginfodClient(config.DebuginfodURL), - ) + if config.Symbolizer.DebuginfodURL != "" { + var err error + sym, err = symbolizer.NewFromConfig(context.Background(), config.Symbolizer) + if err != nil { + return nil, fmt.Errorf("create symbolizer: %w", err) + } } q := QueryBackend{ diff --git a/pkg/experiment/symbolizer/cache.go b/pkg/experiment/symbolizer/cache.go new file mode 100644 index 0000000000..acdca631ce --- /dev/null +++ b/pkg/experiment/symbolizer/cache.go @@ -0,0 +1,91 @@ +package symbolizer + +import ( + "context" + "fmt" + "io" + "time" + + "github.com/grafana/pyroscope/pkg/objstore" +) + +// CacheConfig holds configuration for the debug info cache +type CacheConfig struct { + Enabled bool `yaml:"enabled"` + MaxAge time.Duration `yaml:"max_age"` +} + +func NewObjstoreCache(bucket objstore.Bucket, maxAge time.Duration) *ObjstoreCache { + return &ObjstoreCache{ + bucket: bucket, + maxAge: maxAge, + } +} + +// DebugInfoCache handles caching of debug info files +type DebugInfoCache interface { + Get(ctx context.Context, buildID string) (io.ReadCloser, error) + Put(ctx context.Context, buildID string, reader io.Reader) error +} + +// ObjstoreCache implements DebugInfoCache using S3 storage +type ObjstoreCache struct { + bucket objstore.Bucket + maxAge time.Duration +} + +func (c *ObjstoreCache) Get(ctx context.Context, buildID string) (io.ReadCloser, error) { + // First check if object exists to avoid unnecessary operations + reader, err := c.bucket.Get(ctx, buildID) + if err != nil { + if c.bucket.IsObjNotFoundErr(err) { + return nil, err + } + return nil, fmt.Errorf("get from cache: %w", err) + } + + // Get attributes - this should use the same HEAD request that Get used + attrs, err := c.bucket.Attributes(ctx, buildID) + if err != nil { + reader.Close() + return nil, fmt.Errorf("get cache attributes: %w", err) + } + + // Check if expired + if time.Since(attrs.LastModified) > c.maxAge { + reader.Close() + // Async deletion to not block the request + go func() { + delCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _ = c.bucket.Delete(delCtx, buildID) + }() + return nil, fmt.Errorf("cached object expired") + } + + return reader, nil +} + +func (c *ObjstoreCache) Put(ctx context.Context, buildID string, reader io.Reader) error { + if err := c.bucket.Upload(ctx, buildID, reader); err != nil { + return fmt.Errorf("upload to cache: %w", err) + } + return nil +} + +// NullCache implements DebugInfoCache but performs no caching +type NullCache struct{} + +func NewNullCache() DebugInfoCache { + return &NullCache{} +} + +func (n *NullCache) Get(ctx context.Context, buildID string) (io.ReadCloser, error) { + // Always return cache miss + return nil, fmt.Errorf("cache miss") +} + +func (n *NullCache) Put(ctx context.Context, buildID string, reader io.Reader) error { + // Do nothing + return nil +} diff --git a/pkg/experiment/symbolizer/symbolizer.go b/pkg/experiment/symbolizer/symbolizer.go index 34e748c49c..ac520441b8 100644 --- a/pkg/experiment/symbolizer/symbolizer.go +++ b/pkg/experiment/symbolizer/symbolizer.go @@ -4,7 +4,13 @@ import ( "context" "debug/dwarf" "debug/elf" + "flag" "fmt" + "io" + "os" + "time" + + objstoreclient "github.com/grafana/pyroscope/pkg/objstore/client" ) // DwarfResolver implements the liner interface @@ -37,44 +43,106 @@ func (d *DwarfResolver) Close() error { return d.file.Close() } +type Config struct { + DebuginfodURL string `yaml:"debuginfod_url"` + Cache CacheConfig `yaml:"cache"` + Storage objstoreclient.Config `yaml:"storage"` +} + type Symbolizer struct { client DebuginfodClient + cache DebugInfoCache } -func NewSymbolizer(client DebuginfodClient) *Symbolizer { +func NewSymbolizer(client DebuginfodClient, cache DebugInfoCache) *Symbolizer { + if cache == nil { + cache = NewNullCache() + } return &Symbolizer{ client: client, + cache: cache, + } +} + +func NewFromConfig(ctx context.Context, cfg Config) (*Symbolizer, error) { + client := NewDebuginfodClient(cfg.DebuginfodURL) + + // Default to no caching + var cache = NewNullCache() + + if cfg.Cache.Enabled { + if cfg.Storage.Backend == "" { + return nil, fmt.Errorf("storage configuration required when cache is enabled") + } + bucket, err := objstoreclient.NewBucket(ctx, cfg.Storage, "debuginfo") + if err != nil { + return nil, fmt.Errorf("create debug info storage: %w", err) + } + cache = NewObjstoreCache(bucket, cfg.Cache.MaxAge) } + + return &Symbolizer{ + client: client, + cache: cache, + }, nil } func (s *Symbolizer) Symbolize(ctx context.Context, req Request) error { - // Fetch debug info file - debugFilePath, err := s.client.FetchDebuginfo(req.BuildID) + debugReader, err := s.cache.Get(ctx, req.BuildID) + if err == nil { + defer debugReader.Close() + return s.symbolizeFromReader(ctx, debugReader, req) + } + + // Cache miss - fetch from debuginfod + filepath, err := s.client.FetchDebuginfo(req.BuildID) if err != nil { return fmt.Errorf("fetch debuginfo: %w", err) } - // Open ELF file - f, err := elf.Open(debugFilePath) + // Open for symbolization + f, err := os.Open(filepath) if err != nil { - return fmt.Errorf("open ELF file: %w", err) + return fmt.Errorf("open debug file: %w", err) } defer f.Close() + // Cache it for future use + if _, err := f.Seek(0, 0); err != nil { + return fmt.Errorf("seek file: %w", err) + } + if err := s.cache.Put(ctx, req.BuildID, f); err != nil { + // TODO: Log it but don't fail? + } + + // Seek back to start for symbolization + if _, err := f.Seek(0, 0); err != nil { + return fmt.Errorf("seek file: %w", err) + } + + return s.symbolizeFromReader(ctx, f, req) +} + +func (s *Symbolizer) symbolizeFromReader(ctx context.Context, r io.ReadCloser, req Request) error { + elfFile, err := elf.NewFile(io.NewSectionReader(r.(io.ReaderAt), 0, 1<<63-1)) + if err != nil { + return fmt.Errorf("create ELF file from reader: %w", err) + } + defer elfFile.Close() + // Get executable info for address normalization - ei, err := ExecutableInfoFromELF(f) + ei, err := ExecutableInfoFromELF(elfFile) if err != nil { return fmt.Errorf("executable info from ELF: %w", err) } // Create liner - liner, err := NewDwarfResolver(f) + liner, err := NewDwarfResolver(elfFile) if err != nil { return fmt.Errorf("create liner: %w", err) } //defer liner.Close() - // Process each mapping's locations for _, mapping := range req.Mappings { for _, loc := range mapping.Locations { addr, err := MapRuntimeAddress(loc.Address, ei, Mapping{ @@ -92,7 +160,6 @@ func (s *Symbolizer) Symbolize(ctx context.Context, req Request) error { continue // Skip errors for individual addresses } - // Update the location directly loc.Lines = lines } } @@ -100,40 +167,10 @@ func (s *Symbolizer) Symbolize(ctx context.Context, req Request) error { return nil } -func (s *Symbolizer) SymbolizeAll(ctx context.Context, buildID string) error { - // Reuse the existing debuginfo file - debugFilePath, err := s.client.FetchDebuginfo(buildID) - if err != nil { - return fmt.Errorf("fetch debuginfo: %w", err) - } +func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) { + f.StringVar(&cfg.DebuginfodURL, prefix+".debuginfod-url", "https://debuginfod.elfutils.org", "URL of the debuginfod server") - f, err := elf.Open(debugFilePath) - if err != nil { - return fmt.Errorf("open ELF file: %w", err) - } - defer f.Close() - - debugData, err := f.DWARF() - if err != nil { - return fmt.Errorf("get DWARF data: %w", err) - } - - debugInfo := NewDWARFInfo(debugData) - allSymbols := debugInfo.SymbolizeAllAddresses() - - fmt.Println("\nSymbolizing all addresses in DWARF file:") - fmt.Println("----------------------------------------") - - for addr, lines := range allSymbols { - fmt.Printf("\nAddress: 0x%x\n", addr) - for _, line := range lines { - fmt.Printf(" Function: %s\n", line.Function.Name) - fmt.Printf(" File: %s\n", line.Function.Filename) - fmt.Printf(" Line: %d\n", line.Line) - fmt.Printf(" StartLine: %d\n", line.Function.StartLine) - fmt.Println("----------------------------------------") - } - } - - return nil + cachePrefix := prefix + ".cache" + f.BoolVar(&cfg.Cache.Enabled, cachePrefix+".enabled", false, "Enable debug info caching") + f.DurationVar(&cfg.Cache.MaxAge, cachePrefix+".max-age", 7*24*time.Hour, "Maximum age of cached debug info") } diff --git a/pkg/phlaredb/symdb/resolver_tree.go b/pkg/phlaredb/symdb/resolver_tree.go index dd75ec8184..6e0ef63f6b 100644 --- a/pkg/phlaredb/symdb/resolver_tree.go +++ b/pkg/phlaredb/symdb/resolver_tree.go @@ -2,6 +2,7 @@ package symdb import ( "context" + "fmt" "sync" pprof "github.com/google/pprof/profile" @@ -23,8 +24,9 @@ func buildTree( ) (*model.Tree, error) { // Try debuginfod symbolization first if symbols != nil && symbols.Symbolizer != nil { + //nolint:staticcheck if err := symbolizeLocations(ctx, symbols); err != nil { - // TODO: Log error but continue? partial symbolization is better than none + // TODO: Log/process error but continue? partial symbolization is better than none } } @@ -250,6 +252,8 @@ func minValue(nodes []Node, maxNodes int64) int64 { } func symbolizeLocations(ctx context.Context, symbols *Symbols) error { + var errs []error + type locToSymbolize struct { idx int32 loc *schemav1.InMemoryLocation @@ -259,11 +263,12 @@ func symbolizeLocations(ctx context.Context, symbols *Symbols) error { // Find all locations needing symbolization for i, loc := range symbols.Locations { + locCopy := loc if mapping := &symbols.Mappings[loc.MappingId]; symbols.needsDebuginfodSymbolization(&loc, mapping) { buildIDStr := symbols.Strings[mapping.BuildId] locsByBuildId[buildIDStr] = append(locsByBuildId[buildIDStr], locToSymbolize{ idx: int32(i), - loc: &loc, + loc: &locCopy, mapping: mapping, }) } @@ -290,7 +295,7 @@ func symbolizeLocations(ctx context.Context, symbols *Symbols) error { } if err := symbols.Symbolizer.Symbolize(ctx, req); err != nil { - // TODO: log/process errors but continue with other build IDs + errs = append(errs, fmt.Errorf("symbolize build ID %s: %w", buildID, err)) continue } @@ -328,5 +333,10 @@ func symbolizeLocations(ctx context.Context, symbols *Symbols) error { } } } + + if len(errs) > 0 { + return fmt.Errorf("symbolization errors: %v", errs) + } + return nil } diff --git a/pkg/phlaredb/symdb/resolver_tree_test.go b/pkg/phlaredb/symdb/resolver_tree_test.go index afc4f8fd26..e1c401ee31 100644 --- a/pkg/phlaredb/symdb/resolver_tree_test.go +++ b/pkg/phlaredb/symdb/resolver_tree_test.go @@ -284,7 +284,7 @@ func Test_buildTree_Symbolization(t *testing.T) { return "testdata/unsymbolized.debug", nil }, } - sym := symbolizer.NewSymbolizer(mockClient) + sym := symbolizer.NewSymbolizer(mockClient, nil) symbols.SetSymbolizer(sym) appender := NewSampleAppender() @@ -343,7 +343,7 @@ func Test_buildTree_Symbolization(t *testing.T) { return "", fmt.Errorf("symbolization failed") }, } - sym := symbolizer.NewSymbolizer(mockClient) + sym := symbolizer.NewSymbolizer(mockClient, nil) symbols.SetSymbolizer(sym) appender := NewSampleAppender() From a47732074b8cadf7f8cfb19c5a1a1feb783a01bc Mon Sep 17 00:00:00 2001 From: Marc Sanmiquel Date: Sun, 19 Jan 2025 10:47:01 +0100 Subject: [PATCH 5/5] Adding symbolizer instrumentation --- cmd/symbolization/main.go | 4 +- pkg/experiment/query_backend/backend.go | 2 +- pkg/experiment/symbolizer/cache.go | 36 +++- .../symbolizer/debuginfod_client.go | 24 ++- pkg/experiment/symbolizer/metrics.go | 160 ++++++++++++++++++ pkg/experiment/symbolizer/symbolizer.go | 43 +++-- pkg/phlaredb/symdb/resolver_tree_test.go | 4 +- 7 files changed, 249 insertions(+), 24 deletions(-) create mode 100644 pkg/experiment/symbolizer/metrics.go diff --git a/cmd/symbolization/main.go b/cmd/symbolization/main.go index 487add7a0e..d47d3e7808 100644 --- a/cmd/symbolization/main.go +++ b/cmd/symbolization/main.go @@ -16,12 +16,12 @@ const ( ) func main() { - client := symbolizer.NewDebuginfodClient(debuginfodBaseURL) + client := symbolizer.NewDebuginfodClient(debuginfodBaseURL, nil) // Alternatively, use a local debug info file: //client := &localDebuginfodClient{debugFilePath: "/path/to/your/debug/file"} - s := symbolizer.NewSymbolizer(client, nil) + s := symbolizer.NewSymbolizer(client, nil, nil) ctx := context.Background() _, err := client.FetchDebuginfo(buildID) diff --git a/pkg/experiment/query_backend/backend.go b/pkg/experiment/query_backend/backend.go index 1f2ea83769..30099b3b1e 100644 --- a/pkg/experiment/query_backend/backend.go +++ b/pkg/experiment/query_backend/backend.go @@ -66,7 +66,7 @@ func New( var sym *symbolizer.Symbolizer if config.Symbolizer.DebuginfodURL != "" { var err error - sym, err = symbolizer.NewFromConfig(context.Background(), config.Symbolizer) + sym, err = symbolizer.NewFromConfig(context.Background(), config.Symbolizer, reg) if err != nil { return nil, fmt.Errorf("create symbolizer: %w", err) } diff --git a/pkg/experiment/symbolizer/cache.go b/pkg/experiment/symbolizer/cache.go index acdca631ce..82de47fef6 100644 --- a/pkg/experiment/symbolizer/cache.go +++ b/pkg/experiment/symbolizer/cache.go @@ -15,10 +15,11 @@ type CacheConfig struct { MaxAge time.Duration `yaml:"max_age"` } -func NewObjstoreCache(bucket objstore.Bucket, maxAge time.Duration) *ObjstoreCache { +func NewObjstoreCache(bucket objstore.Bucket, maxAge time.Duration, metrics *Metrics) *ObjstoreCache { return &ObjstoreCache{ - bucket: bucket, - maxAge: maxAge, + bucket: bucket, + maxAge: maxAge, + metrics: metrics, } } @@ -30,17 +31,26 @@ type DebugInfoCache interface { // ObjstoreCache implements DebugInfoCache using S3 storage type ObjstoreCache struct { - bucket objstore.Bucket - maxAge time.Duration + bucket objstore.Bucket + maxAge time.Duration + metrics *Metrics } func (c *ObjstoreCache) Get(ctx context.Context, buildID string) (io.ReadCloser, error) { + c.metrics.cacheRequestsTotal.WithLabelValues("get").Inc() + start := time.Now() + defer func() { + c.metrics.cacheOperationDuration.WithLabelValues("get").Observe(time.Since(start).Seconds()) + }() + // First check if object exists to avoid unnecessary operations reader, err := c.bucket.Get(ctx, buildID) if err != nil { if c.bucket.IsObjNotFoundErr(err) { + c.metrics.cacheMissesTotal.Inc() return nil, err } + c.metrics.cacheRequestErrorsTotal.WithLabelValues("get", "read_error").Inc() return nil, fmt.Errorf("get from cache: %w", err) } @@ -48,28 +58,42 @@ func (c *ObjstoreCache) Get(ctx context.Context, buildID string) (io.ReadCloser, attrs, err := c.bucket.Attributes(ctx, buildID) if err != nil { reader.Close() + c.metrics.cacheRequestErrorsTotal.WithLabelValues("get", "attribute_error").Inc() return nil, fmt.Errorf("get cache attributes: %w", err) } // Check if expired if time.Since(attrs.LastModified) > c.maxAge { reader.Close() + c.metrics.cacheExpiredTotal.Inc() + // Async deletion to not block the request go func() { delCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() - _ = c.bucket.Delete(delCtx, buildID) + if err = c.bucket.Delete(delCtx, buildID); err != nil { + c.metrics.cacheRequestErrorsTotal.WithLabelValues("delete", "delete_error").Inc() + } }() return nil, fmt.Errorf("cached object expired") } + c.metrics.cacheHitsTotal.Inc() return reader, nil } func (c *ObjstoreCache) Put(ctx context.Context, buildID string, reader io.Reader) error { + c.metrics.cacheRequestsTotal.WithLabelValues("put").Inc() + start := time.Now() + defer func() { + c.metrics.cacheOperationDuration.WithLabelValues("put").Observe(time.Since(start).Seconds()) + }() + if err := c.bucket.Upload(ctx, buildID, reader); err != nil { + c.metrics.cacheRequestErrorsTotal.WithLabelValues("put", "upload_error").Inc() return fmt.Errorf("upload to cache: %w", err) } + return nil } diff --git a/pkg/experiment/symbolizer/debuginfod_client.go b/pkg/experiment/symbolizer/debuginfod_client.go index 9b6054e27f..dfbcd11739 100644 --- a/pkg/experiment/symbolizer/debuginfod_client.go +++ b/pkg/experiment/symbolizer/debuginfod_client.go @@ -7,6 +7,7 @@ import ( "os" "path/filepath" "regexp" + "time" ) type DebuginfodClient interface { @@ -15,18 +16,24 @@ type DebuginfodClient interface { type debuginfodClient struct { baseURL string + metrics *Metrics } -func NewDebuginfodClient(baseURL string) DebuginfodClient { +func NewDebuginfodClient(baseURL string, metrics *Metrics) DebuginfodClient { return &debuginfodClient{ baseURL: baseURL, + metrics: metrics, } } // FetchDebuginfo fetches the debuginfo file for a specific build ID. func (c *debuginfodClient) FetchDebuginfo(buildID string) (string, error) { + c.metrics.debuginfodRequestsTotal.Inc() + start := time.Now() + sanitizedBuildID, err := sanitizeBuildID(buildID) if err != nil { + c.metrics.debuginfodRequestErrorsTotal.WithLabelValues("invalid_id").Inc() return "", err } @@ -34,29 +41,44 @@ func (c *debuginfodClient) FetchDebuginfo(buildID string) (string, error) { resp, err := http.Get(url) if err != nil { + c.metrics.debuginfodRequestErrorsTotal.WithLabelValues("http").Inc() + c.metrics.debuginfodRequestDuration.WithLabelValues("error").Observe(time.Since(start).Seconds()) return "", fmt.Errorf("failed to fetch debuginfod: %w", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { + c.metrics.debuginfodRequestErrorsTotal.WithLabelValues("http").Inc() + c.metrics.debuginfodRequestDuration.WithLabelValues("error").Observe(time.Since(start).Seconds()) return "", fmt.Errorf("unexpected HTTP status: %s", resp.Status) } + // Record file size from Content-Length if available + if contentLength := resp.ContentLength; contentLength > 0 { + c.metrics.debuginfodFileSize.Observe(float64(contentLength)) + } + // TODO: Avoid file operations and handle debuginfo in memory. // Save the debuginfo to a temporary file tempDir := os.TempDir() filePath := filepath.Join(tempDir, fmt.Sprintf("%s.elf", sanitizedBuildID)) outFile, err := os.Create(filePath) if err != nil { + c.metrics.debuginfodRequestErrorsTotal.WithLabelValues("file_create").Inc() + c.metrics.debuginfodRequestDuration.WithLabelValues("error").Observe(time.Since(start).Seconds()) return "", fmt.Errorf("failed to create temp file: %w", err) } defer outFile.Close() _, err = io.Copy(outFile, resp.Body) if err != nil { + c.metrics.debuginfodRequestErrorsTotal.WithLabelValues("write").Inc() + c.metrics.debuginfodRequestDuration.WithLabelValues("error").Observe(time.Since(start).Seconds()) return "", fmt.Errorf("failed to write debuginfod to file: %w", err) } + c.metrics.debuginfodRequestDuration.WithLabelValues("success").Observe(time.Since(start).Seconds()) + return filePath, nil } diff --git a/pkg/experiment/symbolizer/metrics.go b/pkg/experiment/symbolizer/metrics.go new file mode 100644 index 0000000000..23ee7b74dc --- /dev/null +++ b/pkg/experiment/symbolizer/metrics.go @@ -0,0 +1,160 @@ +package symbolizer + +import "github.com/prometheus/client_golang/prometheus" + +type Metrics struct { + registerer prometheus.Registerer + + // Debuginfod metrics + debuginfodRequestDuration *prometheus.HistogramVec + debuginfodFileSize prometheus.Histogram + debuginfodRequestsTotal prometheus.Counter + debuginfodRequestErrorsTotal *prometheus.CounterVec + + // Cache metrics + cacheRequestsTotal *prometheus.CounterVec + cacheRequestErrorsTotal *prometheus.CounterVec + cacheHitsTotal prometheus.Counter + cacheMissesTotal prometheus.Counter + cacheOperationDuration *prometheus.HistogramVec + cacheExpiredTotal prometheus.Counter + + // Symbolization metrics + //symbolizationDuration prometheus.Histogram + //symbolizationLocations *prometheus.CounterVec + symbolizationRequestsTotal prometheus.Counter + symbolizationRequestErrorsTotal *prometheus.CounterVec + symbolizationDuration prometheus.Histogram + symbolizationLocationTotal *prometheus.CounterVec +} + +func NewMetrics(reg prometheus.Registerer) *Metrics { + m := &Metrics{ + registerer: reg, + debuginfodRequestDuration: prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Name: "pyroscope_symbolizer_debuginfod_request_duration_seconds", + Help: "Time spent performing debuginfod requests", + Buckets: []float64{0.1, 0.5, 1, 5, 10, 30, 60, 120, 300}, + }, []string{"status"}, + ), + debuginfodFileSize: prometheus.NewHistogram( + prometheus.HistogramOpts{ + Name: "pyroscope_symbolizer_debuginfo_file_size_bytes", + Help: "Size of debug info files fetched from debuginfod", + // 1MB to 4GB + Buckets: prometheus.ExponentialBuckets(1024*1024, 2, 12), + }, + ), + debuginfodRequestsTotal: prometheus.NewCounter(prometheus.CounterOpts{ + Name: "pyroscope_symbolizer_debuginfod_requests_total", + Help: "Total number of debuginfod requests attempted", + }), + debuginfodRequestErrorsTotal: prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "pyroscope_symbolizer_debuginfod_request_errors_total", + Help: "Total number of debuginfod request errors", + }, []string{"reason"}), + cacheRequestsTotal: prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "pyroscope_symbolizer_cache_requests_total", + Help: "Total number of cache requests", + }, []string{"operation"}), + cacheRequestErrorsTotal: prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "pyroscope_symbolizer_cache_request_errors_total", + Help: "Total number of cache request errors", + }, []string{"operation", "reason"}), // get/put, and specific error reasons + cacheHitsTotal: prometheus.NewCounter(prometheus.CounterOpts{ + Name: "pyroscope_symbolizer_cache_hits_total", + Help: "Total number of cache hits", + }), + cacheMissesTotal: prometheus.NewCounter(prometheus.CounterOpts{ + Name: "pyroscope_symbolizer_cache_misses_total", + Help: "Total number of cache misses", + }), + cacheOperationDuration: prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "pyroscope_symbolizer_cache_operation_duration_seconds", + Help: "Time spent performing cache operations", + Buckets: []float64{.01, .05, .1, .5, 1, 5, 10, 30, 60}, + }, + []string{"operation"}, + ), + cacheExpiredTotal: prometheus.NewCounter(prometheus.CounterOpts{ + Name: "pyroscope_symbolizer_cache_expired_total", + Help: "Total number of expired items removed from cache", + }), + symbolizationRequestsTotal: prometheus.NewCounter(prometheus.CounterOpts{ + Name: "pyroscope_symbolizer_requests_total", + Help: "Total number of symbolization requests", + }), + symbolizationRequestErrorsTotal: prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "pyroscope_symbolizer_request_errors_total", + Help: "Total number of symbolization errors", + }, []string{"reason"}), + symbolizationDuration: prometheus.NewHistogram( + prometheus.HistogramOpts{ + Name: "pyroscope_symbolizer_duration_seconds", + Help: "Time spent performing symbolization", + Buckets: []float64{.01, .05, .1, .5, 1, 5, 10, 30}, + }, + ), + symbolizationLocationTotal: prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "pyroscope_symbolizer_locations_total", + Help: "Total number of locations processed", + }, []string{"status"}), + } + m.register() + return m +} + +func (m *Metrics) register() { + if m.registerer == nil { + return + } + + collectors := []prometheus.Collector{ + m.debuginfodRequestDuration, + m.debuginfodFileSize, + m.debuginfodRequestErrorsTotal, + m.debuginfodRequestsTotal, + m.cacheRequestsTotal, + m.cacheRequestErrorsTotal, + m.cacheHitsTotal, + m.cacheMissesTotal, + m.cacheOperationDuration, + m.cacheExpiredTotal, + m.symbolizationRequestsTotal, + m.symbolizationRequestErrorsTotal, + m.symbolizationDuration, + m.symbolizationLocationTotal, + } + + for _, collector := range collectors { + m.registerer.MustRegister(collector) + } +} + +func (m *Metrics) Unregister() { + if m.registerer == nil { + return + } + + collectors := []prometheus.Collector{ + m.debuginfodRequestDuration, + m.debuginfodFileSize, + m.debuginfodRequestErrorsTotal, + m.debuginfodRequestsTotal, + m.cacheRequestsTotal, + m.cacheRequestErrorsTotal, + m.cacheHitsTotal, + m.cacheMissesTotal, + m.cacheOperationDuration, + m.cacheExpiredTotal, + m.symbolizationRequestsTotal, + m.symbolizationRequestErrorsTotal, + m.symbolizationDuration, + m.symbolizationLocationTotal, + } + + for _, collector := range collectors { + m.registerer.Unregister(collector) + } +} diff --git a/pkg/experiment/symbolizer/symbolizer.go b/pkg/experiment/symbolizer/symbolizer.go index ac520441b8..89a8907809 100644 --- a/pkg/experiment/symbolizer/symbolizer.go +++ b/pkg/experiment/symbolizer/symbolizer.go @@ -10,6 +10,8 @@ import ( "os" "time" + "github.com/prometheus/client_golang/prometheus" + objstoreclient "github.com/grafana/pyroscope/pkg/objstore/client" ) @@ -50,22 +52,24 @@ type Config struct { } type Symbolizer struct { - client DebuginfodClient - cache DebugInfoCache + client DebuginfodClient + cache DebugInfoCache + metrics *Metrics } -func NewSymbolizer(client DebuginfodClient, cache DebugInfoCache) *Symbolizer { +func NewSymbolizer(client DebuginfodClient, cache DebugInfoCache, reg prometheus.Registerer) *Symbolizer { if cache == nil { cache = NewNullCache() } return &Symbolizer{ - client: client, - cache: cache, + client: client, + cache: cache, + metrics: NewMetrics(reg), } } -func NewFromConfig(ctx context.Context, cfg Config) (*Symbolizer, error) { - client := NewDebuginfodClient(cfg.DebuginfodURL) +func NewFromConfig(ctx context.Context, cfg Config, reg prometheus.Registerer) (*Symbolizer, error) { + metrics := NewMetrics(reg) // Default to no caching var cache = NewNullCache() @@ -78,16 +82,24 @@ func NewFromConfig(ctx context.Context, cfg Config) (*Symbolizer, error) { if err != nil { return nil, fmt.Errorf("create debug info storage: %w", err) } - cache = NewObjstoreCache(bucket, cfg.Cache.MaxAge) + cache = NewObjstoreCache(bucket, cfg.Cache.MaxAge, metrics) } + client := NewDebuginfodClient(cfg.DebuginfodURL, metrics) + return &Symbolizer{ - client: client, - cache: cache, + client: client, + cache: cache, + metrics: metrics, }, nil } func (s *Symbolizer) Symbolize(ctx context.Context, req Request) error { + start := time.Now() + defer func() { + s.metrics.symbolizationDuration.Observe(time.Since(start).Seconds()) + }() + debugReader, err := s.cache.Get(ctx, req.BuildID) if err == nil { defer debugReader.Close() @@ -97,12 +109,14 @@ func (s *Symbolizer) Symbolize(ctx context.Context, req Request) error { // Cache miss - fetch from debuginfod filepath, err := s.client.FetchDebuginfo(req.BuildID) if err != nil { + s.metrics.symbolizationRequestErrorsTotal.WithLabelValues("debuginfod_error").Inc() return fmt.Errorf("fetch debuginfo: %w", err) } // Open for symbolization f, err := os.Open(filepath) if err != nil { + s.metrics.symbolizationRequestErrorsTotal.WithLabelValues("file_error").Inc() return fmt.Errorf("open debug file: %w", err) } defer f.Close() @@ -126,6 +140,7 @@ func (s *Symbolizer) Symbolize(ctx context.Context, req Request) error { func (s *Symbolizer) symbolizeFromReader(ctx context.Context, r io.ReadCloser, req Request) error { elfFile, err := elf.NewFile(io.NewSectionReader(r.(io.ReaderAt), 0, 1<<63-1)) if err != nil { + s.metrics.symbolizationRequestErrorsTotal.WithLabelValues("elf_error").Inc() return fmt.Errorf("create ELF file from reader: %w", err) } defer elfFile.Close() @@ -133,13 +148,14 @@ func (s *Symbolizer) symbolizeFromReader(ctx context.Context, r io.ReadCloser, r // Get executable info for address normalization ei, err := ExecutableInfoFromELF(elfFile) if err != nil { + s.metrics.symbolizationRequestErrorsTotal.WithLabelValues("elf_info_error").Inc() return fmt.Errorf("executable info from ELF: %w", err) } // Create liner liner, err := NewDwarfResolver(elfFile) if err != nil { - return fmt.Errorf("create liner: %w", err) + s.metrics.symbolizationRequestErrorsTotal.WithLabelValues("dwarf_error").Inc() } //defer liner.Close() @@ -151,16 +167,19 @@ func (s *Symbolizer) symbolizeFromReader(ctx context.Context, r io.ReadCloser, r Offset: loc.Mapping.Offset, }) if err != nil { + s.metrics.symbolizationLocationTotal.WithLabelValues("error").Inc() return fmt.Errorf("normalize address: %w", err) } // Get source lines for the address lines, err := liner.ResolveAddress(ctx, addr) if err != nil { - continue // Skip errors for individual addresses + s.metrics.symbolizationLocationTotal.WithLabelValues("error").Inc() + return fmt.Errorf("resolve address: %w", err) } loc.Lines = lines + s.metrics.symbolizationLocationTotal.WithLabelValues("success").Inc() } } diff --git a/pkg/phlaredb/symdb/resolver_tree_test.go b/pkg/phlaredb/symdb/resolver_tree_test.go index e1c401ee31..006cab047e 100644 --- a/pkg/phlaredb/symdb/resolver_tree_test.go +++ b/pkg/phlaredb/symdb/resolver_tree_test.go @@ -284,7 +284,7 @@ func Test_buildTree_Symbolization(t *testing.T) { return "testdata/unsymbolized.debug", nil }, } - sym := symbolizer.NewSymbolizer(mockClient, nil) + sym := symbolizer.NewSymbolizer(mockClient, nil, nil) symbols.SetSymbolizer(sym) appender := NewSampleAppender() @@ -343,7 +343,7 @@ func Test_buildTree_Symbolization(t *testing.T) { return "", fmt.Errorf("symbolization failed") }, } - sym := symbolizer.NewSymbolizer(mockClient, nil) + sym := symbolizer.NewSymbolizer(mockClient, nil, nil) symbols.SetSymbolizer(sym) appender := NewSampleAppender()