123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354 |
- // Copyright 2022 The Gitea Authors. All rights reserved.
- // SPDX-License-Identifier: MIT
-
- package process
-
- import (
- "fmt"
- "io"
- "runtime/pprof"
- "sort"
- "time"
-
- "github.com/google/pprof/profile"
- )
-
- // StackEntry is an entry on a stacktrace
- type StackEntry struct {
- Function string
- File string
- Line int
- }
-
- // Label represents a pprof label assigned to goroutine stack
- type Label struct {
- Name string
- Value string
- }
-
- // Stack is a stacktrace relating to a goroutine. (Multiple goroutines may have the same stacktrace)
- type Stack struct {
- Count int64 // Number of goroutines with this stack trace
- Description string
- Labels []*Label `json:",omitempty"`
- Entry []*StackEntry `json:",omitempty"`
- }
-
- // A Process is a combined representation of a Process and a Stacktrace for the goroutines associated with it
- type Process struct {
- PID IDType
- ParentPID IDType
- Description string
- Start time.Time
- Type string
-
- Children []*Process `json:",omitempty"`
- Stacks []*Stack `json:",omitempty"`
- }
-
- // Processes gets the processes in a thread safe manner
- func (pm *Manager) Processes(flat, noSystem bool) ([]*Process, int) {
- pm.mutex.Lock()
- processCount := len(pm.processMap)
- processes := make([]*Process, 0, len(pm.processMap))
- if flat {
- for _, process := range pm.processMap {
- if noSystem && process.Type == SystemProcessType {
- continue
- }
- processes = append(processes, process.toProcess())
- }
- } else {
- // We need our own processMap
- processMap := map[IDType]*Process{}
- for _, internalProcess := range pm.processMap {
- process, ok := processMap[internalProcess.PID]
- if !ok {
- process = internalProcess.toProcess()
- processMap[process.PID] = process
- }
-
- // Check its parent
- if process.ParentPID == "" {
- processes = append(processes, process)
- continue
- }
-
- internalParentProcess, ok := pm.processMap[internalProcess.ParentPID]
- if ok {
- parentProcess, ok := processMap[process.ParentPID]
- if !ok {
- parentProcess = internalParentProcess.toProcess()
- processMap[parentProcess.PID] = parentProcess
- }
- parentProcess.Children = append(parentProcess.Children, process)
- continue
- }
-
- processes = append(processes, process)
- }
- }
- pm.mutex.Unlock()
-
- if !flat && noSystem {
- for i := 0; i < len(processes); i++ {
- process := processes[i]
- if process.Type != SystemProcessType {
- continue
- }
- processes[len(processes)-1], processes[i] = processes[i], processes[len(processes)-1]
- processes = append(processes[:len(processes)-1], process.Children...)
- i--
- }
- }
-
- // Sort by process' start time. Oldest process appears first.
- sort.Slice(processes, func(i, j int) bool {
- left, right := processes[i], processes[j]
-
- return left.Start.Before(right.Start)
- })
-
- return processes, processCount
- }
-
- // ProcessStacktraces gets the processes and stacktraces in a thread safe manner
- func (pm *Manager) ProcessStacktraces(flat, noSystem bool) ([]*Process, int, int64, error) {
- var stacks *profile.Profile
- var err error
-
- // We cannot use the pm.ProcessMap here because we will release the mutex ...
- processMap := map[IDType]*Process{}
- var processCount int
-
- // Lock the manager
- pm.mutex.Lock()
- processCount = len(pm.processMap)
-
- // Add a defer to unlock in case there is a panic
- unlocked := false
- defer func() {
- if !unlocked {
- pm.mutex.Unlock()
- }
- }()
-
- processes := make([]*Process, 0, len(pm.processMap))
- if flat {
- for _, internalProcess := range pm.processMap {
- process := internalProcess.toProcess()
- processMap[process.PID] = process
- if noSystem && internalProcess.Type == SystemProcessType {
- continue
- }
- processes = append(processes, process)
- }
- } else {
- for _, internalProcess := range pm.processMap {
- process, ok := processMap[internalProcess.PID]
- if !ok {
- process = internalProcess.toProcess()
- processMap[process.PID] = process
- }
-
- // Check its parent
- if process.ParentPID == "" {
- processes = append(processes, process)
- continue
- }
-
- internalParentProcess, ok := pm.processMap[internalProcess.ParentPID]
- if ok {
- parentProcess, ok := processMap[process.ParentPID]
- if !ok {
- parentProcess = internalParentProcess.toProcess()
- processMap[parentProcess.PID] = parentProcess
- }
- parentProcess.Children = append(parentProcess.Children, process)
- continue
- }
-
- processes = append(processes, process)
- }
- }
-
- // Now from within the lock we need to get the goroutines.
- // Why? If we release the lock then between between filling the above map and getting
- // the stacktraces another process could be created which would then look like a dead process below
- reader, writer := io.Pipe()
- defer reader.Close()
- go func() {
- err := pprof.Lookup("goroutine").WriteTo(writer, 0)
- _ = writer.CloseWithError(err)
- }()
- stacks, err = profile.Parse(reader)
- if err != nil {
- return nil, 0, 0, err
- }
-
- // Unlock the mutex
- pm.mutex.Unlock()
- unlocked = true
-
- goroutineCount := int64(0)
-
- // Now walk through the "Sample" slice in the goroutines stack
- for _, sample := range stacks.Sample {
- // In the "goroutine" pprof profile each sample represents one or more goroutines
- // with the same labels and stacktraces.
-
- // We will represent each goroutine by a `Stack`
- stack := &Stack{}
-
- // Add the non-process associated labels from the goroutine sample to the Stack
- for name, value := range sample.Label {
- if name == DescriptionPProfLabel || name == PIDPProfLabel || (!flat && name == PPIDPProfLabel) || name == ProcessTypePProfLabel {
- continue
- }
-
- // Labels from the "goroutine" pprof profile only have one value.
- // This is because the underlying representation is a map[string]string
- if len(value) != 1 {
- // Unexpected...
- return nil, 0, 0, fmt.Errorf("label: %s in goroutine stack with unexpected number of values: %v", name, value)
- }
-
- stack.Labels = append(stack.Labels, &Label{Name: name, Value: value[0]})
- }
-
- // The number of goroutines that this sample represents is the `stack.Value[0]`
- stack.Count = sample.Value[0]
- goroutineCount += stack.Count
-
- // Now we want to associate this Stack with a Process.
- var process *Process
-
- // Try to get the PID from the goroutine labels
- if pidvalue, ok := sample.Label[PIDPProfLabel]; ok && len(pidvalue) == 1 {
- pid := IDType(pidvalue[0])
-
- // Now try to get the process from our map
- process, ok = processMap[pid]
- if !ok && pid != "" {
- // This means that no process has been found in the process map - but there was a process PID
- // Therefore this goroutine belongs to a dead process and it has escaped control of the process as it
- // should have died with the process context cancellation.
-
- // We need to create a dead process holder for this process and label it appropriately
-
- // get the parent PID
- ppid := IDType("")
- if value, ok := sample.Label[PPIDPProfLabel]; ok && len(value) == 1 {
- ppid = IDType(value[0])
- }
-
- // format the description
- description := "(dead process)"
- if value, ok := sample.Label[DescriptionPProfLabel]; ok && len(value) == 1 {
- description = value[0] + " " + description
- }
-
- // override the type of the process to "code" but add the old type as a label on the first stack
- ptype := NoneProcessType
- if value, ok := sample.Label[ProcessTypePProfLabel]; ok && len(value) == 1 {
- stack.Labels = append(stack.Labels, &Label{Name: ProcessTypePProfLabel, Value: value[0]})
- }
- process = &Process{
- PID: pid,
- ParentPID: ppid,
- Description: description,
- Type: ptype,
- }
-
- // Now add the dead process back to the map and tree so we don't go back through this again.
- processMap[process.PID] = process
- added := false
- if process.ParentPID != "" && !flat {
- if parent, ok := processMap[process.ParentPID]; ok {
- parent.Children = append(parent.Children, process)
- added = true
- }
- }
- if !added {
- processes = append(processes, process)
- }
- }
- }
-
- if process == nil {
- // This means that the sample we're looking has no PID label
- var ok bool
- process, ok = processMap[""]
- if !ok {
- // this is the first time we've come acrross an unassociated goroutine so create a "process" to hold them
- process = &Process{
- Description: "(unassociated)",
- Type: NoneProcessType,
- }
- processMap[process.PID] = process
- processes = append(processes, process)
- }
- }
-
- // The sample.Location represents a stack trace for this goroutine,
- // however each Location can represent multiple lines (mostly due to inlining)
- // so we need to walk the lines too
- for _, location := range sample.Location {
- for _, line := range location.Line {
- entry := &StackEntry{
- Function: line.Function.Name,
- File: line.Function.Filename,
- Line: int(line.Line),
- }
- stack.Entry = append(stack.Entry, entry)
- }
- }
-
- // Now we need a short-descriptive name to call the stack trace if when it is folded and
- // assuming the stack trace has some lines we'll choose the bottom of the stack (i.e. the
- // initial function that started the stack trace.) The top of the stack is unlikely to
- // be very helpful as a lot of the time it will be runtime.select or some other call into
- // a std library.
- stack.Description = "(unknown)"
- if len(stack.Entry) > 0 {
- stack.Description = stack.Entry[len(stack.Entry)-1].Function
- }
-
- process.Stacks = append(process.Stacks, stack)
- }
-
- // restrict to not show system processes
- if noSystem {
- for i := 0; i < len(processes); i++ {
- process := processes[i]
- if process.Type != SystemProcessType && process.Type != NoneProcessType {
- continue
- }
- processes[len(processes)-1], processes[i] = processes[i], processes[len(processes)-1]
- processes = append(processes[:len(processes)-1], process.Children...)
- i--
- }
- }
-
- // Now finally re-sort the processes. Newest process appears first
- after := func(processes []*Process) func(i, j int) bool {
- return func(i, j int) bool {
- left, right := processes[i], processes[j]
- return left.Start.After(right.Start)
- }
- }
- sort.Slice(processes, after(processes))
- if !flat {
-
- var sortChildren func(process *Process)
-
- sortChildren = func(process *Process) {
- sort.Slice(process.Children, after(process.Children))
- for _, child := range process.Children {
- sortChildren(child)
- }
- }
- }
-
- return processes, processCount, goroutineCount, err
- }
|