Copyright 2017 Google LLC Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
Package profiler is a client for the Cloud Profiler service. Usage example: import "cloud.google.com/go/profiler" ... if err := profiler.Start(profiler.Config{Service: "my-service"}); err != nil { // TODO: Handle error. } Calling Start will start a goroutine to collect profiles and upload to the profiler server, at the rhythm specified by the server. The caller must provide the service string in the config, and may provide other information as well. See Config for details. Profiler has CPU, heap and goroutine profiling enabled by default. Mutex profiling can be enabled in the config. Note that goroutine and mutex profiles are shown as "threads" and "contention" profiles in the profiler UI.
package profiler

import (
	
	
	
	
	
	
	
	
	
	
	

	gcemd 
	
	
	
	
	gax 
	
	gtransport 
	pb 
	edpb 
	
	
	grpcmd 
	
)

var (
	config       Config
	startOnce    allowUntilSuccess
For testing only. When the profiling loop has exited without error and this channel is non-nil, "true" will be sent to this channel.
	profilingDone chan bool
)

const (
	apiAddress       = "cloudprofiler.googleapis.com:443"
	xGoogAPIMetadata = "x-goog-api-client"
	zoneNameLabel    = "zone"
	versionLabel     = "version"
	languageLabel    = "language"
	instanceLabel    = "instance"
	scope            = "https://www.googleapis.com/auth/monitoring.write"

Ensure the agent will recover within 1 hour.
	maxBackoff        = time.Hour
	backoffMultiplier = 1.3 // Backoff envelope increases by this factor on each retry.
	retryInfoMetadata = "google.rpc.retryinfo-bin"
)
Config is the profiler configuration.
Service must be provided to start the profiler. It specifies the name of the service under which the profiled data will be recorded and exposed at the Profiler UI for the project. You can specify an arbitrary string, but see Deployment.target at https://github.com/googleapis/googleapis/blob/master/google/devtools/cloudprofiler/v2/profiler.proto for restrictions. If the parameter is not set, the agent will probe GAE_SERVICE environment variable which is present in Google App Engine environment. NOTE: The string should be the same across different replicas of your service so that the globally constant profiling rate is maintained. Do not put things like PID or unique pod ID in the name.
ServiceVersion is an optional field specifying the version of the service. It can be an arbitrary string. Profiler profiles once per minute for each version of each service in each zone. ServiceVersion defaults to GAE_VERSION environment variable if that is set, or to empty string otherwise.
DebugLogging enables detailed debug logging from profiler. It defaults to false.
MutexProfiling enables mutex profiling. It defaults to false. Note that mutex profiling is not supported by Go versions older than Go 1.8.
When true, collecting the CPU profiles is disabled.
When true, collecting the allocation profiles is disabled.
AllocForceGC forces garbage collection before the collection of each heap profile collected to produce the allocation profile. This increases the accuracy of allocation profiling. It defaults to false.
When true, collecting the heap profiles is disabled.
When true, collecting the goroutine profiles is disabled.
When true, the agent sends all telemetries via OpenCensus exporter, which can be viewed in Cloud Trace and Cloud Monitoring. Default is false.
ProjectID is the Cloud Console project ID to use instead of the one set by GOOGLE_CLOUD_PROJECT environment variable or read from the VM metadata server. Set this if you are running the agent in your local environment or anywhere else outside of Google Cloud Platform.
APIAddr is the HTTP endpoint to use to connect to the profiler agent API. Defaults to the production environment, overridable for testing.
Instance is the name of Compute Engine instance the profiler agent runs on. This is normally determined from the Compute Engine metadata server and doesn't need to be initialized. It needs to be set in rare cases where the metadata server is present but is flaky or otherwise misbehave.
Zone is the zone of Compute Engine instance the profiler agent runs on. This is normally determined from the Compute Engine metadata server and doesn't need to be initialized. It needs to be set in rare cases where the metadata server is present but is flaky or otherwise misbehave.
numProfiles is the number of profiles which should be collected before the profile collection loop exits.When numProfiles is 0, profiles will be collected for the duration of the program. For testing only.
allowUntilSuccess is an object that will perform action till it succeeds once. This is a modified form of Go's sync.Once
do calls function f only if it hasnt returned nil previously. Once f returns nil, do will not call function f any more. This is a modified form of Go's sync.Once.Do
func ( *allowUntilSuccess) ( func() error) ( error) {
	.m.Lock()
	defer .m.Unlock()
	if .done == 0 {
		if  = ();  == nil {
			.done = 1
		}
	} else {
		debugLog("profiler.Start() called again after it was previously called")
		 = nil
	}
	return 
}
Start starts a goroutine to collect and upload profiles. The caller must provide the service string in the config. See Config for details. Start should only be called once. Any additional calls will be ignored.
func ( Config,  ...option.ClientOption) error {
	 := startOnce.do(func() error {
		return start(, ...)
	})
	return 
}

func ( Config,  ...option.ClientOption) error {
	if  := initializeConfig();  != nil {
		debugLog("failed to initialize config: %v", )
		return 
	}
	if config.MutexProfiling {
		if mutexEnabled = enableMutexProfiling(); !mutexEnabled {
			return fmt.Errorf("mutex profiling is not supported by %s, requires Go 1.8 or later", runtime.Version())
		}
	}

	 := context.Background()

	 := []option.ClientOption{
		option.WithEndpoint(config.APIAddr),
		option.WithScopes(scope),
		option.WithUserAgent(fmt.Sprintf("gcloud-go-profiler/%s", version.Repo)),
	}
	if !config.EnableOCTelemetry {
		 = append(, option.WithTelemetryDisabled())
	}
	 = append(, ...)

	,  := dialGRPC(, ...)
	if  != nil {
		debugLog("failed to dial GRPC: %v", )
		return 
	}

	,  := initializeAgent(pb.NewProfilerServiceClient())
	if  != nil {
		debugLog("failed to start the profiling agent: %v", )
		return 
	}
	go pollProfilerService(withXGoogHeader(), )
	return nil
}

func ( string,  ...interface{}) {
	if config.DebugLogging {
		log.Printf(, ...)
	}
}
agent polls the profiler server for instructions on behalf of a task, and collects and uploads profiles as requested.
abortedBackoffDuration retrieves the retry duration from gRPC trailing metadata, which is set by the profiler server.
func ( grpcmd.MD) (time.Duration, error) {
	 := [retryInfoMetadata]
	if len() <= 0 {
		return 0, errors.New("no retry info")
	}

	var  edpb.RetryInfo
	if  := proto.Unmarshal([]byte([0]), &);  != nil {
		return 0, 
	} else if ,  := ptypes.Duration(.RetryDelay);  != nil {
		return 0, 
	} else {
		if  < 0 {
			return 0, errors.New("negative retry duration")
		}
		return , nil
	}
}

type retryer struct {
	backoff gax.Backoff
	md      grpcmd.MD
}

func ( *retryer) ( error) (time.Duration, bool) {
	,  := status.FromError()
	if  != nil && .Code() == codes.Aborted {
		,  := abortedBackoffDuration(.md)
		if  == nil {
			return , true
		}
		debugLog("failed to get backoff duration: %v", )
	}
	return .backoff.Pause(), true
}
createProfile talks to the profiler server to create profile. In case of error, the goroutine will sleep and retry. Sleep duration may be specified by the server. Otherwise it will be an exponentially increasing value, bounded by maxBackoff.
func ( *agent) ( context.Context) *pb.Profile {
	 := pb.CreateProfileRequest{
		Parent:      "projects/" + .deployment.ProjectId,
		Deployment:  .deployment,
		ProfileType: .profileTypes,
	}

	var  *pb.Profile
	 := grpcmd.New(map[string]string{})

	gax.Invoke(, func( context.Context,  gax.CallSettings) error {
		debugLog("creating a new profile via profiler service")
		var  error
		,  = .client.CreateProfile(, &, grpc.Trailer(&))
		if  != nil {
			debugLog("failed to create profile, will retry: %v", )
		}
		return 
	}, gax.WithRetry(func() gax.Retryer {
		return &retryer{
			backoff: gax.Backoff{
				Initial:    initialBackoff,
				Max:        maxBackoff,
				Multiplier: backoffMultiplier,
			},
			md: ,
		}
	}))

	debugLog("successfully created profile %v", .GetProfileType())
	return 
}

func ( *agent) ( context.Context,  *pb.Profile) {
	var  bytes.Buffer
	 := .GetProfileType()

	 := false
	for ,  := range .profileTypes {
		if  ==  {
			 = true
			break
		}
	}

	if ! {
		debugLog("skipping collection of disabled profile type: %v", )
		return
	}

	switch  {
	case pb.ProfileType_CPU:
		,  := ptypes.Duration(.Duration)
		if  != nil {
			debugLog("failed to get profile duration for CPU profile: %v", )
			return
		}
		if  := startCPUProfile(&);  != nil {
			debugLog("failed to start CPU profile: %v", )
			return
		}
		sleep(, )
		stopCPUProfile()
	case pb.ProfileType_HEAP:
		if  := heapProfile(&);  != nil {
			debugLog("failed to write heap profile: %v", )
			return
		}
	case pb.ProfileType_HEAP_ALLOC:
		,  := ptypes.Duration(.Duration)
		if  != nil {
			debugLog("failed to get profile duration for allocation profile: %v", )
			return
		}
		if  := deltaAllocProfile(, , config.AllocForceGC, &);  != nil {
			debugLog("failed to collect allocation profile: %v", )
			return
		}
	case pb.ProfileType_THREADS:
		if  := pprof.Lookup("goroutine").WriteTo(&, 0);  != nil {
			debugLog("failed to collect goroutine profile: %v", )
			return
		}
	case pb.ProfileType_CONTENTION:
		,  := ptypes.Duration(.Duration)
		if  != nil {
			debugLog("failed to get profile duration: %v", )
			return
		}
		if  := deltaMutexProfile(, , &);  != nil {
			debugLog("failed to collect mutex profile: %v", )
			return
		}
	default:
		debugLog("unexpected profile type: %v", )
		return
	}

	.ProfileBytes = .Bytes()
	.Labels = .profileLabels
	 := pb.UpdateProfileRequest{Profile: }
Upload profile, discard profile in case of error.
	debugLog("start uploading profile")
	if ,  := .client.UpdateProfile(, &);  != nil {
		debugLog("failed to upload profile: %v", )
	}
}
deltaMutexProfile writes mutex profile changes over a time period specified with 'duration' to 'prof'.
func ( context.Context,  time.Duration,  *bytes.Buffer) error {
	if !mutexEnabled {
		return errors.New("mutex profiling is not enabled")
	}
	,  := mutexProfile()
	if  != nil {
		return 
	}
	sleep(, )
	,  := mutexProfile()
	if  != nil {
		return 
	}

	.Scale(-1)
	,  = profile.Merge([]*profile.Profile{, })
	if  != nil {
		return 
	}

	return .Write()
}

func () (*profile.Profile, error) {
	 := pprof.Lookup("mutex")
	if  == nil {
		return nil, errors.New("mutex profiling is not supported")
	}
	var  bytes.Buffer
	if  := .WriteTo(&, 0);  != nil {
		return nil, 
	}
	return profile.Parse(&)
}
withXGoogHeader sets the name and version of the application in the `x-goog-api-client` header passed on each request. Intended for use by Google-written clients.
func ( context.Context,  ...string) context.Context {
	 := append([]string{"gl-go", version.Go(), "gccl", version.Repo}, ...)
	 = append(, "gax", gax.Version, "grpc", grpc.Version)

	,  := grpcmd.FromOutgoingContext()
	 = .Copy()
	[xGoogAPIMetadata] = []string{gax.XGoogHeader(...)}
	return grpcmd.NewOutgoingContext(, )
}
initializeAgent initializes the profiling agent. It returns an error if profile collection should not be started because collection is disabled for all profile types.
func ( pb.ProfilerServiceClient) (*agent, error) {
	 := map[string]string{languageLabel: "go"}
	if config.Zone != "" {
		[zoneNameLabel] = config.Zone
	}
	if config.ServiceVersion != "" {
		[versionLabel] = config.ServiceVersion
	}
	 := &pb.Deployment{
		ProjectId: config.ProjectID,
		Target:    config.Service,
		Labels:    ,
	}

	 := map[string]string{}

	if config.Instance != "" {
		[instanceLabel] = config.Instance
	}

	var  []pb.ProfileType
	if !config.NoCPUProfiling {
		 = append(, pb.ProfileType_CPU)
	}
	if !config.NoHeapProfiling {
		 = append(, pb.ProfileType_HEAP)
	}
	if !config.NoGoroutineProfiling {
		 = append(, pb.ProfileType_THREADS)
	}
	if !config.NoAllocProfiling {
		 = append(, pb.ProfileType_HEAP_ALLOC)
	}
	if mutexEnabled {
		 = append(, pb.ProfileType_CONTENTION)
	}

	if len() == 0 {
		return nil, fmt.Errorf("collection is not enabled for any profile types")
	}

	return &agent{
		client:        ,
		deployment:    ,
		profileLabels: ,
		profileTypes:  ,
	}, nil
}

func ( Config) error {
	config = 

	if config.Service == "" {
		for ,  := range []string{"GAE_SERVICE", "K_SERVICE"} {
			if  := os.Getenv();  != "" {
				config.Service = 
				break
			}
		}
	}
	if config.Service == "" {
		return errors.New("service name must be configured")
	}
	if !serviceRegexp.MatchString(config.Service) {
		return fmt.Errorf("service name %q does not match regular expression %v", config.Service, serviceRegexp)
	}

	if config.ServiceVersion == "" {
		for ,  := range []string{"GAE_VERSION", "K_REVISION"} {
			if  := os.Getenv();  != "" {
				config.ServiceVersion = 
				break
			}
		}
	}

Cloud Shell and App Engine set this environment variable to the project ID, so use it if present. In case of App Engine the project ID is also available from the GCE metadata server, but by using the environment variable saves one request to the metadata server. The environment project ID is only used if no project ID is provided in the configuration.
		config.ProjectID = 
	}
	if onGCE() {
		var  error
		if config.ProjectID == "" {
			if config.ProjectID,  = getProjectID();  != nil {
				return fmt.Errorf("failed to get the project ID from Compute Engine metadata: %v", )
			}
		}

		if config.Zone == "" {
			if config.Zone,  = getZone();  != nil {
				return fmt.Errorf("failed to get zone from Compute Engine metadata: %v", )
			}
		}

		if config.Instance == "" {
			if config.Instance,  = getInstanceName();  != nil {
				if ,  := .(gcemd.NotDefinedError); ! {
					return fmt.Errorf("failed to get instance name from Compute Engine metadata: %v", )
				}
				debugLog("failed to get instance name from Compute Engine metadata, will use empty name: %v", )
			}
		}
	} else {
		if config.ProjectID == "" {
			return fmt.Errorf("project ID must be specified in the configuration if running outside of GCP")
		}
	}

	if config.APIAddr == "" {
		config.APIAddr = apiAddress
	}
	return nil
}
pollProfilerService starts an endless loop to poll the profiler server for instructions, and collects and uploads profiles as requested.
func ( context.Context,  *agent) {
	debugLog("Cloud Profiler Go Agent version: %s", version.Repo)
	debugLog("profiler has started")
	for  := 0; config.numProfiles == 0 ||  < config.numProfiles; ++ {
		 := .createProfile()
		.profileAndUpload(, )
	}

	if profilingDone != nil {
		profilingDone <- true
	}