Copyright 2019 The Go Authors. All rights reserved. Use of this source code is governed by a BSD-style license that can be found in the LICENSE file.
Package fetch provides a way to fetch modules from a proxy.
package fetch

import (
	
	
	
	
	
	
	
	
	
	
	

	
	
	
	
	
	
	
	
	
	
	
	
	
	
)

var (
	ErrModuleContainsNoPackages = errors.New("module contains 0 packages")
	errMalformedZip             = errors.New("module zip is malformed")
)

var (
	fetchLatency = stats.Float64(
		"go-discovery/worker/fetch-latency",
		"Latency of a fetch request.",
		stats.UnitSeconds,
	)
	fetchesShedded = stats.Int64(
		"go-discovery/worker/fetch-shedded",
		"Count of shedded fetches.",
		stats.UnitDimensionless,
	)
	fetchedPackages = stats.Int64(
		"go-discovery/worker/fetch-package-count",
		"Count of successfully fetched packages.",
		stats.UnitDimensionless,
	)
FetchLatencyDistribution aggregates frontend fetch request latency by status code. It does not count shedded requests.
	FetchLatencyDistribution = &view.View{
		Name:        "go-discovery/worker/fetch-latency",
		Measure:     fetchLatency,
		Aggregation: ochttp.DefaultLatencyDistribution,
		Description: "Fetch latency by result status.",
		TagKeys:     []tag.Key{dcensus.KeyStatus},
FetchResponseCount counts fetch responses by status.
	FetchResponseCount = &view.View{
		Name:        "go-discovery/worker/fetch-count",
		Measure:     fetchLatency,
		Aggregation: view.Count(),
		Description: "Fetch request count by result status",
		TagKeys:     []tag.Key{dcensus.KeyStatus},
FetchPackageCount counts how many packages were successfully fetched.
	FetchPackageCount = &view.View{
		Name:        "go-discovery/worker/fetch-package-count",
		Measure:     fetchedPackages,
		Aggregation: view.Count(),
		Description: "Count of packages successfully fetched",
SheddedFetchCount counts the number of fetches that were shedded.
	SheddedFetchCount = &view.View{
		Name:        "go-discovery/worker/fetch-shedded",
		Measure:     fetchesShedded,
		Aggregation: view.Count(),
		Description: "Count of shedded fetches",
	}
)

type FetchResult struct {
	ModulePath       string
	RequestedVersion string
	ResolvedVersion  string
	MainVersion      string
HasGoMod says whether the zip contain a go.mod file. If Module (below) is non-nil, then Module.HasGoMod will be the same value. But HasGoMod will be populated even if Module is nil because there were problems with it, as long as we can download and read the zip.
	HasGoMod             bool
	GoModPath            string
	Status               int
	Error                error
	Defer                func() // caller must defer this on all code paths
	Module               *internal.Module
	PackageVersionStates []*internal.PackageVersionState
}
FetchModule queries the proxy or the Go repo for the requested module version, downloads the module zip, and processes the contents to return an *internal.Module and related information. Even if err is non-nil, the result may contain useful information, like the go.mod path. Callers of FetchModule must defer fr.Defer() immediately after the call.
func ( context.Context, ,  string,  *proxy.Client,  *source.Client) ( *FetchResult) {
	 := time.Now()
	defer func() {
		 := float64(time.Since().Seconds())
		dcensus.RecordWithTag(, dcensus.KeyStatus, strconv.Itoa(.Status), fetchLatency.M())
		if .Status < 300 {
			stats.Record(, fetchedPackages.M(int64(len(.PackageVersionStates))))
		}
	}()

	 = &FetchResult{
		ModulePath:       ,
		RequestedVersion: ,
		Defer:            func() {},
	}
	defer derrors.Wrap(&.Error, "FetchModule(%q, %q)", , )

	,  := fetchModule(, , , )
	.Error = 
	if  != nil {
		.Status = derrors.ToStatus(.Error)
	}
	if .Status == 0 {
		.Status = http.StatusOK
	}
	if  != nil {
		finishFetchInfo(, .Status, .Error)
	}
	return 
}

func ( context.Context,  *FetchResult,  *proxy.Client,  *source.Client) (*FetchInfo, error) {
	,  := GetInfo(, .ModulePath, .RequestedVersion, )
	if  != nil {
		return nil, 
	}
	.ResolvedVersion = .Version
	 := .Time

	var  int64
	if zipLoadShedder != nil {
		var  error
		,  = getZipSize(, .ModulePath, .ResolvedVersion, )
		if  != nil {
			return nil,
Load shed or mark module as too large. We treat zip size as a proxy for the total memory consumed by processing a module, and use it to decide whether we can currently afford to process a module.
		,  := zipLoadShedder.shouldShed(uint64())
		.Defer = 
		if  {
			stats.Record(, fetchesShedded.M(1))
			return nil, fmt.Errorf("%w: size=%dMi", derrors.SheddingLoad, /mib)
		}
		if  > maxModuleZipSize {
			log.Warningf(, "FetchModule: %s@%s zip size %dMi exceeds max %dMi",
				.ModulePath, .ResolvedVersion, /mib, maxModuleZipSize/mib)
			return nil, derrors.ModuleTooLarge
		}
	}
Proceed with the fetch.
	 := &FetchInfo{
		ModulePath: .ModulePath,
		Version:    .ResolvedVersion,
		ZipSize:    uint64(),
		Start:      time.Now(),
	}
	startFetchInfo()

	var  *zip.Reader
	if .ModulePath == stdlib.ModulePath {
		var  string
		, , ,  = stdlib.Zip(.RequestedVersion)
		if  != nil {
			return ,
If the requested version is a branch name like "master" or "main", we cannot determine the right resolved version until we start working with the repo.
		.ResolvedVersion = 
		.Version = 
	} else {
		,  = .Zip(, .ModulePath, .ResolvedVersion)
		if  != nil {
			return , 
		}
	}
Set fr.HasGoMod as early as possible, because the go command uses it to decide the latest version in some cases (see fetchRawLatestVersion in this package) and all it requires is a valid zip.
	if .ModulePath == stdlib.ModulePath {
		.HasGoMod = true
	} else {
		.HasGoMod = hasGoModFile(, .ModulePath, .ResolvedVersion)
	}
getGoModPath may return a non-empty goModPath even if the error is non-nil, if the module version is an alternative module.
	var  []byte
	.GoModPath, ,  = getGoModPath(, .ModulePath, .ResolvedVersion, )
	if  != nil {
		return , 
	}

	, ,  := processZipFile(, .ModulePath, .ResolvedVersion, , , )
	if  != nil {
		return , 
	}
	.HasGoMod = .HasGoMod
	if  != nil {
		if  := processGoModFile(, );  != nil {
			return , fmt.Errorf("%v: %w", .Error(), derrors.BadModule)
		}
	}
	.Module = 
	.PackageVersionStates = 
	for ,  := range .PackageVersionStates {
		if .Status != http.StatusOK {
			.Status = derrors.ToStatus(derrors.HasIncompletePackages)
		}
	}
	return , nil
}
GetInfo returns the result of a request to the proxy .info endpoint. If the modulePath is "std", a request to @master will return an empty commit time.
func ( context.Context, ,  string,  *proxy.Client) ( *proxy.VersionInfo,  error) {
	if  == stdlib.ModulePath {
		var  string
		,  = stdlib.ZipInfo()
		if  != nil {
			return nil, 
		}
		return &proxy.VersionInfo{Version: }, nil
	}
	return .Info(, , )
}

func ( context.Context, ,  string,  *proxy.Client) ( int64,  error) {
	if  == stdlib.ModulePath {
		return stdlib.EstimatedZipSize, nil
	}
	return .ZipSize(, , )
}
getGoModPath returns the module path from the go.mod file, as well as the contents of the file obtained from the proxy. If modulePath is the standardl library, then the contents will be nil.
func ( context.Context, ,  string,  *proxy.Client) (string, []byte, error) {
	if  == stdlib.ModulePath {
		return stdlib.ModulePath, nil, nil
	}
	,  := .Mod(, , )
	if  != nil {
		return "", nil, 
	}
	 := modfile.ModulePath()
	if  == "" {
		return "", nil, fmt.Errorf("go.mod has no module path: %w", derrors.BadModule)
	}
The module path in the go.mod file doesn't match the path of the zip file. Don't insert the module. Store an AlternativeModule status in module_version_states.
		return , , fmt.Errorf("module path=%s, go.mod path=%s: %w", , , derrors.AlternativeModule)
	}
	return , , nil
}
processZipFile extracts information from the module version zip.
func ( context.Context,  string,  string,  time.Time,  *zip.Reader,  *source.Client) ( *internal.Module,  []*internal.PackageVersionState,  error) {
	defer derrors.Wrap(&, "processZipFile(%q, %q)", , )

	,  := trace.StartSpan(, "fetch.processZipFile")
	defer .End()

	,  := source.ModuleInfo(, , , )
	if  != nil {
		log.Infof(, "error getting source info: %v", )
	}
	,  := extractReadmesFromZip(, , )
	if  != nil {
		return nil, nil, fmt.Errorf("extractReadmesFromZip(%q, %q, zipReader): %v", , , )
	}
	 := func( string,  ...interface{}) {
		log.Infof(, , ...)
	}
	 := licenses.NewDetector(, , , )
	 := .AllLicenses()
	, ,  := extractPackagesFromZip(, , , , , )
	if errors.Is(, ErrModuleContainsNoPackages) || errors.Is(, errMalformedZip) {
		return nil, nil, fmt.Errorf("%v: %w", .Error(), derrors.BadModule)
	}
	if  != nil {
		return nil, nil, fmt.Errorf("extractPackagesFromZip(%q, %q, zipReader, %v): %v", , , , )
	}
	return &internal.Module{
		ModuleInfo: internal.ModuleInfo{
			ModulePath:        ,
			Version:           ,
			CommitTime:        ,
			IsRedistributable: .ModuleIsRedistributable(),
HasGoMod is populated by the caller.
		},
		Licenses: ,
		Units:    moduleUnits(, , , , ),
	}, , nil
}

func ( *zip.Reader, ,  string) bool {
	return zipFile(, path.Join(moduleVersionDir(, ), "go.mod")) != nil
}
processGoModFile populates mod with information extracted from the contents of the go.mod file.
func ( []byte,  *internal.Module) ( error) {
	defer derrors.Wrap(&, "processGoModFile")

	,  := modfile.Parse("go.mod", , nil)
	if  != nil {
		return 
	}
	.Deprecated, .DeprecationComment = extractDeprecatedComment()
	return nil
}
extractDeprecatedComment looks for "Deprecated" comments in the line comments before the module declaration. If it finds one, it returns true along with the text after "Deprecated:". Otherwise it returns false, "".
func ( *modfile.File) (bool, string) {
	const  = "Deprecated:"

	if .Module == nil {
		return false, ""
	}
	for ,  := range append(.Module.Syntax.Before, .Module.Syntax.Suffix...) {
		 := strings.TrimSpace(strings.TrimPrefix(.Token, "//"))
		if strings.HasPrefix(, ) {
			return true, strings.TrimSpace([len():])
		}
	}
	return false, ""
}
moduleVersionDir formats the content subdirectory for the given modulePath and version.
func (,  string) string {
	return fmt.Sprintf("%s@%s", , )
}
zipFile returns the file in r whose name matches the given name, or nil if there isn't one.
func ( *zip.Reader,  string) *zip.File {
	for ,  := range .File {
		if .Name ==  {
			return 
		}
	}
	return nil
}

type FetchInfo struct {
	ModulePath string
	Version    string
	ZipSize    uint64
	Start      time.Time
	Finish     time.Time
	Status     int
	Error      error
}

var (
	fetchInfoMu  sync.Mutex
	fetchInfoMap = map[*FetchInfo]struct{}{}
)

func () {
	const  = time.Minute
	go func() {
		for {
			 := time.Now()
			fetchInfoMu.Lock()
			for  := range fetchInfoMap {
				if !.Finish.IsZero() && .Sub(.Finish) >  {
					delete(fetchInfoMap, )
				}
			}
			fetchInfoMu.Unlock()
			time.Sleep()
		}
	}()
}

func ( *FetchInfo) {
	fetchInfoMu.Lock()
	defer fetchInfoMu.Unlock()
	fetchInfoMap[] = struct{}{}
}

func ( *FetchInfo,  int,  error) {
	fetchInfoMu.Lock()
	defer fetchInfoMu.Unlock()
	.Finish = time.Now()
	.Status = 
	.Error = 
}
FetchInfos returns information about all fetches in progress, sorted by start time.
func () []*FetchInfo {
	var  []*FetchInfo
	fetchInfoMu.Lock()
Copy to avoid races on Status and Error when read by worker home page.
		 := *
		 = append(, &)
	}
Order first by done-ness, then by age.
	sort.Slice(, func(,  int) bool {
		if ([].Status == 0) == ([].Status == 0) {
			return [].Start.Before([].Start)
		}
		return [].Status == 0
	})
	return