1// Copyright 2018 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5package packages
6
7// See doc.go for package documentation and implementation notes.
8
9import (
10	"context"
11	"encoding/json"
12	"fmt"
13	"go/ast"
14	"go/parser"
15	"go/scanner"
16	"go/token"
17	"go/types"
18	"io/ioutil"
19	"log"
20	"os"
21	"path/filepath"
22	"sync"
23
24	"golang.org/x/tools/go/gcexportdata"
25)
26
27// A LoadMode specifies the amount of detail to return when loading.
28// Higher-numbered modes cause Load to return more information,
29// but may be slower. Load may return more information than requested.
30type LoadMode int
31
32const (
33	// LoadFiles finds the packages and computes their source file lists.
34	// Package fields: ID, Name, Errors, GoFiles, and OtherFiles.
35	LoadFiles LoadMode = iota
36
37	// LoadImports adds import information for each package
38	// and its dependencies.
39	// Package fields added: Imports.
40	LoadImports
41
42	// LoadTypes adds type information for package-level
43	// declarations in the packages matching the patterns.
44	// Package fields added: Types, Fset, and IllTyped.
45	// This mode uses type information provided by the build system when
46	// possible, and may fill in the ExportFile field.
47	LoadTypes
48
49	// LoadSyntax adds typed syntax trees for the packages matching the patterns.
50	// Package fields added: Syntax, and TypesInfo, for direct pattern matches only.
51	LoadSyntax
52
53	// LoadAllSyntax adds typed syntax trees for the packages matching the patterns
54	// and all dependencies.
55	// Package fields added: Types, Fset, Illtyped, Syntax, and TypesInfo,
56	// for all packages in the import graph.
57	LoadAllSyntax
58)
59
60// An Config specifies details about how packages should be loaded.
61// The zero value is a valid configuration.
62// Calls to Load do not modify this struct.
63type Config struct {
64	// Mode controls the level of information returned for each package.
65	Mode LoadMode
66
67	// Context specifies the context for the load operation.
68	// If the context is cancelled, the loader may stop early
69	// and return an ErrCancelled error.
70	// If Context is nil, the load cannot be cancelled.
71	Context context.Context
72
73	// Dir is the directory in which to run the build system's query tool
74	// that provides information about the packages.
75	// If Dir is empty, the tool is run in the current directory.
76	Dir string
77
78	// Env is the environment to use when invoking the build system's query tool.
79	// If Env is nil, the current environment is used.
80	// As in os/exec's Cmd, only the last value in the slice for
81	// each environment key is used. To specify the setting of only
82	// a few variables, append to the current environment, as in:
83	//
84	//	opt.Env = append(os.Environ(), "GOOS=plan9", "GOARCH=386")
85	//
86	Env []string
87
88	// BuildFlags is a list of command-line flags to be passed through to
89	// the build system's query tool.
90	BuildFlags []string
91
92	// Fset provides source position information for syntax trees and types.
93	// If Fset is nil, the loader will create a new FileSet.
94	Fset *token.FileSet
95
96	// ParseFile is called to read and parse each file
97	// when preparing a package's type-checked syntax tree.
98	// It must be safe to call ParseFile simultaneously from multiple goroutines.
99	// If ParseFile is nil, the loader will uses parser.ParseFile.
100	//
101	// ParseFile should parse the source from src and use filename only for
102	// recording position information.
103	//
104	// An application may supply a custom implementation of ParseFile
105	// to change the effective file contents or the behavior of the parser,
106	// or to modify the syntax tree. For example, selectively eliminating
107	// unwanted function bodies can significantly accelerate type checking.
108	ParseFile func(fset *token.FileSet, filename string, src []byte) (*ast.File, error)
109
110	// If Tests is set, the loader includes not just the packages
111	// matching a particular pattern but also any related test packages,
112	// including test-only variants of the package and the test executable.
113	//
114	// For example, when using the go command, loading "fmt" with Tests=true
115	// returns four packages, with IDs "fmt" (the standard package),
116	// "fmt [fmt.test]" (the package as compiled for the test),
117	// "fmt_test" (the test functions from source files in package fmt_test),
118	// and "fmt.test" (the test binary).
119	//
120	// In build systems with explicit names for tests,
121	// setting Tests may have no effect.
122	Tests bool
123
124	// Overlay provides a mapping of absolute file paths to file contents.
125	// If the file  with the given path already exists, the parser will use the
126	// alternative file contents provided by the map.
127	//
128	// The Package.Imports map may not include packages that are imported only
129	// by the alternative file contents provided by Overlay. This may cause
130	// type-checking to fail.
131	Overlay map[string][]byte
132}
133
134// driver is the type for functions that query the build system for the
135// packages named by the patterns.
136type driver func(cfg *Config, patterns ...string) (*driverResponse, error)
137
138// driverResponse contains the results for a driver query.
139type driverResponse struct {
140	// Sizes, if not nil, is the types.Sizes to use when type checking.
141	Sizes *types.StdSizes
142
143	// Roots is the set of package IDs that make up the root packages.
144	// We have to encode this separately because when we encode a single package
145	// we cannot know if it is one of the roots as that requires knowledge of the
146	// graph it is part of.
147	Roots []string `json:",omitempty"`
148
149	// Packages is the full set of packages in the graph.
150	// The packages are not connected into a graph.
151	// The Imports if populated will be stubs that only have their ID set.
152	// Imports will be connected and then type and syntax information added in a
153	// later pass (see refine).
154	Packages []*Package
155}
156
157// Load loads and returns the Go packages named by the given patterns.
158//
159// Config specifies loading options;
160// nil behaves the same as an empty Config.
161//
162// Load returns an error if any of the patterns was invalid
163// as defined by the underlying build system.
164// It may return an empty list of packages without an error,
165// for instance for an empty expansion of a valid wildcard.
166// Errors associated with a particular package are recorded in the
167// corresponding Package's Errors list, and do not cause Load to
168// return an error. Clients may need to handle such errors before
169// proceeding with further analysis. The PrintErrors function is
170// provided for convenient display of all errors.
171func Load(cfg *Config, patterns ...string) ([]*Package, error) {
172	l := newLoader(cfg)
173	response, err := defaultDriver(&l.Config, patterns...)
174	if err != nil {
175		return nil, err
176	}
177	l.sizes = response.Sizes
178	return l.refine(response.Roots, response.Packages...)
179}
180
181// defaultDriver is a driver that looks for an external driver binary, and if
182// it does not find it falls back to the built in go list driver.
183func defaultDriver(cfg *Config, patterns ...string) (*driverResponse, error) {
184	driver := findExternalDriver(cfg)
185	if driver == nil {
186		driver = goListDriver
187	}
188	return driver(cfg, patterns...)
189}
190
191// A Package describes a loaded Go package.
192type Package struct {
193	// ID is a unique identifier for a package,
194	// in a syntax provided by the underlying build system.
195	//
196	// Because the syntax varies based on the build system,
197	// clients should treat IDs as opaque and not attempt to
198	// interpret them.
199	ID string
200
201	// Name is the package name as it appears in the package source code.
202	Name string
203
204	// PkgPath is the package path as used by the go/types package.
205	PkgPath string
206
207	// Errors contains any errors encountered querying the metadata
208	// of the package, or while parsing or type-checking its files.
209	Errors []Error
210
211	// GoFiles lists the absolute file paths of the package's Go source files.
212	GoFiles []string
213
214	// CompiledGoFiles lists the absolute file paths of the package's source
215	// files that were presented to the compiler.
216	// This may differ from GoFiles if files are processed before compilation.
217	CompiledGoFiles []string
218
219	// OtherFiles lists the absolute file paths of the package's non-Go source files,
220	// including assembly, C, C++, Fortran, Objective-C, SWIG, and so on.
221	OtherFiles []string
222
223	// ExportFile is the absolute path to a file containing type
224	// information for the package as provided by the build system.
225	ExportFile string
226
227	// Imports maps import paths appearing in the package's Go source files
228	// to corresponding loaded Packages.
229	Imports map[string]*Package
230
231	// Types provides type information for the package.
232	// Modes LoadTypes and above set this field for packages matching the
233	// patterns; type information for dependencies may be missing or incomplete.
234	// Mode LoadAllSyntax sets this field for all packages, including dependencies.
235	Types *types.Package
236
237	// Fset provides position information for Types, TypesInfo, and Syntax.
238	// It is set only when Types is set.
239	Fset *token.FileSet
240
241	// IllTyped indicates whether the package or any dependency contains errors.
242	// It is set only when Types is set.
243	IllTyped bool
244
245	// Syntax is the package's syntax trees, for the files listed in CompiledGoFiles.
246	//
247	// Mode LoadSyntax sets this field for packages matching the patterns.
248	// Mode LoadAllSyntax sets this field for all packages, including dependencies.
249	Syntax []*ast.File
250
251	// TypesInfo provides type information about the package's syntax trees.
252	// It is set only when Syntax is set.
253	TypesInfo *types.Info
254}
255
256// An Error describes a problem with a package's metadata, syntax, or types.
257type Error struct {
258	Pos  string // "file:line:col" or "file:line" or "" or "-"
259	Msg  string
260	Kind ErrorKind
261}
262
263// ErrorKind describes the source of the error, allowing the user to
264// differentiate between errors generated by the driver, the parser, or the
265// type-checker.
266type ErrorKind int
267
268const (
269	UnknownError ErrorKind = iota
270	ListError
271	ParseError
272	TypeError
273)
274
275func (err Error) Error() string {
276	pos := err.Pos
277	if pos == "" {
278		pos = "-" // like token.Position{}.String()
279	}
280	return pos + ": " + err.Msg
281}
282
283// flatPackage is the JSON form of Package
284// It drops all the type and syntax fields, and transforms the Imports
285//
286// TODO(adonovan): identify this struct with Package, effectively
287// publishing the JSON protocol.
288type flatPackage struct {
289	ID              string
290	Name            string            `json:",omitempty"`
291	PkgPath         string            `json:",omitempty"`
292	Errors          []Error           `json:",omitempty"`
293	GoFiles         []string          `json:",omitempty"`
294	CompiledGoFiles []string          `json:",omitempty"`
295	OtherFiles      []string          `json:",omitempty"`
296	ExportFile      string            `json:",omitempty"`
297	Imports         map[string]string `json:",omitempty"`
298}
299
300// MarshalJSON returns the Package in its JSON form.
301// For the most part, the structure fields are written out unmodified, and
302// the type and syntax fields are skipped.
303// The imports are written out as just a map of path to package id.
304// The errors are written using a custom type that tries to preserve the
305// structure of error types we know about.
306//
307// This method exists to enable support for additional build systems.  It is
308// not intended for use by clients of the API and we may change the format.
309func (p *Package) MarshalJSON() ([]byte, error) {
310	flat := &flatPackage{
311		ID:              p.ID,
312		Name:            p.Name,
313		PkgPath:         p.PkgPath,
314		Errors:          p.Errors,
315		GoFiles:         p.GoFiles,
316		CompiledGoFiles: p.CompiledGoFiles,
317		OtherFiles:      p.OtherFiles,
318		ExportFile:      p.ExportFile,
319	}
320	if len(p.Imports) > 0 {
321		flat.Imports = make(map[string]string, len(p.Imports))
322		for path, ipkg := range p.Imports {
323			flat.Imports[path] = ipkg.ID
324		}
325	}
326	return json.Marshal(flat)
327}
328
329// UnmarshalJSON reads in a Package from its JSON format.
330// See MarshalJSON for details about the format accepted.
331func (p *Package) UnmarshalJSON(b []byte) error {
332	flat := &flatPackage{}
333	if err := json.Unmarshal(b, &flat); err != nil {
334		return err
335	}
336	*p = Package{
337		ID:              flat.ID,
338		Name:            flat.Name,
339		PkgPath:         flat.PkgPath,
340		Errors:          flat.Errors,
341		GoFiles:         flat.GoFiles,
342		CompiledGoFiles: flat.CompiledGoFiles,
343		OtherFiles:      flat.OtherFiles,
344		ExportFile:      flat.ExportFile,
345	}
346	if len(flat.Imports) > 0 {
347		p.Imports = make(map[string]*Package, len(flat.Imports))
348		for path, id := range flat.Imports {
349			p.Imports[path] = &Package{ID: id}
350		}
351	}
352	return nil
353}
354
355func (p *Package) String() string { return p.ID }
356
357// loaderPackage augments Package with state used during the loading phase
358type loaderPackage struct {
359	*Package
360	importErrors map[string]error // maps each bad import to its error
361	loadOnce     sync.Once
362	color        uint8 // for cycle detection
363	needsrc      bool  // load from source (Mode >= LoadTypes)
364	needtypes    bool  // type information is either requested or depended on
365	initial      bool  // package was matched by a pattern
366}
367
368// loader holds the working state of a single call to load.
369type loader struct {
370	pkgs map[string]*loaderPackage
371	Config
372	sizes    types.Sizes
373	exportMu sync.Mutex // enforces mutual exclusion of exportdata operations
374}
375
376func newLoader(cfg *Config) *loader {
377	ld := &loader{}
378	if cfg != nil {
379		ld.Config = *cfg
380	}
381	if ld.Config.Env == nil {
382		ld.Config.Env = os.Environ()
383	}
384	if ld.Context == nil {
385		ld.Context = context.Background()
386	}
387	if ld.Dir == "" {
388		if dir, err := os.Getwd(); err == nil {
389			ld.Dir = dir
390		}
391	}
392
393	if ld.Mode >= LoadTypes {
394		if ld.Fset == nil {
395			ld.Fset = token.NewFileSet()
396		}
397
398		// ParseFile is required even in LoadTypes mode
399		// because we load source if export data is missing.
400		if ld.ParseFile == nil {
401			ld.ParseFile = func(fset *token.FileSet, filename string, src []byte) (*ast.File, error) {
402				var isrc interface{}
403				if src != nil {
404					isrc = src
405				}
406				const mode = parser.AllErrors | parser.ParseComments
407				return parser.ParseFile(fset, filename, isrc, mode)
408			}
409		}
410	}
411	return ld
412}
413
414// refine connects the supplied packages into a graph and then adds type and
415// and syntax information as requested by the LoadMode.
416func (ld *loader) refine(roots []string, list ...*Package) ([]*Package, error) {
417	rootMap := make(map[string]int, len(roots))
418	for i, root := range roots {
419		rootMap[root] = i
420	}
421	ld.pkgs = make(map[string]*loaderPackage)
422	// first pass, fixup and build the map and roots
423	var initial = make([]*loaderPackage, len(roots))
424	for _, pkg := range list {
425		rootIndex := -1
426		if i, found := rootMap[pkg.ID]; found {
427			rootIndex = i
428		}
429		lpkg := &loaderPackage{
430			Package: pkg,
431			needtypes: ld.Mode >= LoadAllSyntax ||
432				ld.Mode >= LoadTypes && rootIndex >= 0,
433			needsrc: ld.Mode >= LoadAllSyntax ||
434				ld.Mode >= LoadSyntax && rootIndex >= 0 ||
435				pkg.ExportFile == "" && pkg.PkgPath != "unsafe",
436		}
437		ld.pkgs[lpkg.ID] = lpkg
438		if rootIndex >= 0 {
439			initial[rootIndex] = lpkg
440			lpkg.initial = true
441		}
442	}
443	for i, root := range roots {
444		if initial[i] == nil {
445			return nil, fmt.Errorf("root package %v is missing", root)
446		}
447	}
448
449	// Materialize the import graph.
450
451	const (
452		white = 0 // new
453		grey  = 1 // in progress
454		black = 2 // complete
455	)
456
457	// visit traverses the import graph, depth-first,
458	// and materializes the graph as Packages.Imports.
459	//
460	// Valid imports are saved in the Packages.Import map.
461	// Invalid imports (cycles and missing nodes) are saved in the importErrors map.
462	// Thus, even in the presence of both kinds of errors, the Import graph remains a DAG.
463	//
464	// visit returns whether the package needs src or has a transitive
465	// dependency on a package that does. These are the only packages
466	// for which we load source code.
467	var stack []*loaderPackage
468	var visit func(lpkg *loaderPackage) bool
469	var srcPkgs []*loaderPackage
470	visit = func(lpkg *loaderPackage) bool {
471		switch lpkg.color {
472		case black:
473			return lpkg.needsrc
474		case grey:
475			panic("internal error: grey node")
476		}
477		lpkg.color = grey
478		stack = append(stack, lpkg) // push
479		stubs := lpkg.Imports       // the structure form has only stubs with the ID in the Imports
480		lpkg.Imports = make(map[string]*Package, len(stubs))
481		for importPath, ipkg := range stubs {
482			var importErr error
483			imp := ld.pkgs[ipkg.ID]
484			if imp == nil {
485				// (includes package "C" when DisableCgo)
486				importErr = fmt.Errorf("missing package: %q", ipkg.ID)
487			} else if imp.color == grey {
488				importErr = fmt.Errorf("import cycle: %s", stack)
489			}
490			if importErr != nil {
491				if lpkg.importErrors == nil {
492					lpkg.importErrors = make(map[string]error)
493				}
494				lpkg.importErrors[importPath] = importErr
495				continue
496			}
497
498			if visit(imp) {
499				lpkg.needsrc = true
500			}
501			lpkg.Imports[importPath] = imp.Package
502		}
503		if lpkg.needsrc {
504			srcPkgs = append(srcPkgs, lpkg)
505		}
506		stack = stack[:len(stack)-1] // pop
507		lpkg.color = black
508
509		return lpkg.needsrc
510	}
511
512	if ld.Mode < LoadImports {
513		//we do this to drop the stub import packages that we are not even going to try to resolve
514		for _, lpkg := range initial {
515			lpkg.Imports = nil
516		}
517	} else {
518		// For each initial package, create its import DAG.
519		for _, lpkg := range initial {
520			visit(lpkg)
521		}
522	}
523	for _, lpkg := range srcPkgs {
524		// Complete type information is required for the
525		// immediate dependencies of each source package.
526		for _, ipkg := range lpkg.Imports {
527			imp := ld.pkgs[ipkg.ID]
528			imp.needtypes = true
529		}
530	}
531	// Load type data if needed, starting at
532	// the initial packages (roots of the import DAG).
533	if ld.Mode >= LoadTypes {
534		var wg sync.WaitGroup
535		for _, lpkg := range initial {
536			wg.Add(1)
537			go func(lpkg *loaderPackage) {
538				ld.loadRecursive(lpkg)
539				wg.Done()
540			}(lpkg)
541		}
542		wg.Wait()
543	}
544
545	result := make([]*Package, len(initial))
546	for i, lpkg := range initial {
547		result[i] = lpkg.Package
548	}
549	return result, nil
550}
551
552// loadRecursive loads the specified package and its dependencies,
553// recursively, in parallel, in topological order.
554// It is atomic and idempotent.
555// Precondition: ld.Mode >= LoadTypes.
556func (ld *loader) loadRecursive(lpkg *loaderPackage) {
557	lpkg.loadOnce.Do(func() {
558		// Load the direct dependencies, in parallel.
559		var wg sync.WaitGroup
560		for _, ipkg := range lpkg.Imports {
561			imp := ld.pkgs[ipkg.ID]
562			wg.Add(1)
563			go func(imp *loaderPackage) {
564				ld.loadRecursive(imp)
565				wg.Done()
566			}(imp)
567		}
568		wg.Wait()
569
570		ld.loadPackage(lpkg)
571	})
572}
573
574// loadPackage loads the specified package.
575// It must be called only once per Package,
576// after immediate dependencies are loaded.
577// Precondition: ld.Mode >= LoadTypes.
578func (ld *loader) loadPackage(lpkg *loaderPackage) {
579	if lpkg.PkgPath == "unsafe" {
580		// Fill in the blanks to avoid surprises.
581		lpkg.Types = types.Unsafe
582		lpkg.Fset = ld.Fset
583		lpkg.Syntax = []*ast.File{}
584		lpkg.TypesInfo = new(types.Info)
585		return
586	}
587
588	// Call NewPackage directly with explicit name.
589	// This avoids skew between golist and go/types when the files'
590	// package declarations are inconsistent.
591	lpkg.Types = types.NewPackage(lpkg.PkgPath, lpkg.Name)
592	lpkg.Fset = ld.Fset
593
594	// Subtle: we populate all Types fields with an empty Package
595	// before loading export data so that export data processing
596	// never has to create a types.Package for an indirect dependency,
597	// which would then require that such created packages be explicitly
598	// inserted back into the Import graph as a final step after export data loading.
599	// The Diamond test exercises this case.
600	if !lpkg.needtypes {
601		return
602	}
603	if !lpkg.needsrc {
604		ld.loadFromExportData(lpkg)
605		return // not a source package, don't get syntax trees
606	}
607
608	appendError := func(err error) {
609		// Convert various error types into the one true Error.
610		var errs []Error
611		switch err := err.(type) {
612		case Error:
613			// from driver
614			errs = append(errs, err)
615
616		case *os.PathError:
617			// from parser
618			errs = append(errs, Error{
619				Pos:  err.Path + ":1",
620				Msg:  err.Err.Error(),
621				Kind: ParseError,
622			})
623
624		case scanner.ErrorList:
625			// from parser
626			for _, err := range err {
627				errs = append(errs, Error{
628					Pos:  err.Pos.String(),
629					Msg:  err.Msg,
630					Kind: ParseError,
631				})
632			}
633
634		case types.Error:
635			// from type checker
636			errs = append(errs, Error{
637				Pos:  err.Fset.Position(err.Pos).String(),
638				Msg:  err.Msg,
639				Kind: TypeError,
640			})
641
642		default:
643			// unexpected impoverished error from parser?
644			errs = append(errs, Error{
645				Pos:  "-",
646				Msg:  err.Error(),
647				Kind: UnknownError,
648			})
649
650			// If you see this error message, please file a bug.
651			log.Printf("internal error: error %q (%T) without position", err, err)
652		}
653
654		lpkg.Errors = append(lpkg.Errors, errs...)
655	}
656
657	files, errs := ld.parseFiles(lpkg.CompiledGoFiles)
658	for _, err := range errs {
659		appendError(err)
660	}
661
662	lpkg.Syntax = files
663
664	lpkg.TypesInfo = &types.Info{
665		Types:      make(map[ast.Expr]types.TypeAndValue),
666		Defs:       make(map[*ast.Ident]types.Object),
667		Uses:       make(map[*ast.Ident]types.Object),
668		Implicits:  make(map[ast.Node]types.Object),
669		Scopes:     make(map[ast.Node]*types.Scope),
670		Selections: make(map[*ast.SelectorExpr]*types.Selection),
671	}
672
673	importer := importerFunc(func(path string) (*types.Package, error) {
674		if path == "unsafe" {
675			return types.Unsafe, nil
676		}
677
678		// The imports map is keyed by import path.
679		ipkg := lpkg.Imports[path]
680		if ipkg == nil {
681			if err := lpkg.importErrors[path]; err != nil {
682				return nil, err
683			}
684			// There was skew between the metadata and the
685			// import declarations, likely due to an edit
686			// race, or because the ParseFile feature was
687			// used to supply alternative file contents.
688			return nil, fmt.Errorf("no metadata for %s", path)
689		}
690
691		if ipkg.Types != nil && ipkg.Types.Complete() {
692			return ipkg.Types, nil
693		}
694		log.Fatalf("internal error: nil Pkg importing %q from %q", path, lpkg)
695		panic("unreachable")
696	})
697
698	// type-check
699	tc := &types.Config{
700		Importer: importer,
701
702		// Type-check bodies of functions only in non-initial packages.
703		// Example: for import graph A->B->C and initial packages {A,C},
704		// we can ignore function bodies in B.
705		IgnoreFuncBodies: ld.Mode < LoadAllSyntax && !lpkg.initial,
706
707		Error: appendError,
708		Sizes: ld.sizes,
709	}
710	types.NewChecker(tc, ld.Fset, lpkg.Types, lpkg.TypesInfo).Files(lpkg.Syntax)
711
712	lpkg.importErrors = nil // no longer needed
713
714	// If !Cgo, the type-checker uses FakeImportC mode, so
715	// it doesn't invoke the importer for import "C",
716	// nor report an error for the import,
717	// or for any undefined C.f reference.
718	// We must detect this explicitly and correctly
719	// mark the package as IllTyped (by reporting an error).
720	// TODO(adonovan): if these errors are annoying,
721	// we could just set IllTyped quietly.
722	if tc.FakeImportC {
723	outer:
724		for _, f := range lpkg.Syntax {
725			for _, imp := range f.Imports {
726				if imp.Path.Value == `"C"` {
727					err := types.Error{Fset: ld.Fset, Pos: imp.Pos(), Msg: `import "C" ignored`}
728					appendError(err)
729					break outer
730				}
731			}
732		}
733	}
734
735	// Record accumulated errors.
736	illTyped := len(lpkg.Errors) > 0
737	if !illTyped {
738		for _, imp := range lpkg.Imports {
739			if imp.IllTyped {
740				illTyped = true
741				break
742			}
743		}
744	}
745	lpkg.IllTyped = illTyped
746}
747
748// An importFunc is an implementation of the single-method
749// types.Importer interface based on a function value.
750type importerFunc func(path string) (*types.Package, error)
751
752func (f importerFunc) Import(path string) (*types.Package, error) { return f(path) }
753
754// We use a counting semaphore to limit
755// the number of parallel I/O calls per process.
756var ioLimit = make(chan bool, 20)
757
758// parseFiles reads and parses the Go source files and returns the ASTs
759// of the ones that could be at least partially parsed, along with a
760// list of I/O and parse errors encountered.
761//
762// Because files are scanned in parallel, the token.Pos
763// positions of the resulting ast.Files are not ordered.
764//
765func (ld *loader) parseFiles(filenames []string) ([]*ast.File, []error) {
766	var wg sync.WaitGroup
767	n := len(filenames)
768	parsed := make([]*ast.File, n)
769	errors := make([]error, n)
770	for i, file := range filenames {
771		wg.Add(1)
772		go func(i int, filename string) {
773			ioLimit <- true // wait
774			// ParseFile may return both an AST and an error.
775			var src []byte
776			for f, contents := range ld.Config.Overlay {
777				if sameFile(f, filename) {
778					src = contents
779				}
780			}
781			var err error
782			if src == nil {
783				src, err = ioutil.ReadFile(filename)
784			}
785			if err != nil {
786				parsed[i], errors[i] = nil, err
787			} else {
788				parsed[i], errors[i] = ld.ParseFile(ld.Fset, filename, src)
789			}
790			<-ioLimit // signal
791			wg.Done()
792		}(i, file)
793	}
794	wg.Wait()
795
796	// Eliminate nils, preserving order.
797	var o int
798	for _, f := range parsed {
799		if f != nil {
800			parsed[o] = f
801			o++
802		}
803	}
804	parsed = parsed[:o]
805
806	o = 0
807	for _, err := range errors {
808		if err != nil {
809			errors[o] = err
810			o++
811		}
812	}
813	errors = errors[:o]
814
815	return parsed, errors
816}
817
818// sameFile returns true if x and y have the same basename and denote
819// the same file.
820//
821func sameFile(x, y string) bool {
822	if filepath.Base(x) == filepath.Base(y) { // (optimisation)
823		if xi, err := os.Stat(x); err == nil {
824			if yi, err := os.Stat(y); err == nil {
825				return os.SameFile(xi, yi)
826			}
827		}
828	}
829	return false
830}
831
832// loadFromExportData returns type information for the specified
833// package, loading it from an export data file on the first request.
834func (ld *loader) loadFromExportData(lpkg *loaderPackage) (*types.Package, error) {
835	if lpkg.PkgPath == "" {
836		log.Fatalf("internal error: Package %s has no PkgPath", lpkg)
837	}
838
839	// Because gcexportdata.Read has the potential to create or
840	// modify the types.Package for each node in the transitive
841	// closure of dependencies of lpkg, all exportdata operations
842	// must be sequential. (Finer-grained locking would require
843	// changes to the gcexportdata API.)
844	//
845	// The exportMu lock guards the Package.Pkg field and the
846	// types.Package it points to, for each Package in the graph.
847	//
848	// Not all accesses to Package.Pkg need to be protected by exportMu:
849	// graph ordering ensures that direct dependencies of source
850	// packages are fully loaded before the importer reads their Pkg field.
851	ld.exportMu.Lock()
852	defer ld.exportMu.Unlock()
853
854	if tpkg := lpkg.Types; tpkg != nil && tpkg.Complete() {
855		return tpkg, nil // cache hit
856	}
857
858	lpkg.IllTyped = true // fail safe
859
860	if lpkg.ExportFile == "" {
861		// Errors while building export data will have been printed to stderr.
862		return nil, fmt.Errorf("no export data file")
863	}
864	f, err := os.Open(lpkg.ExportFile)
865	if err != nil {
866		return nil, err
867	}
868	defer f.Close()
869
870	// Read gc export data.
871	//
872	// We don't currently support gccgo export data because all
873	// underlying workspaces use the gc toolchain. (Even build
874	// systems that support gccgo don't use it for workspace
875	// queries.)
876	r, err := gcexportdata.NewReader(f)
877	if err != nil {
878		return nil, fmt.Errorf("reading %s: %v", lpkg.ExportFile, err)
879	}
880
881	// Build the view.
882	//
883	// The gcexportdata machinery has no concept of package ID.
884	// It identifies packages by their PkgPath, which although not
885	// globally unique is unique within the scope of one invocation
886	// of the linker, type-checker, or gcexportdata.
887	//
888	// So, we must build a PkgPath-keyed view of the global
889	// (conceptually ID-keyed) cache of packages and pass it to
890	// gcexportdata. The view must contain every existing
891	// package that might possibly be mentioned by the
892	// current package---its transitive closure.
893	//
894	// In loadPackage, we unconditionally create a types.Package for
895	// each dependency so that export data loading does not
896	// create new ones.
897	//
898	// TODO(adonovan): it would be simpler and more efficient
899	// if the export data machinery invoked a callback to
900	// get-or-create a package instead of a map.
901	//
902	view := make(map[string]*types.Package) // view seen by gcexportdata
903	seen := make(map[*loaderPackage]bool)   // all visited packages
904	var visit func(pkgs map[string]*Package)
905	visit = func(pkgs map[string]*Package) {
906		for _, p := range pkgs {
907			lpkg := ld.pkgs[p.ID]
908			if !seen[lpkg] {
909				seen[lpkg] = true
910				view[lpkg.PkgPath] = lpkg.Types
911				visit(lpkg.Imports)
912			}
913		}
914	}
915	visit(lpkg.Imports)
916
917	viewLen := len(view) + 1 // adding the self package
918	// Parse the export data.
919	// (May modify incomplete packages in view but not create new ones.)
920	tpkg, err := gcexportdata.Read(r, ld.Fset, view, lpkg.PkgPath)
921	if err != nil {
922		return nil, fmt.Errorf("reading %s: %v", lpkg.ExportFile, err)
923	}
924	if viewLen != len(view) {
925		log.Fatalf("Unexpected package creation during export data loading")
926	}
927
928	lpkg.Types = tpkg
929	lpkg.IllTyped = false
930
931	return tpkg, nil
932}
933
934func usesExportData(cfg *Config) bool {
935	return LoadTypes <= cfg.Mode && cfg.Mode < LoadAllSyntax
936}
937