1// Copyright 2018 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5package packages
6
7import (
8	"encoding/json"
9	"fmt"
10	"go/build"
11	"io/ioutil"
12	"os"
13	"os/exec"
14	"path/filepath"
15	"sort"
16	"strings"
17
18	"golang.org/x/tools/go/internal/cgo"
19)
20
21// TODO(matloob): Delete this file once Go 1.12 is released.
22
23// This file provides backwards compatibility support for
24// loading for versions of Go earlier than 1.10.4. This support is meant to
25// assist with migration to the Package API until there's
26// widespread adoption of these newer Go versions.
27// This support will be removed once Go 1.12 is released
28// in Q1 2019.
29
30func golistDriverFallback(cfg *Config, words ...string) (*driverResponse, error) {
31	// Turn absolute paths into GOROOT and GOPATH-relative paths to provide to go list.
32	// This will have surprising behavior if GOROOT or GOPATH contain multiple packages with the same
33	// path and a user provides an absolute path to a directory that's shadowed by an earlier
34	// directory in GOROOT or GOPATH with the same package path.
35	words = cleanAbsPaths(cfg, words)
36
37	original, deps, err := getDeps(cfg, words...)
38	if err != nil {
39		return nil, err
40	}
41
42	var tmpdir string // used for generated cgo files
43	var needsTestVariant []struct {
44		pkg, xtestPkg *Package
45	}
46
47	var response driverResponse
48	allPkgs := make(map[string]bool)
49	addPackage := func(p *jsonPackage) {
50		id := p.ImportPath
51
52		if allPkgs[id] {
53			return
54		}
55		allPkgs[id] = true
56
57		isRoot := original[id] != nil
58		pkgpath := id
59
60		if pkgpath == "unsafe" {
61			p.GoFiles = nil // ignore fake unsafe.go file
62		}
63
64		importMap := func(importlist []string) map[string]*Package {
65			importMap := make(map[string]*Package)
66			for _, id := range importlist {
67
68				if id == "C" {
69					for _, path := range []string{"unsafe", "syscall", "runtime/cgo"} {
70						if pkgpath != path && importMap[path] == nil {
71							importMap[path] = &Package{ID: path}
72						}
73					}
74					continue
75				}
76				importMap[vendorlessPath(id)] = &Package{ID: id}
77			}
78			return importMap
79		}
80		compiledGoFiles := absJoin(p.Dir, p.GoFiles)
81		// Use a function to simplify control flow. It's just a bunch of gotos.
82		var cgoErrors []error
83		var outdir string
84		getOutdir := func() (string, error) {
85			if outdir != "" {
86				return outdir, nil
87			}
88			if tmpdir == "" {
89				if tmpdir, err = ioutil.TempDir("", "gopackages"); err != nil {
90					return "", err
91				}
92			}
93			// Add a "go-build" component to the path to make the tests think the files are in the cache.
94			// This allows the same test to test the pre- and post-Go 1.11 go list logic because the Go 1.11
95			// go list generates test mains in the cache, and the test code knows not to rely on paths in the
96			// cache to stay stable.
97			outdir = filepath.Join(tmpdir, "go-build", strings.Replace(p.ImportPath, "/", "_", -1))
98			if err := os.MkdirAll(outdir, 0755); err != nil {
99				outdir = ""
100				return "", err
101			}
102			return outdir, nil
103		}
104		processCgo := func() bool {
105			// Suppress any cgo errors. Any relevant errors will show up in typechecking.
106			// TODO(matloob): Skip running cgo if Mode < LoadTypes.
107			outdir, err := getOutdir()
108			if err != nil {
109				cgoErrors = append(cgoErrors, err)
110				return false
111			}
112			files, _, err := runCgo(p.Dir, outdir, cfg.Env)
113			if err != nil {
114				cgoErrors = append(cgoErrors, err)
115				return false
116			}
117			compiledGoFiles = append(compiledGoFiles, files...)
118			return true
119		}
120		if len(p.CgoFiles) == 0 || !processCgo() {
121			compiledGoFiles = append(compiledGoFiles, absJoin(p.Dir, p.CgoFiles)...) // Punt to typechecker.
122		}
123		if isRoot {
124			response.Roots = append(response.Roots, id)
125		}
126		pkg := &Package{
127			ID:              id,
128			Name:            p.Name,
129			GoFiles:         absJoin(p.Dir, p.GoFiles, p.CgoFiles),
130			CompiledGoFiles: compiledGoFiles,
131			OtherFiles:      absJoin(p.Dir, otherFiles(p)...),
132			PkgPath:         pkgpath,
133			Imports:         importMap(p.Imports),
134			// TODO(matloob): set errors on the Package to cgoErrors
135		}
136		if p.Error != nil {
137			pkg.Errors = append(pkg.Errors, Error{
138				Pos: p.Error.Pos,
139				Msg: p.Error.Err,
140			})
141		}
142		response.Packages = append(response.Packages, pkg)
143		if cfg.Tests && isRoot {
144			testID := fmt.Sprintf("%s [%s.test]", id, id)
145			if len(p.TestGoFiles) > 0 || len(p.XTestGoFiles) > 0 {
146				response.Roots = append(response.Roots, testID)
147				testPkg := &Package{
148					ID:              testID,
149					Name:            p.Name,
150					GoFiles:         absJoin(p.Dir, p.GoFiles, p.CgoFiles, p.TestGoFiles),
151					CompiledGoFiles: append(compiledGoFiles, absJoin(p.Dir, p.TestGoFiles)...),
152					OtherFiles:      absJoin(p.Dir, otherFiles(p)...),
153					PkgPath:         pkgpath,
154					Imports:         importMap(append(p.Imports, p.TestImports...)),
155					// TODO(matloob): set errors on the Package to cgoErrors
156				}
157				response.Packages = append(response.Packages, testPkg)
158				var xtestPkg *Package
159				if len(p.XTestGoFiles) > 0 {
160					xtestID := fmt.Sprintf("%s_test [%s.test]", id, id)
161					response.Roots = append(response.Roots, xtestID)
162					// Generate test variants for all packages q where a path exists
163					// such that xtestPkg -> ... -> q -> ... -> p (where p is the package under test)
164					// and rewrite all import map entries of p to point to testPkg (the test variant of
165					// p), and of each q  to point to the test variant of that q.
166					xtestPkg = &Package{
167						ID:              xtestID,
168						Name:            p.Name + "_test",
169						GoFiles:         absJoin(p.Dir, p.XTestGoFiles),
170						CompiledGoFiles: absJoin(p.Dir, p.XTestGoFiles),
171						PkgPath:         pkgpath + "_test",
172						Imports:         importMap(p.XTestImports),
173					}
174					// Add to list of packages we need to rewrite imports for to refer to test variants.
175					// We may need to create a test variant of a package that hasn't been loaded yet, so
176					// the test variants need to be created later.
177					needsTestVariant = append(needsTestVariant, struct{ pkg, xtestPkg *Package }{pkg, xtestPkg})
178					response.Packages = append(response.Packages, xtestPkg)
179				}
180				// testmain package
181				testmainID := id + ".test"
182				response.Roots = append(response.Roots, testmainID)
183				imports := map[string]*Package{}
184				imports[testPkg.PkgPath] = &Package{ID: testPkg.ID}
185				if xtestPkg != nil {
186					imports[xtestPkg.PkgPath] = &Package{ID: xtestPkg.ID}
187				}
188				testmainPkg := &Package{
189					ID:      testmainID,
190					Name:    "main",
191					PkgPath: testmainID,
192					Imports: imports,
193				}
194				response.Packages = append(response.Packages, testmainPkg)
195				outdir, err := getOutdir()
196				if err != nil {
197					testmainPkg.Errors = append(testmainPkg.Errors, Error{
198						Pos:  "-",
199						Msg:  fmt.Sprintf("failed to generate testmain: %v", err),
200						Kind: ListError,
201					})
202					return
203				}
204				testmain := filepath.Join(outdir, "testmain.go")
205				extraimports, extradeps, err := generateTestmain(testmain, testPkg, xtestPkg)
206				if err != nil {
207					testmainPkg.Errors = append(testmainPkg.Errors, Error{
208						Pos:  "-",
209						Msg:  fmt.Sprintf("failed to generate testmain: %v", err),
210						Kind: ListError,
211					})
212				}
213				deps = append(deps, extradeps...)
214				for _, imp := range extraimports { // testing, testing/internal/testdeps, and maybe os
215					imports[imp] = &Package{ID: imp}
216				}
217				testmainPkg.GoFiles = []string{testmain}
218				testmainPkg.CompiledGoFiles = []string{testmain}
219			}
220		}
221	}
222
223	for _, pkg := range original {
224		addPackage(pkg)
225	}
226	if cfg.Mode < LoadImports || len(deps) == 0 {
227		return &response, nil
228	}
229
230	buf, err := invokeGo(cfg, golistArgsFallback(cfg, deps)...)
231	if err != nil {
232		return nil, err
233	}
234
235	// Decode the JSON and convert it to Package form.
236	for dec := json.NewDecoder(buf); dec.More(); {
237		p := new(jsonPackage)
238		if err := dec.Decode(p); err != nil {
239			return nil, fmt.Errorf("JSON decoding failed: %v", err)
240		}
241
242		addPackage(p)
243	}
244
245	for _, v := range needsTestVariant {
246		createTestVariants(&response, v.pkg, v.xtestPkg)
247	}
248
249	return &response, nil
250}
251
252func createTestVariants(response *driverResponse, pkgUnderTest, xtestPkg *Package) {
253	allPkgs := make(map[string]*Package)
254	for _, pkg := range response.Packages {
255		allPkgs[pkg.ID] = pkg
256	}
257	needsTestVariant := make(map[string]bool)
258	needsTestVariant[pkgUnderTest.ID] = true
259	var needsVariantRec func(p *Package) bool
260	needsVariantRec = func(p *Package) bool {
261		if needsTestVariant[p.ID] {
262			return true
263		}
264		for _, imp := range p.Imports {
265			if needsVariantRec(allPkgs[imp.ID]) {
266				// Don't break because we want to make sure all dependencies
267				// have been processed, and all required test variants of our dependencies
268				// exist.
269				needsTestVariant[p.ID] = true
270			}
271		}
272		if !needsTestVariant[p.ID] {
273			return false
274		}
275		// Create a clone of the package. It will share the same strings and lists of source files,
276		// but that's okay. It's only necessary for the Imports map to have a separate identity.
277		testVariant := *p
278		testVariant.ID = fmt.Sprintf("%s [%s.test]", p.ID, pkgUnderTest.ID)
279		testVariant.Imports = make(map[string]*Package)
280		for imp, pkg := range p.Imports {
281			testVariant.Imports[imp] = pkg
282			if needsTestVariant[pkg.ID] {
283				testVariant.Imports[imp] = &Package{ID: fmt.Sprintf("%s [%s.test]", pkg.ID, pkgUnderTest.ID)}
284			}
285		}
286		response.Packages = append(response.Packages, &testVariant)
287		return needsTestVariant[p.ID]
288	}
289	// finally, update the xtest package's imports
290	for imp, pkg := range xtestPkg.Imports {
291		if allPkgs[pkg.ID] == nil {
292			fmt.Printf("for %s: package %s doesn't exist\n", xtestPkg.ID, pkg.ID)
293		}
294		if needsVariantRec(allPkgs[pkg.ID]) {
295			xtestPkg.Imports[imp] = &Package{ID: fmt.Sprintf("%s [%s.test]", pkg.ID, pkgUnderTest.ID)}
296		}
297	}
298}
299
300// cleanAbsPaths replaces all absolute paths with GOPATH- and GOROOT-relative
301// paths. If an absolute path is not GOPATH- or GOROOT- relative, it is left as an
302// absolute path so an error can be returned later.
303func cleanAbsPaths(cfg *Config, words []string) []string {
304	var searchpaths []string
305	var cleaned = make([]string, len(words))
306	for i := range cleaned {
307		cleaned[i] = words[i]
308		// Ignore relative directory paths (they must already be goroot-relative) and Go source files
309		// (absolute source files are already allowed for ad-hoc packages).
310		// TODO(matloob): Can there be non-.go files in ad-hoc packages.
311		if !filepath.IsAbs(cleaned[i]) || strings.HasSuffix(cleaned[i], ".go") {
312			continue
313		}
314		// otherwise, it's an absolute path. Search GOPATH and GOROOT to find it.
315		if searchpaths == nil {
316			cmd := exec.Command("go", "env", "GOPATH", "GOROOT")
317			cmd.Env = cfg.Env
318			out, err := cmd.Output()
319			if err != nil {
320				searchpaths = []string{}
321				continue // suppress the error, it will show up again when running go list
322			}
323			lines := strings.Split(string(out), "\n")
324			if len(lines) != 3 || lines[0] == "" || lines[1] == "" || lines[2] != "" {
325				continue // suppress error
326			}
327			// first line is GOPATH
328			for _, path := range filepath.SplitList(lines[0]) {
329				searchpaths = append(searchpaths, filepath.Join(path, "src"))
330			}
331			// second line is GOROOT
332			searchpaths = append(searchpaths, filepath.Join(lines[1], "src"))
333		}
334		for _, sp := range searchpaths {
335			if strings.HasPrefix(cleaned[i], sp) {
336				cleaned[i] = strings.TrimPrefix(cleaned[i], sp)
337				cleaned[i] = strings.TrimLeft(cleaned[i], string(filepath.Separator))
338			}
339		}
340	}
341	return cleaned
342}
343
344// vendorlessPath returns the devendorized version of the import path ipath.
345// For example, VendorlessPath("foo/bar/vendor/a/b") returns "a/b".
346// Copied from golang.org/x/tools/imports/fix.go.
347func vendorlessPath(ipath string) string {
348	// Devendorize for use in import statement.
349	if i := strings.LastIndex(ipath, "/vendor/"); i >= 0 {
350		return ipath[i+len("/vendor/"):]
351	}
352	if strings.HasPrefix(ipath, "vendor/") {
353		return ipath[len("vendor/"):]
354	}
355	return ipath
356}
357
358// getDeps runs an initial go list to determine all the dependency packages.
359func getDeps(cfg *Config, words ...string) (originalSet map[string]*jsonPackage, deps []string, err error) {
360	buf, err := invokeGo(cfg, golistArgsFallback(cfg, words)...)
361	if err != nil {
362		return nil, nil, err
363	}
364
365	depsSet := make(map[string]bool)
366	originalSet = make(map[string]*jsonPackage)
367	var testImports []string
368
369	// Extract deps from the JSON.
370	for dec := json.NewDecoder(buf); dec.More(); {
371		p := new(jsonPackage)
372		if err := dec.Decode(p); err != nil {
373			return nil, nil, fmt.Errorf("JSON decoding failed: %v", err)
374		}
375
376		originalSet[p.ImportPath] = p
377		for _, dep := range p.Deps {
378			depsSet[dep] = true
379		}
380		if cfg.Tests {
381			// collect the additional imports of the test packages.
382			pkgTestImports := append(p.TestImports, p.XTestImports...)
383			for _, imp := range pkgTestImports {
384				if depsSet[imp] {
385					continue
386				}
387				depsSet[imp] = true
388				testImports = append(testImports, imp)
389			}
390		}
391	}
392	// Get the deps of the packages imported by tests.
393	if len(testImports) > 0 {
394		buf, err = invokeGo(cfg, golistArgsFallback(cfg, testImports)...)
395		if err != nil {
396			return nil, nil, err
397		}
398		// Extract deps from the JSON.
399		for dec := json.NewDecoder(buf); dec.More(); {
400			p := new(jsonPackage)
401			if err := dec.Decode(p); err != nil {
402				return nil, nil, fmt.Errorf("JSON decoding failed: %v", err)
403			}
404			for _, dep := range p.Deps {
405				depsSet[dep] = true
406			}
407		}
408	}
409
410	for orig := range originalSet {
411		delete(depsSet, orig)
412	}
413
414	deps = make([]string, 0, len(depsSet))
415	for dep := range depsSet {
416		deps = append(deps, dep)
417	}
418	sort.Strings(deps) // ensure output is deterministic
419	return originalSet, deps, nil
420}
421
422func golistArgsFallback(cfg *Config, words []string) []string {
423	fullargs := []string{"list", "-e", "-json"}
424	fullargs = append(fullargs, cfg.BuildFlags...)
425	fullargs = append(fullargs, "--")
426	fullargs = append(fullargs, words...)
427	return fullargs
428}
429
430func runCgo(pkgdir, tmpdir string, env []string) (files, displayfiles []string, err error) {
431	// Use go/build to open cgo files and determine the cgo flags, etc, from them.
432	// This is tricky so it's best to avoid reimplementing as much as we can, and
433	// we plan to delete this support once Go 1.12 is released anyways.
434	// TODO(matloob): This isn't completely correct because we're using the Default
435	// context. Perhaps we should more accurately fill in the context.
436	bp, err := build.ImportDir(pkgdir, build.ImportMode(0))
437	if err != nil {
438		return nil, nil, err
439	}
440	for _, ev := range env {
441		if v := strings.TrimPrefix(ev, "CGO_CPPFLAGS"); v != ev {
442			bp.CgoCPPFLAGS = append(bp.CgoCPPFLAGS, strings.Fields(v)...)
443		} else if v := strings.TrimPrefix(ev, "CGO_CFLAGS"); v != ev {
444			bp.CgoCFLAGS = append(bp.CgoCFLAGS, strings.Fields(v)...)
445		} else if v := strings.TrimPrefix(ev, "CGO_CXXFLAGS"); v != ev {
446			bp.CgoCXXFLAGS = append(bp.CgoCXXFLAGS, strings.Fields(v)...)
447		} else if v := strings.TrimPrefix(ev, "CGO_LDFLAGS"); v != ev {
448			bp.CgoLDFLAGS = append(bp.CgoLDFLAGS, strings.Fields(v)...)
449		}
450	}
451	return cgo.Run(bp, pkgdir, tmpdir, true)
452}
453