packages.go 49 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559
  1. // Copyright 2018 The Go Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style
  3. // license that can be found in the LICENSE file.
  4. package packages
  5. // See doc.go for package documentation and implementation notes.
  6. import (
  7. "context"
  8. "encoding/json"
  9. "errors"
  10. "fmt"
  11. "go/ast"
  12. "go/parser"
  13. "go/scanner"
  14. "go/token"
  15. "go/types"
  16. "log"
  17. "os"
  18. "path/filepath"
  19. "runtime"
  20. "strings"
  21. "sync"
  22. "sync/atomic"
  23. "time"
  24. "golang.org/x/sync/errgroup"
  25. "golang.org/x/tools/go/gcexportdata"
  26. "golang.org/x/tools/internal/gocommand"
  27. "golang.org/x/tools/internal/packagesinternal"
  28. "golang.org/x/tools/internal/typesinternal"
  29. )
  30. // A LoadMode controls the amount of detail to return when loading.
  31. // The bits below can be combined to specify which fields should be
  32. // filled in the result packages.
  33. //
  34. // The zero value is a special case, equivalent to combining
  35. // the NeedName, NeedFiles, and NeedCompiledGoFiles bits.
  36. //
  37. // ID and Errors (if present) will always be filled.
  38. // [Load] may return more information than requested.
  39. //
  40. // The Mode flag is a union of several bits named NeedName,
  41. // NeedFiles, and so on, each of which determines whether
  42. // a given field of Package (Name, Files, etc) should be
  43. // populated.
  44. //
  45. // For convenience, we provide named constants for the most
  46. // common combinations of Need flags:
  47. //
  48. // [LoadFiles] lists of files in each package
  49. // [LoadImports] ... plus imports
  50. // [LoadTypes] ... plus type information
  51. // [LoadSyntax] ... plus type-annotated syntax
  52. // [LoadAllSyntax] ... for all dependencies
  53. //
  54. // Unfortunately there are a number of open bugs related to
  55. // interactions among the LoadMode bits:
  56. // - https://go.dev/issue/56633
  57. // - https://go.dev/issue/56677
  58. // - https://go.dev/issue/58726
  59. // - https://go.dev/issue/63517
  60. type LoadMode int
  61. const (
  62. // NeedName adds Name and PkgPath.
  63. NeedName LoadMode = 1 << iota
  64. // NeedFiles adds Dir, GoFiles, OtherFiles, and IgnoredFiles
  65. NeedFiles
  66. // NeedCompiledGoFiles adds CompiledGoFiles.
  67. NeedCompiledGoFiles
  68. // NeedImports adds Imports. If NeedDeps is not set, the Imports field will contain
  69. // "placeholder" Packages with only the ID set.
  70. NeedImports
  71. // NeedDeps adds the fields requested by the LoadMode in the packages in Imports.
  72. NeedDeps
  73. // NeedExportFile adds ExportFile.
  74. NeedExportFile
  75. // NeedTypes adds Types, Fset, and IllTyped.
  76. NeedTypes
  77. // NeedSyntax adds Syntax and Fset.
  78. NeedSyntax
  79. // NeedTypesInfo adds TypesInfo and Fset.
  80. NeedTypesInfo
  81. // NeedTypesSizes adds TypesSizes.
  82. NeedTypesSizes
  83. // needInternalDepsErrors adds the internal deps errors field for use by gopls.
  84. needInternalDepsErrors
  85. // NeedForTest adds ForTest.
  86. //
  87. // Tests must also be set on the context for this field to be populated.
  88. NeedForTest
  89. // typecheckCgo enables full support for type checking cgo. Requires Go 1.15+.
  90. // Modifies CompiledGoFiles and Types, and has no effect on its own.
  91. typecheckCgo
  92. // NeedModule adds Module.
  93. NeedModule
  94. // NeedEmbedFiles adds EmbedFiles.
  95. NeedEmbedFiles
  96. // NeedEmbedPatterns adds EmbedPatterns.
  97. NeedEmbedPatterns
  98. // NeedTarget adds Target.
  99. NeedTarget
  100. // Be sure to update loadmode_string.go when adding new items!
  101. )
  102. const (
  103. // LoadFiles loads the name and file names for the initial packages.
  104. LoadFiles = NeedName | NeedFiles | NeedCompiledGoFiles
  105. // LoadImports loads the name, file names, and import mapping for the initial packages.
  106. LoadImports = LoadFiles | NeedImports
  107. // LoadTypes loads exported type information for the initial packages.
  108. LoadTypes = LoadImports | NeedTypes | NeedTypesSizes
  109. // LoadSyntax loads typed syntax for the initial packages.
  110. LoadSyntax = LoadTypes | NeedSyntax | NeedTypesInfo
  111. // LoadAllSyntax loads typed syntax for the initial packages and all dependencies.
  112. LoadAllSyntax = LoadSyntax | NeedDeps
  113. // Deprecated: NeedExportsFile is a historical misspelling of NeedExportFile.
  114. //
  115. //go:fix inline
  116. NeedExportsFile = NeedExportFile
  117. )
  118. // A Config specifies details about how packages should be loaded.
  119. // The zero value is a valid configuration.
  120. //
  121. // Calls to [Load] do not modify this struct.
  122. type Config struct {
  123. // Mode controls the level of information returned for each package.
  124. Mode LoadMode
  125. // Context specifies the context for the load operation.
  126. // Cancelling the context may cause [Load] to abort and
  127. // return an error.
  128. Context context.Context
  129. // Logf is the logger for the config.
  130. // If the user provides a logger, debug logging is enabled.
  131. // If the GOPACKAGESDEBUG environment variable is set to true,
  132. // but the logger is nil, default to log.Printf.
  133. Logf func(format string, args ...any)
  134. // Dir is the directory in which to run the build system's query tool
  135. // that provides information about the packages.
  136. // If Dir is empty, the tool is run in the current directory.
  137. Dir string
  138. // Env is the environment to use when invoking the build system's query tool.
  139. // If Env is nil, the current environment is used.
  140. // As in os/exec's Cmd, only the last value in the slice for
  141. // each environment key is used. To specify the setting of only
  142. // a few variables, append to the current environment, as in:
  143. //
  144. // opt.Env = append(os.Environ(), "GOOS=plan9", "GOARCH=386")
  145. //
  146. Env []string
  147. // BuildFlags is a list of command-line flags to be passed through to
  148. // the build system's query tool.
  149. BuildFlags []string
  150. // Fset provides source position information for syntax trees and types.
  151. // If Fset is nil, Load will use a new fileset, but preserve Fset's value.
  152. Fset *token.FileSet
  153. // ParseFile is called to read and parse each file
  154. // when preparing a package's type-checked syntax tree.
  155. // It must be safe to call ParseFile simultaneously from multiple goroutines.
  156. // If ParseFile is nil, the loader will uses parser.ParseFile.
  157. //
  158. // ParseFile should parse the source from src and use filename only for
  159. // recording position information.
  160. //
  161. // An application may supply a custom implementation of ParseFile
  162. // to change the effective file contents or the behavior of the parser,
  163. // or to modify the syntax tree. For example, selectively eliminating
  164. // unwanted function bodies can significantly accelerate type checking.
  165. ParseFile func(fset *token.FileSet, filename string, src []byte) (*ast.File, error)
  166. // If Tests is set, the loader includes not just the packages
  167. // matching a particular pattern but also any related test packages,
  168. // including test-only variants of the package and the test executable.
  169. //
  170. // For example, when using the go command, loading "fmt" with Tests=true
  171. // returns four packages, with IDs "fmt" (the standard package),
  172. // "fmt [fmt.test]" (the package as compiled for the test),
  173. // "fmt_test" (the test functions from source files in package fmt_test),
  174. // and "fmt.test" (the test binary).
  175. //
  176. // In build systems with explicit names for tests,
  177. // setting Tests may have no effect.
  178. Tests bool
  179. // Overlay is a mapping from absolute file paths to file contents.
  180. //
  181. // For each map entry, [Load] uses the alternative file
  182. // contents provided by the overlay mapping instead of reading
  183. // from the file system. This mechanism can be used to enable
  184. // editor-integrated tools to correctly analyze the contents
  185. // of modified but unsaved buffers, for example.
  186. //
  187. // The overlay mapping is passed to the build system's driver
  188. // (see "The driver protocol") so that it too can report
  189. // consistent package metadata about unsaved files. However,
  190. // drivers may vary in their level of support for overlays.
  191. Overlay map[string][]byte
  192. }
  193. // Load loads and returns the Go packages named by the given patterns.
  194. //
  195. // The cfg parameter specifies loading options; nil behaves the same as an empty [Config].
  196. //
  197. // The [Config.Mode] field is a set of bits that determine what kinds
  198. // of information should be computed and returned. Modes that require
  199. // more information tend to be slower. See [LoadMode] for details
  200. // and important caveats. Its zero value is equivalent to
  201. // [NeedName] | [NeedFiles] | [NeedCompiledGoFiles].
  202. //
  203. // Each call to Load returns a new set of [Package] instances.
  204. // The Packages and their Imports form a directed acyclic graph.
  205. //
  206. // If the [NeedTypes] mode flag was set, each call to Load uses a new
  207. // [types.Importer], so [types.Object] and [types.Type] values from
  208. // different calls to Load must not be mixed as they will have
  209. // inconsistent notions of type identity.
  210. //
  211. // If any of the patterns was invalid as defined by the
  212. // underlying build system, Load returns an error.
  213. // It may return an empty list of packages without an error,
  214. // for instance for an empty expansion of a valid wildcard.
  215. // Errors associated with a particular package are recorded in the
  216. // corresponding Package's Errors list, and do not cause Load to
  217. // return an error. Clients may need to handle such errors before
  218. // proceeding with further analysis. The [PrintErrors] function is
  219. // provided for convenient display of all errors.
  220. func Load(cfg *Config, patterns ...string) ([]*Package, error) {
  221. ld := newLoader(cfg)
  222. response, external, err := defaultDriver(&ld.Config, patterns...)
  223. if err != nil {
  224. return nil, err
  225. }
  226. ld.sizes = types.SizesFor(response.Compiler, response.Arch)
  227. if ld.sizes == nil && ld.Config.Mode&(NeedTypes|NeedTypesSizes|NeedTypesInfo) != 0 {
  228. // Type size information is needed but unavailable.
  229. if external {
  230. // An external driver may fail to populate the Compiler/GOARCH fields,
  231. // especially since they are relatively new (see #63700).
  232. // Provide a sensible fallback in this case.
  233. ld.sizes = types.SizesFor("gc", runtime.GOARCH)
  234. if ld.sizes == nil { // gccgo-only arch
  235. ld.sizes = types.SizesFor("gc", "amd64")
  236. }
  237. } else {
  238. // Go list should never fail to deliver accurate size information.
  239. // Reject the whole Load since the error is the same for every package.
  240. return nil, fmt.Errorf("can't determine type sizes for compiler %q on GOARCH %q",
  241. response.Compiler, response.Arch)
  242. }
  243. }
  244. return ld.refine(response)
  245. }
  246. // defaultDriver is a driver that implements go/packages' fallback behavior.
  247. // It will try to request to an external driver, if one exists. If there's
  248. // no external driver, or the driver returns a response with NotHandled set,
  249. // defaultDriver will fall back to the go list driver.
  250. // The boolean result indicates that an external driver handled the request.
  251. func defaultDriver(cfg *Config, patterns ...string) (*DriverResponse, bool, error) {
  252. const (
  253. // windowsArgMax specifies the maximum command line length for
  254. // the Windows' CreateProcess function.
  255. windowsArgMax = 32767
  256. // maxEnvSize is a very rough estimation of the maximum environment
  257. // size of a user.
  258. maxEnvSize = 16384
  259. // safeArgMax specifies the maximum safe command line length to use
  260. // by the underlying driver excl. the environment. We choose the Windows'
  261. // ARG_MAX as the starting point because it's one of the lowest ARG_MAX
  262. // constants out of the different supported platforms,
  263. // e.g., https://www.in-ulm.de/~mascheck/various/argmax/#results.
  264. safeArgMax = windowsArgMax - maxEnvSize
  265. )
  266. chunks, err := splitIntoChunks(patterns, safeArgMax)
  267. if err != nil {
  268. return nil, false, err
  269. }
  270. if driver := findExternalDriver(cfg); driver != nil {
  271. response, err := callDriverOnChunks(driver, cfg, chunks)
  272. if err != nil {
  273. return nil, false, err
  274. } else if !response.NotHandled {
  275. return response, true, nil
  276. }
  277. // not handled: fall through
  278. }
  279. // go list fallback
  280. // Write overlays once, as there are many calls
  281. // to 'go list' (one per chunk plus others too).
  282. overlayFile, cleanupOverlay, err := gocommand.WriteOverlays(cfg.Overlay)
  283. if err != nil {
  284. return nil, false, err
  285. }
  286. defer cleanupOverlay()
  287. var runner gocommand.Runner // (shared across many 'go list' calls)
  288. driver := func(cfg *Config, patterns []string) (*DriverResponse, error) {
  289. return goListDriver(cfg, &runner, overlayFile, patterns)
  290. }
  291. response, err := callDriverOnChunks(driver, cfg, chunks)
  292. if err != nil {
  293. return nil, false, err
  294. }
  295. return response, false, err
  296. }
  297. // splitIntoChunks chunks the slice so that the total number of characters
  298. // in a chunk is no longer than argMax.
  299. func splitIntoChunks(patterns []string, argMax int) ([][]string, error) {
  300. if argMax <= 0 {
  301. return nil, errors.New("failed to split patterns into chunks, negative safe argMax value")
  302. }
  303. var chunks [][]string
  304. charsInChunk := 0
  305. nextChunkStart := 0
  306. for i, v := range patterns {
  307. vChars := len(v)
  308. if vChars > argMax {
  309. // a single pattern is longer than the maximum safe ARG_MAX, hardly should happen
  310. return nil, errors.New("failed to split patterns into chunks, a pattern is too long")
  311. }
  312. charsInChunk += vChars + 1 // +1 is for a whitespace between patterns that has to be counted too
  313. if charsInChunk > argMax {
  314. chunks = append(chunks, patterns[nextChunkStart:i])
  315. nextChunkStart = i
  316. charsInChunk = vChars
  317. }
  318. }
  319. // add the last chunk
  320. if nextChunkStart < len(patterns) {
  321. chunks = append(chunks, patterns[nextChunkStart:])
  322. }
  323. return chunks, nil
  324. }
  325. func callDriverOnChunks(driver driver, cfg *Config, chunks [][]string) (*DriverResponse, error) {
  326. if len(chunks) == 0 {
  327. return driver(cfg, nil)
  328. }
  329. responses := make([]*DriverResponse, len(chunks))
  330. errNotHandled := errors.New("driver returned NotHandled")
  331. var g errgroup.Group
  332. for i, chunk := range chunks {
  333. g.Go(func() (err error) {
  334. responses[i], err = driver(cfg, chunk)
  335. if responses[i] != nil && responses[i].NotHandled {
  336. err = errNotHandled
  337. }
  338. return err
  339. })
  340. }
  341. if err := g.Wait(); err != nil {
  342. if errors.Is(err, errNotHandled) {
  343. return &DriverResponse{NotHandled: true}, nil
  344. }
  345. return nil, err
  346. }
  347. return mergeResponses(responses...), nil
  348. }
  349. func mergeResponses(responses ...*DriverResponse) *DriverResponse {
  350. if len(responses) == 0 {
  351. return nil
  352. }
  353. response := newDeduper()
  354. response.dr.NotHandled = false
  355. response.dr.Compiler = responses[0].Compiler
  356. response.dr.Arch = responses[0].Arch
  357. response.dr.GoVersion = responses[0].GoVersion
  358. for _, v := range responses {
  359. response.addAll(v)
  360. }
  361. return response.dr
  362. }
  363. // A Package describes a loaded Go package.
  364. //
  365. // It also defines part of the JSON schema of [DriverResponse].
  366. // See the package documentation for an overview.
  367. type Package struct {
  368. // ID is a unique identifier for a package,
  369. // in a syntax provided by the underlying build system.
  370. //
  371. // Because the syntax varies based on the build system,
  372. // clients should treat IDs as opaque and not attempt to
  373. // interpret them.
  374. ID string
  375. // Name is the package name as it appears in the package source code.
  376. Name string
  377. // PkgPath is the package path as used by the go/types package.
  378. PkgPath string
  379. // Dir is the directory associated with the package, if it exists.
  380. //
  381. // For packages listed by the go command, this is the directory containing
  382. // the package files.
  383. Dir string
  384. // Errors contains any errors encountered querying the metadata
  385. // of the package, or while parsing or type-checking its files.
  386. Errors []Error
  387. // TypeErrors contains the subset of errors produced during type checking.
  388. TypeErrors []types.Error
  389. // GoFiles lists the absolute file paths of the package's Go source files.
  390. // It may include files that should not be compiled, for example because
  391. // they contain non-matching build tags, are documentary pseudo-files such as
  392. // unsafe/unsafe.go or builtin/builtin.go, or are subject to cgo preprocessing.
  393. GoFiles []string
  394. // CompiledGoFiles lists the absolute file paths of the package's source
  395. // files that are suitable for type checking.
  396. // This may differ from GoFiles if files are processed before compilation.
  397. CompiledGoFiles []string
  398. // OtherFiles lists the absolute file paths of the package's non-Go source files,
  399. // including assembly, C, C++, Fortran, Objective-C, SWIG, and so on.
  400. OtherFiles []string
  401. // EmbedFiles lists the absolute file paths of the package's files
  402. // embedded with go:embed.
  403. EmbedFiles []string
  404. // EmbedPatterns lists the absolute file patterns of the package's
  405. // files embedded with go:embed.
  406. EmbedPatterns []string
  407. // IgnoredFiles lists source files that are not part of the package
  408. // using the current build configuration but that might be part of
  409. // the package using other build configurations.
  410. IgnoredFiles []string
  411. // ExportFile is the absolute path to a file containing type
  412. // information for the package as provided by the build system.
  413. ExportFile string
  414. // Target is the absolute install path of the .a file, for libraries,
  415. // and of the executable file, for binaries.
  416. Target string
  417. // Imports maps import paths appearing in the package's Go source files
  418. // to corresponding loaded Packages.
  419. Imports map[string]*Package
  420. // Module is the module information for the package if it exists.
  421. //
  422. // Note: it may be missing for std and cmd; see Go issue #65816.
  423. Module *Module
  424. // -- The following fields are not part of the driver JSON schema. --
  425. // Types provides type information for the package.
  426. // The NeedTypes LoadMode bit sets this field for packages matching the
  427. // patterns; type information for dependencies may be missing or incomplete,
  428. // unless NeedDeps and NeedImports are also set.
  429. //
  430. // Each call to [Load] returns a consistent set of type
  431. // symbols, as defined by the comment at [types.Identical].
  432. // Avoid mixing type information from two or more calls to [Load].
  433. Types *types.Package `json:"-"`
  434. // Fset provides position information for Types, TypesInfo, and Syntax.
  435. // It is set only when Types is set.
  436. Fset *token.FileSet `json:"-"`
  437. // IllTyped indicates whether the package or any dependency contains errors.
  438. // It is set only when Types is set.
  439. IllTyped bool `json:"-"`
  440. // Syntax is the package's syntax trees, for the files listed in CompiledGoFiles.
  441. //
  442. // The NeedSyntax LoadMode bit populates this field for packages matching the patterns.
  443. // If NeedDeps and NeedImports are also set, this field will also be populated
  444. // for dependencies.
  445. //
  446. // Syntax is kept in the same order as CompiledGoFiles, with the caveat that nils are
  447. // removed. If parsing returned nil, Syntax may be shorter than CompiledGoFiles.
  448. Syntax []*ast.File `json:"-"`
  449. // TypesInfo provides type information about the package's syntax trees.
  450. // It is set only when Syntax is set.
  451. TypesInfo *types.Info `json:"-"`
  452. // TypesSizes provides the effective size function for types in TypesInfo.
  453. TypesSizes types.Sizes `json:"-"`
  454. // -- internal --
  455. // ForTest is the package under test, if any.
  456. ForTest string
  457. // depsErrors is the DepsErrors field from the go list response, if any.
  458. depsErrors []*packagesinternal.PackageError
  459. }
  460. // Module provides module information for a package.
  461. //
  462. // It also defines part of the JSON schema of [DriverResponse].
  463. // See the package documentation for an overview.
  464. type Module struct {
  465. Path string // module path
  466. Version string // module version
  467. Replace *Module // replaced by this module
  468. Time *time.Time // time version was created
  469. Main bool // is this the main module?
  470. Indirect bool // is this module only an indirect dependency of main module?
  471. Dir string // directory holding files for this module, if any
  472. GoMod string // path to go.mod file used when loading this module, if any
  473. GoVersion string // go version used in module
  474. Error *ModuleError // error loading module
  475. }
  476. // ModuleError holds errors loading a module.
  477. type ModuleError struct {
  478. Err string // the error itself
  479. }
  480. func init() {
  481. packagesinternal.GetDepsErrors = func(p any) []*packagesinternal.PackageError {
  482. return p.(*Package).depsErrors
  483. }
  484. packagesinternal.TypecheckCgo = int(typecheckCgo)
  485. packagesinternal.DepsErrors = int(needInternalDepsErrors)
  486. }
  487. // An Error describes a problem with a package's metadata, syntax, or types.
  488. type Error struct {
  489. Pos string // "file:line:col" or "file:line" or "" or "-"
  490. Msg string
  491. Kind ErrorKind
  492. }
  493. // ErrorKind describes the source of the error, allowing the user to
  494. // differentiate between errors generated by the driver, the parser, or the
  495. // type-checker.
  496. type ErrorKind int
  497. const (
  498. UnknownError ErrorKind = iota
  499. ListError
  500. ParseError
  501. TypeError
  502. )
  503. func (err Error) Error() string {
  504. pos := err.Pos
  505. if pos == "" {
  506. pos = "-" // like token.Position{}.String()
  507. }
  508. return pos + ": " + err.Msg
  509. }
  510. // flatPackage is the JSON form of Package
  511. // It drops all the type and syntax fields, and transforms the Imports
  512. //
  513. // TODO(adonovan): identify this struct with Package, effectively
  514. // publishing the JSON protocol.
  515. type flatPackage struct {
  516. ID string
  517. Name string `json:",omitempty"`
  518. PkgPath string `json:",omitempty"`
  519. Errors []Error `json:",omitempty"`
  520. GoFiles []string `json:",omitempty"`
  521. CompiledGoFiles []string `json:",omitempty"`
  522. OtherFiles []string `json:",omitempty"`
  523. EmbedFiles []string `json:",omitempty"`
  524. EmbedPatterns []string `json:",omitempty"`
  525. IgnoredFiles []string `json:",omitempty"`
  526. ExportFile string `json:",omitempty"`
  527. Imports map[string]string `json:",omitempty"`
  528. }
  529. // MarshalJSON returns the Package in its JSON form.
  530. // For the most part, the structure fields are written out unmodified, and
  531. // the type and syntax fields are skipped.
  532. // The imports are written out as just a map of path to package id.
  533. // The errors are written using a custom type that tries to preserve the
  534. // structure of error types we know about.
  535. //
  536. // This method exists to enable support for additional build systems. It is
  537. // not intended for use by clients of the API and we may change the format.
  538. func (p *Package) MarshalJSON() ([]byte, error) {
  539. flat := &flatPackage{
  540. ID: p.ID,
  541. Name: p.Name,
  542. PkgPath: p.PkgPath,
  543. Errors: p.Errors,
  544. GoFiles: p.GoFiles,
  545. CompiledGoFiles: p.CompiledGoFiles,
  546. OtherFiles: p.OtherFiles,
  547. EmbedFiles: p.EmbedFiles,
  548. EmbedPatterns: p.EmbedPatterns,
  549. IgnoredFiles: p.IgnoredFiles,
  550. ExportFile: p.ExportFile,
  551. }
  552. if len(p.Imports) > 0 {
  553. flat.Imports = make(map[string]string, len(p.Imports))
  554. for path, ipkg := range p.Imports {
  555. flat.Imports[path] = ipkg.ID
  556. }
  557. }
  558. return json.Marshal(flat)
  559. }
  560. // UnmarshalJSON reads in a Package from its JSON format.
  561. // See MarshalJSON for details about the format accepted.
  562. func (p *Package) UnmarshalJSON(b []byte) error {
  563. flat := &flatPackage{}
  564. if err := json.Unmarshal(b, &flat); err != nil {
  565. return err
  566. }
  567. *p = Package{
  568. ID: flat.ID,
  569. Name: flat.Name,
  570. PkgPath: flat.PkgPath,
  571. Errors: flat.Errors,
  572. GoFiles: flat.GoFiles,
  573. CompiledGoFiles: flat.CompiledGoFiles,
  574. OtherFiles: flat.OtherFiles,
  575. EmbedFiles: flat.EmbedFiles,
  576. EmbedPatterns: flat.EmbedPatterns,
  577. IgnoredFiles: flat.IgnoredFiles,
  578. ExportFile: flat.ExportFile,
  579. }
  580. if len(flat.Imports) > 0 {
  581. p.Imports = make(map[string]*Package, len(flat.Imports))
  582. for path, id := range flat.Imports {
  583. p.Imports[path] = &Package{ID: id}
  584. }
  585. }
  586. return nil
  587. }
  588. func (p *Package) String() string { return p.ID }
  589. // loaderPackage augments Package with state used during the loading phase
  590. type loaderPackage struct {
  591. *Package
  592. importErrors map[string]error // maps each bad import to its error
  593. preds []*loaderPackage // packages that import this one
  594. unfinishedSuccs atomic.Int32 // number of direct imports not yet loaded
  595. color uint8 // for cycle detection
  596. needsrc bool // load from source (Mode >= LoadTypes)
  597. needtypes bool // type information is either requested or depended on
  598. initial bool // package was matched by a pattern
  599. goVersion int // minor version number of go command on PATH
  600. }
  601. // loader holds the working state of a single call to load.
  602. type loader struct {
  603. pkgs map[string]*loaderPackage // keyed by Package.ID
  604. Config
  605. sizes types.Sizes // non-nil if needed by mode
  606. parseCache map[string]*parseValue
  607. parseCacheMu sync.Mutex
  608. exportMu sync.Mutex // enforces mutual exclusion of exportdata operations
  609. // Config.Mode contains the implied mode (see impliedLoadMode).
  610. // Implied mode contains all the fields we need the data for.
  611. // In requestedMode there are the actually requested fields.
  612. // We'll zero them out before returning packages to the user.
  613. // This makes it easier for us to get the conditions where
  614. // we need certain modes right.
  615. requestedMode LoadMode
  616. }
  617. type parseValue struct {
  618. f *ast.File
  619. err error
  620. ready chan struct{}
  621. }
  622. func newLoader(cfg *Config) *loader {
  623. ld := &loader{
  624. parseCache: map[string]*parseValue{},
  625. }
  626. if cfg != nil {
  627. ld.Config = *cfg
  628. // If the user has provided a logger, use it.
  629. ld.Config.Logf = cfg.Logf
  630. }
  631. if ld.Config.Logf == nil {
  632. // If the GOPACKAGESDEBUG environment variable is set to true,
  633. // but the user has not provided a logger, default to log.Printf.
  634. if debug {
  635. ld.Config.Logf = log.Printf
  636. } else {
  637. ld.Config.Logf = func(format string, args ...any) {}
  638. }
  639. }
  640. if ld.Config.Mode == 0 {
  641. ld.Config.Mode = NeedName | NeedFiles | NeedCompiledGoFiles // Preserve zero behavior of Mode for backwards compatibility.
  642. }
  643. if ld.Config.Env == nil {
  644. ld.Config.Env = os.Environ()
  645. }
  646. if ld.Context == nil {
  647. ld.Context = context.Background()
  648. }
  649. if ld.Dir == "" {
  650. if dir, err := os.Getwd(); err == nil {
  651. ld.Dir = dir
  652. }
  653. }
  654. // Save the actually requested fields. We'll zero them out before returning packages to the user.
  655. ld.requestedMode = ld.Mode
  656. ld.Mode = impliedLoadMode(ld.Mode)
  657. if ld.Mode&(NeedSyntax|NeedTypes|NeedTypesInfo) != 0 {
  658. if ld.Fset == nil {
  659. ld.Fset = token.NewFileSet()
  660. }
  661. // ParseFile is required even in LoadTypes mode
  662. // because we load source if export data is missing.
  663. if ld.ParseFile == nil {
  664. ld.ParseFile = func(fset *token.FileSet, filename string, src []byte) (*ast.File, error) {
  665. // We implicitly promise to keep doing ast.Object resolution. :(
  666. const mode = parser.AllErrors | parser.ParseComments
  667. return parser.ParseFile(fset, filename, src, mode)
  668. }
  669. }
  670. }
  671. return ld
  672. }
  673. // refine connects the supplied packages into a graph and then adds type
  674. // and syntax information as requested by the LoadMode.
  675. func (ld *loader) refine(response *DriverResponse) ([]*Package, error) {
  676. roots := response.Roots
  677. rootMap := make(map[string]int, len(roots))
  678. for i, root := range roots {
  679. rootMap[root] = i
  680. }
  681. ld.pkgs = make(map[string]*loaderPackage)
  682. // first pass, fixup and build the map and roots
  683. var initial = make([]*loaderPackage, len(roots))
  684. for _, pkg := range response.Packages {
  685. rootIndex := -1
  686. if i, found := rootMap[pkg.ID]; found {
  687. rootIndex = i
  688. }
  689. // Overlays can invalidate export data.
  690. // TODO(matloob): make this check fine-grained based on dependencies on overlaid files
  691. exportDataInvalid := len(ld.Overlay) > 0 || pkg.ExportFile == "" && pkg.PkgPath != "unsafe"
  692. // This package needs type information if the caller requested types and the package is
  693. // either a root, or it's a non-root and the user requested dependencies ...
  694. needtypes := (ld.Mode&(NeedTypes|NeedTypesInfo) != 0 && (rootIndex >= 0 || ld.Mode&NeedDeps != 0))
  695. // This package needs source if the call requested source (or types info, which implies source)
  696. // and the package is either a root, or itas a non- root and the user requested dependencies...
  697. needsrc := ((ld.Mode&(NeedSyntax|NeedTypesInfo) != 0 && (rootIndex >= 0 || ld.Mode&NeedDeps != 0)) ||
  698. // ... or if we need types and the exportData is invalid. We fall back to (incompletely)
  699. // typechecking packages from source if they fail to compile.
  700. (ld.Mode&(NeedTypes|NeedTypesInfo) != 0 && exportDataInvalid)) && pkg.PkgPath != "unsafe"
  701. lpkg := &loaderPackage{
  702. Package: pkg,
  703. needtypes: needtypes,
  704. needsrc: needsrc,
  705. goVersion: response.GoVersion,
  706. }
  707. ld.pkgs[lpkg.ID] = lpkg
  708. if rootIndex >= 0 {
  709. initial[rootIndex] = lpkg
  710. lpkg.initial = true
  711. }
  712. }
  713. for i, root := range roots {
  714. if initial[i] == nil {
  715. return nil, fmt.Errorf("root package %v is missing", root)
  716. }
  717. }
  718. // Materialize the import graph if it is needed (NeedImports),
  719. // or if we'll be using loadPackages (Need{Syntax|Types|TypesInfo}).
  720. var leaves []*loaderPackage // packages with no unfinished successors
  721. if ld.Mode&(NeedImports|NeedSyntax|NeedTypes|NeedTypesInfo) != 0 {
  722. const (
  723. white = 0 // new
  724. grey = 1 // in progress
  725. black = 2 // complete
  726. )
  727. // visit traverses the import graph, depth-first,
  728. // and materializes the graph as Packages.Imports.
  729. //
  730. // Valid imports are saved in the Packages.Import map.
  731. // Invalid imports (cycles and missing nodes) are saved in the importErrors map.
  732. // Thus, even in the presence of both kinds of errors,
  733. // the Import graph remains a DAG.
  734. //
  735. // visit returns whether the package needs src or has a transitive
  736. // dependency on a package that does. These are the only packages
  737. // for which we load source code.
  738. var stack []*loaderPackage
  739. var visit func(from, lpkg *loaderPackage) bool
  740. visit = func(from, lpkg *loaderPackage) bool {
  741. if lpkg.color == grey {
  742. panic("internal error: grey node")
  743. }
  744. if lpkg.color == white {
  745. lpkg.color = grey
  746. stack = append(stack, lpkg) // push
  747. stubs := lpkg.Imports // the structure form has only stubs with the ID in the Imports
  748. lpkg.Imports = make(map[string]*Package, len(stubs))
  749. for importPath, ipkg := range stubs {
  750. var importErr error
  751. imp := ld.pkgs[ipkg.ID]
  752. if imp == nil {
  753. // (includes package "C" when DisableCgo)
  754. importErr = fmt.Errorf("missing package: %q", ipkg.ID)
  755. } else if imp.color == grey {
  756. importErr = fmt.Errorf("import cycle: %s", stack)
  757. }
  758. if importErr != nil {
  759. if lpkg.importErrors == nil {
  760. lpkg.importErrors = make(map[string]error)
  761. }
  762. lpkg.importErrors[importPath] = importErr
  763. continue
  764. }
  765. if visit(lpkg, imp) {
  766. lpkg.needsrc = true
  767. }
  768. lpkg.Imports[importPath] = imp.Package
  769. }
  770. // -- postorder --
  771. // Complete type information is required for the
  772. // immediate dependencies of each source package.
  773. if lpkg.needsrc && ld.Mode&NeedTypes != 0 {
  774. for _, ipkg := range lpkg.Imports {
  775. ld.pkgs[ipkg.ID].needtypes = true
  776. }
  777. }
  778. // NeedTypeSizes causes TypeSizes to be set even
  779. // on packages for which types aren't needed.
  780. if ld.Mode&NeedTypesSizes != 0 {
  781. lpkg.TypesSizes = ld.sizes
  782. }
  783. // Add packages with no imports directly to the queue of leaves.
  784. if len(lpkg.Imports) == 0 {
  785. leaves = append(leaves, lpkg)
  786. }
  787. stack = stack[:len(stack)-1] // pop
  788. lpkg.color = black
  789. }
  790. // Add edge from predecessor.
  791. if from != nil {
  792. from.unfinishedSuccs.Add(+1) // incref
  793. lpkg.preds = append(lpkg.preds, from)
  794. }
  795. return lpkg.needsrc
  796. }
  797. // For each initial package, create its import DAG.
  798. for _, lpkg := range initial {
  799. visit(nil, lpkg)
  800. }
  801. } else {
  802. // !NeedImports: drop the stub (ID-only) import packages
  803. // that we are not even going to try to resolve.
  804. for _, lpkg := range initial {
  805. lpkg.Imports = nil
  806. }
  807. }
  808. // Load type data and syntax if needed, starting at
  809. // the initial packages (roots of the import DAG).
  810. if ld.Mode&(NeedSyntax|NeedTypes|NeedTypesInfo) != 0 {
  811. // We avoid using g.SetLimit to limit concurrency as
  812. // it makes g.Go stop accepting work, which prevents
  813. // workers from enqeuing, and thus finishing, and thus
  814. // allowing the group to make progress: deadlock.
  815. //
  816. // Instead we use the ioLimit and cpuLimit semaphores.
  817. g, _ := errgroup.WithContext(ld.Context)
  818. // enqueues adds a package to the type-checking queue.
  819. // It must have no unfinished successors.
  820. var enqueue func(*loaderPackage)
  821. enqueue = func(lpkg *loaderPackage) {
  822. g.Go(func() error {
  823. // Parse and type-check.
  824. ld.loadPackage(lpkg)
  825. // Notify each waiting predecessor,
  826. // and enqueue it when it becomes a leaf.
  827. for _, pred := range lpkg.preds {
  828. if pred.unfinishedSuccs.Add(-1) == 0 { // decref
  829. enqueue(pred)
  830. }
  831. }
  832. return nil
  833. })
  834. }
  835. // Load leaves first, adding new packages
  836. // to the queue as they become leaves.
  837. for _, leaf := range leaves {
  838. enqueue(leaf)
  839. }
  840. if err := g.Wait(); err != nil {
  841. return nil, err // cancelled
  842. }
  843. }
  844. // If the context is done, return its error and
  845. // throw out [likely] incomplete packages.
  846. if err := ld.Context.Err(); err != nil {
  847. return nil, err
  848. }
  849. result := make([]*Package, len(initial))
  850. for i, lpkg := range initial {
  851. result[i] = lpkg.Package
  852. }
  853. for i := range ld.pkgs {
  854. // Clear all unrequested fields,
  855. // to catch programs that use more than they request.
  856. if ld.requestedMode&NeedName == 0 {
  857. ld.pkgs[i].Name = ""
  858. ld.pkgs[i].PkgPath = ""
  859. }
  860. if ld.requestedMode&NeedFiles == 0 {
  861. ld.pkgs[i].GoFiles = nil
  862. ld.pkgs[i].OtherFiles = nil
  863. ld.pkgs[i].IgnoredFiles = nil
  864. }
  865. if ld.requestedMode&NeedEmbedFiles == 0 {
  866. ld.pkgs[i].EmbedFiles = nil
  867. }
  868. if ld.requestedMode&NeedEmbedPatterns == 0 {
  869. ld.pkgs[i].EmbedPatterns = nil
  870. }
  871. if ld.requestedMode&NeedCompiledGoFiles == 0 {
  872. ld.pkgs[i].CompiledGoFiles = nil
  873. }
  874. if ld.requestedMode&NeedImports == 0 {
  875. ld.pkgs[i].Imports = nil
  876. }
  877. if ld.requestedMode&NeedExportFile == 0 {
  878. ld.pkgs[i].ExportFile = ""
  879. }
  880. if ld.requestedMode&NeedTypes == 0 {
  881. ld.pkgs[i].Types = nil
  882. ld.pkgs[i].IllTyped = false
  883. }
  884. if ld.requestedMode&NeedSyntax == 0 {
  885. ld.pkgs[i].Syntax = nil
  886. }
  887. if ld.requestedMode&(NeedSyntax|NeedTypes|NeedTypesInfo) == 0 {
  888. ld.pkgs[i].Fset = nil
  889. }
  890. if ld.requestedMode&NeedTypesInfo == 0 {
  891. ld.pkgs[i].TypesInfo = nil
  892. }
  893. if ld.requestedMode&NeedTypesSizes == 0 {
  894. ld.pkgs[i].TypesSizes = nil
  895. }
  896. if ld.requestedMode&NeedModule == 0 {
  897. ld.pkgs[i].Module = nil
  898. }
  899. }
  900. return result, nil
  901. }
  902. // loadPackage loads/parses/typechecks the specified package.
  903. // It must be called only once per Package,
  904. // after immediate dependencies are loaded.
  905. // Precondition: ld.Mode&(NeedSyntax|NeedTypes|NeedTypesInfo) != 0.
  906. func (ld *loader) loadPackage(lpkg *loaderPackage) {
  907. if lpkg.PkgPath == "unsafe" {
  908. // Fill in the blanks to avoid surprises.
  909. lpkg.Types = types.Unsafe
  910. lpkg.Fset = ld.Fset
  911. lpkg.Syntax = []*ast.File{}
  912. lpkg.TypesInfo = new(types.Info)
  913. lpkg.TypesSizes = ld.sizes
  914. return
  915. }
  916. // Call NewPackage directly with explicit name.
  917. // This avoids skew between golist and go/types when the files'
  918. // package declarations are inconsistent.
  919. lpkg.Types = types.NewPackage(lpkg.PkgPath, lpkg.Name)
  920. lpkg.Fset = ld.Fset
  921. // Start shutting down if the context is done and do not load
  922. // source or export data files.
  923. // Packages that import this one will have ld.Context.Err() != nil.
  924. // ld.Context.Err() will be returned later by refine.
  925. if ld.Context.Err() != nil {
  926. return
  927. }
  928. // Subtle: we populate all Types fields with an empty Package
  929. // before loading export data so that export data processing
  930. // never has to create a types.Package for an indirect dependency,
  931. // which would then require that such created packages be explicitly
  932. // inserted back into the Import graph as a final step after export data loading.
  933. // (Hence this return is after the Types assignment.)
  934. // The Diamond test exercises this case.
  935. if !lpkg.needtypes && !lpkg.needsrc {
  936. return
  937. }
  938. // TODO(adonovan): this condition looks wrong:
  939. // I think it should be lpkg.needtypes && !lpg.needsrc,
  940. // so that NeedSyntax without NeedTypes can be satisfied by export data.
  941. if !lpkg.needsrc {
  942. if err := ld.loadFromExportData(lpkg); err != nil {
  943. lpkg.Errors = append(lpkg.Errors, Error{
  944. Pos: "-",
  945. Msg: err.Error(),
  946. Kind: UnknownError, // e.g. can't find/open/parse export data
  947. })
  948. }
  949. return // not a source package, don't get syntax trees
  950. }
  951. appendError := func(err error) {
  952. // Convert various error types into the one true Error.
  953. var errs []Error
  954. switch err := err.(type) {
  955. case Error:
  956. // from driver
  957. errs = append(errs, err)
  958. case *os.PathError:
  959. // from parser
  960. errs = append(errs, Error{
  961. Pos: err.Path + ":1",
  962. Msg: err.Err.Error(),
  963. Kind: ParseError,
  964. })
  965. case scanner.ErrorList:
  966. // from parser
  967. for _, err := range err {
  968. errs = append(errs, Error{
  969. Pos: err.Pos.String(),
  970. Msg: err.Msg,
  971. Kind: ParseError,
  972. })
  973. }
  974. case types.Error:
  975. // from type checker
  976. lpkg.TypeErrors = append(lpkg.TypeErrors, err)
  977. errs = append(errs, Error{
  978. Pos: err.Fset.Position(err.Pos).String(),
  979. Msg: err.Msg,
  980. Kind: TypeError,
  981. })
  982. default:
  983. // unexpected impoverished error from parser?
  984. errs = append(errs, Error{
  985. Pos: "-",
  986. Msg: err.Error(),
  987. Kind: UnknownError,
  988. })
  989. // If you see this error message, please file a bug.
  990. log.Printf("internal error: error %q (%T) without position", err, err)
  991. }
  992. lpkg.Errors = append(lpkg.Errors, errs...)
  993. }
  994. // If the go command on the PATH is newer than the runtime,
  995. // then the go/{scanner,ast,parser,types} packages from the
  996. // standard library may be unable to process the files
  997. // selected by go list.
  998. //
  999. // There is currently no way to downgrade the effective
  1000. // version of the go command (see issue 52078), so we proceed
  1001. // with the newer go command but, in case of parse or type
  1002. // errors, we emit an additional diagnostic.
  1003. //
  1004. // See:
  1005. // - golang.org/issue/52078 (flag to set release tags)
  1006. // - golang.org/issue/50825 (gopls legacy version support)
  1007. // - golang.org/issue/55883 (go/packages confusing error)
  1008. //
  1009. // Should we assert a hard minimum of (currently) go1.16 here?
  1010. var runtimeVersion int
  1011. if _, err := fmt.Sscanf(runtime.Version(), "go1.%d", &runtimeVersion); err == nil && runtimeVersion < lpkg.goVersion {
  1012. defer func() {
  1013. if len(lpkg.Errors) > 0 {
  1014. appendError(Error{
  1015. Pos: "-",
  1016. Msg: fmt.Sprintf("This application uses version go1.%d of the source-processing packages but runs version go1.%d of 'go list'. It may fail to process source files that rely on newer language features. If so, rebuild the application using a newer version of Go.", runtimeVersion, lpkg.goVersion),
  1017. Kind: UnknownError,
  1018. })
  1019. }
  1020. }()
  1021. }
  1022. if ld.Config.Mode&NeedTypes != 0 && len(lpkg.CompiledGoFiles) == 0 && lpkg.ExportFile != "" {
  1023. // The config requested loading sources and types, but sources are missing.
  1024. // Add an error to the package and fall back to loading from export data.
  1025. appendError(Error{"-", fmt.Sprintf("sources missing for package %s", lpkg.ID), ParseError})
  1026. _ = ld.loadFromExportData(lpkg) // ignore any secondary errors
  1027. return // can't get syntax trees for this package
  1028. }
  1029. files, errs := ld.parseFiles(lpkg.CompiledGoFiles)
  1030. for _, err := range errs {
  1031. appendError(err)
  1032. }
  1033. lpkg.Syntax = files
  1034. if ld.Config.Mode&(NeedTypes|NeedTypesInfo) == 0 {
  1035. return
  1036. }
  1037. // Start shutting down if the context is done and do not type check.
  1038. // Packages that import this one will have ld.Context.Err() != nil.
  1039. // ld.Context.Err() will be returned later by refine.
  1040. if ld.Context.Err() != nil {
  1041. return
  1042. }
  1043. // Populate TypesInfo only if needed, as it
  1044. // causes the type checker to work much harder.
  1045. if ld.Config.Mode&NeedTypesInfo != 0 {
  1046. lpkg.TypesInfo = &types.Info{
  1047. Types: make(map[ast.Expr]types.TypeAndValue),
  1048. Defs: make(map[*ast.Ident]types.Object),
  1049. Uses: make(map[*ast.Ident]types.Object),
  1050. Implicits: make(map[ast.Node]types.Object),
  1051. Instances: make(map[*ast.Ident]types.Instance),
  1052. Scopes: make(map[ast.Node]*types.Scope),
  1053. Selections: make(map[*ast.SelectorExpr]*types.Selection),
  1054. FileVersions: make(map[*ast.File]string),
  1055. }
  1056. }
  1057. lpkg.TypesSizes = ld.sizes
  1058. importer := importerFunc(func(path string) (*types.Package, error) {
  1059. if path == "unsafe" {
  1060. return types.Unsafe, nil
  1061. }
  1062. // The imports map is keyed by import path.
  1063. ipkg := lpkg.Imports[path]
  1064. if ipkg == nil {
  1065. if err := lpkg.importErrors[path]; err != nil {
  1066. return nil, err
  1067. }
  1068. // There was skew between the metadata and the
  1069. // import declarations, likely due to an edit
  1070. // race, or because the ParseFile feature was
  1071. // used to supply alternative file contents.
  1072. return nil, fmt.Errorf("no metadata for %s", path)
  1073. }
  1074. if ipkg.Types != nil && ipkg.Types.Complete() {
  1075. return ipkg.Types, nil
  1076. }
  1077. log.Fatalf("internal error: package %q without types was imported from %q", path, lpkg)
  1078. panic("unreachable")
  1079. })
  1080. // type-check
  1081. tc := &types.Config{
  1082. Importer: importer,
  1083. // Type-check bodies of functions only in initial packages.
  1084. // Example: for import graph A->B->C and initial packages {A,C},
  1085. // we can ignore function bodies in B.
  1086. IgnoreFuncBodies: ld.Mode&NeedDeps == 0 && !lpkg.initial,
  1087. Error: appendError,
  1088. Sizes: ld.sizes, // may be nil
  1089. }
  1090. if lpkg.Module != nil && lpkg.Module.GoVersion != "" {
  1091. tc.GoVersion = "go" + lpkg.Module.GoVersion
  1092. }
  1093. if (ld.Mode & typecheckCgo) != 0 {
  1094. if !typesinternal.SetUsesCgo(tc) {
  1095. appendError(Error{
  1096. Msg: "typecheckCgo requires Go 1.15+",
  1097. Kind: ListError,
  1098. })
  1099. return
  1100. }
  1101. }
  1102. // Type-checking is CPU intensive.
  1103. cpuLimit <- unit{} // acquire a token
  1104. defer func() { <-cpuLimit }() // release a token
  1105. typErr := types.NewChecker(tc, ld.Fset, lpkg.Types, lpkg.TypesInfo).Files(lpkg.Syntax)
  1106. lpkg.importErrors = nil // no longer needed
  1107. // In go/types go1.21 and go1.22, Checker.Files failed fast with a
  1108. // a "too new" error, without calling tc.Error and without
  1109. // proceeding to type-check the package (#66525).
  1110. // We rely on the runtimeVersion error to give the suggested remedy.
  1111. if typErr != nil && len(lpkg.Errors) == 0 && len(lpkg.Syntax) > 0 {
  1112. if msg := typErr.Error(); strings.HasPrefix(msg, "package requires newer Go version") {
  1113. appendError(types.Error{
  1114. Fset: ld.Fset,
  1115. Pos: lpkg.Syntax[0].Package,
  1116. Msg: msg,
  1117. })
  1118. }
  1119. }
  1120. // If !Cgo, the type-checker uses FakeImportC mode, so
  1121. // it doesn't invoke the importer for import "C",
  1122. // nor report an error for the import,
  1123. // or for any undefined C.f reference.
  1124. // We must detect this explicitly and correctly
  1125. // mark the package as IllTyped (by reporting an error).
  1126. // TODO(adonovan): if these errors are annoying,
  1127. // we could just set IllTyped quietly.
  1128. if tc.FakeImportC {
  1129. outer:
  1130. for _, f := range lpkg.Syntax {
  1131. for _, imp := range f.Imports {
  1132. if imp.Path.Value == `"C"` {
  1133. err := types.Error{Fset: ld.Fset, Pos: imp.Pos(), Msg: `import "C" ignored`}
  1134. appendError(err)
  1135. break outer
  1136. }
  1137. }
  1138. }
  1139. }
  1140. // If types.Checker.Files had an error that was unreported,
  1141. // make sure to report the unknown error so the package is illTyped.
  1142. if typErr != nil && len(lpkg.Errors) == 0 {
  1143. appendError(typErr)
  1144. }
  1145. // Record accumulated errors.
  1146. illTyped := len(lpkg.Errors) > 0
  1147. if !illTyped {
  1148. for _, imp := range lpkg.Imports {
  1149. if imp.IllTyped {
  1150. illTyped = true
  1151. break
  1152. }
  1153. }
  1154. }
  1155. lpkg.IllTyped = illTyped
  1156. }
  1157. // An importFunc is an implementation of the single-method
  1158. // types.Importer interface based on a function value.
  1159. type importerFunc func(path string) (*types.Package, error)
  1160. func (f importerFunc) Import(path string) (*types.Package, error) { return f(path) }
  1161. // We use a counting semaphore to limit
  1162. // the number of parallel I/O calls or CPU threads per process.
  1163. var (
  1164. ioLimit = make(chan unit, 20)
  1165. cpuLimit = make(chan unit, runtime.GOMAXPROCS(0))
  1166. )
  1167. func (ld *loader) parseFile(filename string) (*ast.File, error) {
  1168. ld.parseCacheMu.Lock()
  1169. v, ok := ld.parseCache[filename]
  1170. if ok {
  1171. // cache hit
  1172. ld.parseCacheMu.Unlock()
  1173. <-v.ready
  1174. } else {
  1175. // cache miss
  1176. v = &parseValue{ready: make(chan struct{})}
  1177. ld.parseCache[filename] = v
  1178. ld.parseCacheMu.Unlock()
  1179. var src []byte
  1180. for f, contents := range ld.Config.Overlay {
  1181. // TODO(adonovan): Inefficient for large overlays.
  1182. // Do an exact name-based map lookup
  1183. // (for nonexistent files) followed by a
  1184. // FileID-based map lookup (for existing ones).
  1185. if sameFile(f, filename) {
  1186. src = contents
  1187. break
  1188. }
  1189. }
  1190. var err error
  1191. if src == nil {
  1192. ioLimit <- unit{} // acquire a token
  1193. src, err = os.ReadFile(filename)
  1194. <-ioLimit // release a token
  1195. }
  1196. if err != nil {
  1197. v.err = err
  1198. } else {
  1199. // Parsing is CPU intensive.
  1200. cpuLimit <- unit{} // acquire a token
  1201. v.f, v.err = ld.ParseFile(ld.Fset, filename, src)
  1202. <-cpuLimit // release a token
  1203. }
  1204. close(v.ready)
  1205. }
  1206. return v.f, v.err
  1207. }
  1208. // parseFiles reads and parses the Go source files and returns the ASTs
  1209. // of the ones that could be at least partially parsed, along with a
  1210. // list of I/O and parse errors encountered.
  1211. //
  1212. // Because files are scanned in parallel, the token.Pos
  1213. // positions of the resulting ast.Files are not ordered.
  1214. func (ld *loader) parseFiles(filenames []string) ([]*ast.File, []error) {
  1215. var (
  1216. n = len(filenames)
  1217. parsed = make([]*ast.File, n)
  1218. errors = make([]error, n)
  1219. )
  1220. var g errgroup.Group
  1221. for i, filename := range filenames {
  1222. // This creates goroutines unnecessarily in the
  1223. // cache-hit case, but that case is uncommon.
  1224. g.Go(func() error {
  1225. parsed[i], errors[i] = ld.parseFile(filename)
  1226. return nil
  1227. })
  1228. }
  1229. g.Wait()
  1230. // Eliminate nils, preserving order.
  1231. var o int
  1232. for _, f := range parsed {
  1233. if f != nil {
  1234. parsed[o] = f
  1235. o++
  1236. }
  1237. }
  1238. parsed = parsed[:o]
  1239. o = 0
  1240. for _, err := range errors {
  1241. if err != nil {
  1242. errors[o] = err
  1243. o++
  1244. }
  1245. }
  1246. errors = errors[:o]
  1247. return parsed, errors
  1248. }
  1249. // sameFile returns true if x and y have the same basename and denote
  1250. // the same file.
  1251. func sameFile(x, y string) bool {
  1252. if x == y {
  1253. // It could be the case that y doesn't exist.
  1254. // For instance, it may be an overlay file that
  1255. // hasn't been written to disk. To handle that case
  1256. // let x == y through. (We added the exact absolute path
  1257. // string to the CompiledGoFiles list, so the unwritten
  1258. // overlay case implies x==y.)
  1259. return true
  1260. }
  1261. if strings.EqualFold(filepath.Base(x), filepath.Base(y)) { // (optimisation)
  1262. if xi, err := os.Stat(x); err == nil {
  1263. if yi, err := os.Stat(y); err == nil {
  1264. return os.SameFile(xi, yi)
  1265. }
  1266. }
  1267. }
  1268. return false
  1269. }
  1270. // loadFromExportData ensures that type information is present for the specified
  1271. // package, loading it from an export data file on the first request.
  1272. // On success it sets lpkg.Types to a new Package.
  1273. func (ld *loader) loadFromExportData(lpkg *loaderPackage) error {
  1274. if lpkg.PkgPath == "" {
  1275. log.Fatalf("internal error: Package %s has no PkgPath", lpkg)
  1276. }
  1277. // Because gcexportdata.Read has the potential to create or
  1278. // modify the types.Package for each node in the transitive
  1279. // closure of dependencies of lpkg, all exportdata operations
  1280. // must be sequential. (Finer-grained locking would require
  1281. // changes to the gcexportdata API.)
  1282. //
  1283. // The exportMu lock guards the lpkg.Types field and the
  1284. // types.Package it points to, for each loaderPackage in the graph.
  1285. //
  1286. // Not all accesses to Package.Pkg need to be protected by exportMu:
  1287. // graph ordering ensures that direct dependencies of source
  1288. // packages are fully loaded before the importer reads their Pkg field.
  1289. ld.exportMu.Lock()
  1290. defer ld.exportMu.Unlock()
  1291. if tpkg := lpkg.Types; tpkg != nil && tpkg.Complete() {
  1292. return nil // cache hit
  1293. }
  1294. lpkg.IllTyped = true // fail safe
  1295. if lpkg.ExportFile == "" {
  1296. // Errors while building export data will have been printed to stderr.
  1297. return fmt.Errorf("no export data file")
  1298. }
  1299. f, err := os.Open(lpkg.ExportFile)
  1300. if err != nil {
  1301. return err
  1302. }
  1303. defer f.Close()
  1304. // Read gc export data.
  1305. //
  1306. // We don't currently support gccgo export data because all
  1307. // underlying workspaces use the gc toolchain. (Even build
  1308. // systems that support gccgo don't use it for workspace
  1309. // queries.)
  1310. r, err := gcexportdata.NewReader(f)
  1311. if err != nil {
  1312. return fmt.Errorf("reading %s: %v", lpkg.ExportFile, err)
  1313. }
  1314. // Build the view.
  1315. //
  1316. // The gcexportdata machinery has no concept of package ID.
  1317. // It identifies packages by their PkgPath, which although not
  1318. // globally unique is unique within the scope of one invocation
  1319. // of the linker, type-checker, or gcexportdata.
  1320. //
  1321. // So, we must build a PkgPath-keyed view of the global
  1322. // (conceptually ID-keyed) cache of packages and pass it to
  1323. // gcexportdata. The view must contain every existing
  1324. // package that might possibly be mentioned by the
  1325. // current package---its transitive closure.
  1326. //
  1327. // In loadPackage, we unconditionally create a types.Package for
  1328. // each dependency so that export data loading does not
  1329. // create new ones.
  1330. //
  1331. // TODO(adonovan): it would be simpler and more efficient
  1332. // if the export data machinery invoked a callback to
  1333. // get-or-create a package instead of a map.
  1334. //
  1335. view := make(map[string]*types.Package) // view seen by gcexportdata
  1336. seen := make(map[*loaderPackage]bool) // all visited packages
  1337. var visit func(pkgs map[string]*Package)
  1338. visit = func(pkgs map[string]*Package) {
  1339. for _, p := range pkgs {
  1340. lpkg := ld.pkgs[p.ID]
  1341. if !seen[lpkg] {
  1342. seen[lpkg] = true
  1343. view[lpkg.PkgPath] = lpkg.Types
  1344. visit(lpkg.Imports)
  1345. }
  1346. }
  1347. }
  1348. visit(lpkg.Imports)
  1349. viewLen := len(view) + 1 // adding the self package
  1350. // Parse the export data.
  1351. // (May modify incomplete packages in view but not create new ones.)
  1352. tpkg, err := gcexportdata.Read(r, ld.Fset, view, lpkg.PkgPath)
  1353. if err != nil {
  1354. return fmt.Errorf("reading %s: %v", lpkg.ExportFile, err)
  1355. }
  1356. if _, ok := view["go.shape"]; ok {
  1357. // Account for the pseudopackage "go.shape" that gets
  1358. // created by generic code.
  1359. viewLen++
  1360. }
  1361. if viewLen != len(view) {
  1362. log.Panicf("golang.org/x/tools/go/packages: unexpected new packages during load of %s", lpkg.PkgPath)
  1363. }
  1364. lpkg.Types = tpkg
  1365. lpkg.IllTyped = false
  1366. return nil
  1367. }
  1368. // impliedLoadMode returns loadMode with its dependencies.
  1369. func impliedLoadMode(loadMode LoadMode) LoadMode {
  1370. if loadMode&(NeedDeps|NeedTypes|NeedTypesInfo) != 0 {
  1371. // All these things require knowing the import graph.
  1372. loadMode |= NeedImports
  1373. }
  1374. if loadMode&NeedTypes != 0 {
  1375. // Types require the GoVersion from Module.
  1376. loadMode |= NeedModule
  1377. }
  1378. return loadMode
  1379. }
  1380. func usesExportData(cfg *Config) bool {
  1381. return cfg.Mode&NeedExportFile != 0 || cfg.Mode&NeedTypes != 0 && cfg.Mode&NeedDeps == 0
  1382. }
  1383. type unit struct{}