You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

walk.go 7.6KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275
  1. // Copyright 2018 The Go Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style
  3. // license that can be found in the LICENSE file.
  4. // Package gopathwalk is like filepath.Walk but specialized for finding Go
  5. // packages, particularly in $GOPATH and $GOROOT.
  6. package gopathwalk
  7. import (
  8. "bufio"
  9. "bytes"
  10. "fmt"
  11. "go/build"
  12. "io/ioutil"
  13. "log"
  14. "os"
  15. "path/filepath"
  16. "strings"
  17. "time"
  18. "golang.org/x/tools/internal/fastwalk"
  19. )
  20. // Options controls the behavior of a Walk call.
  21. type Options struct {
  22. // If Logf is non-nil, debug logging is enabled through this function.
  23. Logf func(format string, args ...interface{})
  24. // Search module caches. Also disables legacy goimports ignore rules.
  25. ModulesEnabled bool
  26. }
  27. // RootType indicates the type of a Root.
  28. type RootType int
  29. const (
  30. RootUnknown RootType = iota
  31. RootGOROOT
  32. RootGOPATH
  33. RootCurrentModule
  34. RootModuleCache
  35. RootOther
  36. )
  37. // A Root is a starting point for a Walk.
  38. type Root struct {
  39. Path string
  40. Type RootType
  41. }
  42. // SrcDirsRoots returns the roots from build.Default.SrcDirs(). Not modules-compatible.
  43. func SrcDirsRoots(ctx *build.Context) []Root {
  44. var roots []Root
  45. roots = append(roots, Root{filepath.Join(ctx.GOROOT, "src"), RootGOROOT})
  46. for _, p := range filepath.SplitList(ctx.GOPATH) {
  47. roots = append(roots, Root{filepath.Join(p, "src"), RootGOPATH})
  48. }
  49. return roots
  50. }
  51. // Walk walks Go source directories ($GOROOT, $GOPATH, etc) to find packages.
  52. // For each package found, add will be called (concurrently) with the absolute
  53. // paths of the containing source directory and the package directory.
  54. // add will be called concurrently.
  55. func Walk(roots []Root, add func(root Root, dir string), opts Options) {
  56. WalkSkip(roots, add, func(Root, string) bool { return false }, opts)
  57. }
  58. // WalkSkip walks Go source directories ($GOROOT, $GOPATH, etc) to find packages.
  59. // For each package found, add will be called (concurrently) with the absolute
  60. // paths of the containing source directory and the package directory.
  61. // For each directory that will be scanned, skip will be called (concurrently)
  62. // with the absolute paths of the containing source directory and the directory.
  63. // If skip returns false on a directory it will be processed.
  64. // add will be called concurrently.
  65. // skip will be called concurrently.
  66. func WalkSkip(roots []Root, add func(root Root, dir string), skip func(root Root, dir string) bool, opts Options) {
  67. for _, root := range roots {
  68. walkDir(root, add, skip, opts)
  69. }
  70. }
  71. // walkDir creates a walker and starts fastwalk with this walker.
  72. func walkDir(root Root, add func(Root, string), skip func(root Root, dir string) bool, opts Options) {
  73. if _, err := os.Stat(root.Path); os.IsNotExist(err) {
  74. if opts.Logf != nil {
  75. opts.Logf("skipping nonexistent directory: %v", root.Path)
  76. }
  77. return
  78. }
  79. start := time.Now()
  80. if opts.Logf != nil {
  81. opts.Logf("gopathwalk: scanning %s", root.Path)
  82. }
  83. w := &walker{
  84. root: root,
  85. add: add,
  86. skip: skip,
  87. opts: opts,
  88. }
  89. w.init()
  90. if err := fastwalk.Walk(root.Path, w.walk); err != nil {
  91. log.Printf("gopathwalk: scanning directory %v: %v", root.Path, err)
  92. }
  93. if opts.Logf != nil {
  94. opts.Logf("gopathwalk: scanned %s in %v", root.Path, time.Since(start))
  95. }
  96. }
  97. // walker is the callback for fastwalk.Walk.
  98. type walker struct {
  99. root Root // The source directory to scan.
  100. add func(Root, string) // The callback that will be invoked for every possible Go package dir.
  101. skip func(Root, string) bool // The callback that will be invoked for every dir. dir is skipped if it returns true.
  102. opts Options // Options passed to Walk by the user.
  103. ignoredDirs []os.FileInfo // The ignored directories, loaded from .goimportsignore files.
  104. }
  105. // init initializes the walker based on its Options
  106. func (w *walker) init() {
  107. var ignoredPaths []string
  108. if w.root.Type == RootModuleCache {
  109. ignoredPaths = []string{"cache"}
  110. }
  111. if !w.opts.ModulesEnabled && w.root.Type == RootGOPATH {
  112. ignoredPaths = w.getIgnoredDirs(w.root.Path)
  113. ignoredPaths = append(ignoredPaths, "v", "mod")
  114. }
  115. for _, p := range ignoredPaths {
  116. full := filepath.Join(w.root.Path, p)
  117. if fi, err := os.Stat(full); err == nil {
  118. w.ignoredDirs = append(w.ignoredDirs, fi)
  119. if w.opts.Logf != nil {
  120. w.opts.Logf("Directory added to ignore list: %s", full)
  121. }
  122. } else if w.opts.Logf != nil {
  123. w.opts.Logf("Error statting ignored directory: %v", err)
  124. }
  125. }
  126. }
  127. // getIgnoredDirs reads an optional config file at <path>/.goimportsignore
  128. // of relative directories to ignore when scanning for go files.
  129. // The provided path is one of the $GOPATH entries with "src" appended.
  130. func (w *walker) getIgnoredDirs(path string) []string {
  131. file := filepath.Join(path, ".goimportsignore")
  132. slurp, err := ioutil.ReadFile(file)
  133. if w.opts.Logf != nil {
  134. if err != nil {
  135. w.opts.Logf("%v", err)
  136. } else {
  137. w.opts.Logf("Read %s", file)
  138. }
  139. }
  140. if err != nil {
  141. return nil
  142. }
  143. var ignoredDirs []string
  144. bs := bufio.NewScanner(bytes.NewReader(slurp))
  145. for bs.Scan() {
  146. line := strings.TrimSpace(bs.Text())
  147. if line == "" || strings.HasPrefix(line, "#") {
  148. continue
  149. }
  150. ignoredDirs = append(ignoredDirs, line)
  151. }
  152. return ignoredDirs
  153. }
  154. // shouldSkipDir reports whether the file should be skipped or not.
  155. func (w *walker) shouldSkipDir(fi os.FileInfo, dir string) bool {
  156. for _, ignoredDir := range w.ignoredDirs {
  157. if os.SameFile(fi, ignoredDir) {
  158. return true
  159. }
  160. }
  161. if w.skip != nil {
  162. // Check with the user specified callback.
  163. return w.skip(w.root, dir)
  164. }
  165. return false
  166. }
  167. // walk walks through the given path.
  168. func (w *walker) walk(path string, typ os.FileMode) error {
  169. dir := filepath.Dir(path)
  170. if typ.IsRegular() {
  171. if dir == w.root.Path && (w.root.Type == RootGOROOT || w.root.Type == RootGOPATH) {
  172. // Doesn't make sense to have regular files
  173. // directly in your $GOPATH/src or $GOROOT/src.
  174. return fastwalk.ErrSkipFiles
  175. }
  176. if !strings.HasSuffix(path, ".go") {
  177. return nil
  178. }
  179. w.add(w.root, dir)
  180. return fastwalk.ErrSkipFiles
  181. }
  182. if typ == os.ModeDir {
  183. base := filepath.Base(path)
  184. if base == "" || base[0] == '.' || base[0] == '_' ||
  185. base == "testdata" ||
  186. (w.root.Type == RootGOROOT && w.opts.ModulesEnabled && base == "vendor") ||
  187. (!w.opts.ModulesEnabled && base == "node_modules") {
  188. return filepath.SkipDir
  189. }
  190. fi, err := os.Lstat(path)
  191. if err == nil && w.shouldSkipDir(fi, path) {
  192. return filepath.SkipDir
  193. }
  194. return nil
  195. }
  196. if typ == os.ModeSymlink {
  197. base := filepath.Base(path)
  198. if strings.HasPrefix(base, ".#") {
  199. // Emacs noise.
  200. return nil
  201. }
  202. fi, err := os.Lstat(path)
  203. if err != nil {
  204. // Just ignore it.
  205. return nil
  206. }
  207. if w.shouldTraverse(dir, fi) {
  208. return fastwalk.ErrTraverseLink
  209. }
  210. }
  211. return nil
  212. }
  213. // shouldTraverse reports whether the symlink fi, found in dir,
  214. // should be followed. It makes sure symlinks were never visited
  215. // before to avoid symlink loops.
  216. func (w *walker) shouldTraverse(dir string, fi os.FileInfo) bool {
  217. path := filepath.Join(dir, fi.Name())
  218. target, err := filepath.EvalSymlinks(path)
  219. if err != nil {
  220. return false
  221. }
  222. ts, err := os.Stat(target)
  223. if err != nil {
  224. fmt.Fprintln(os.Stderr, err)
  225. return false
  226. }
  227. if !ts.IsDir() {
  228. return false
  229. }
  230. if w.shouldSkipDir(ts, dir) {
  231. return false
  232. }
  233. // Check for symlink loops by statting each directory component
  234. // and seeing if any are the same file as ts.
  235. for {
  236. parent := filepath.Dir(path)
  237. if parent == path {
  238. // Made it to the root without seeing a cycle.
  239. // Use this symlink.
  240. return true
  241. }
  242. parentInfo, err := os.Stat(parent)
  243. if err != nil {
  244. return false
  245. }
  246. if os.SameFile(ts, parentInfo) {
  247. // Cycle. Don't traverse.
  248. return false
  249. }
  250. path = parent
  251. }
  252. }