internal/lsp: support directory inclusion/exclusion filters

Users working in large repositories may want to include only selected
directories in their workspace to avoid memory usage and performance
slowdowns. Add support for inclusion/exclusion filters that control what
directories are searched for workspace packages and modules. Packages
that are excluded by the filter may still be loaded as non-workspace
packages if other things depend on them.

For a description of the option's syntax, see the documentation.

Note that because we don't have any way to communicate the filters to
packages.Load, we still run go list on the unfiltered workspace scope,
then throw away the irrelevant packages. That may cost us, especially in
workspaces with many files.

Comments on the naming welcome. Also, if you know any places I may have
missed applying the filter, please do tell. One thing I thought of is
file watching, but that's covered because allKnownSubdirs works off of
workspace files and those are already filtered.

Possible enhancements:
 - Support glob patterns.
 - Apply filters during the goimports scan.
 - Figure out how to apply the filters to packages.Load. I don't know
 how to do it while still being build system neutral though.

Closes golang/go#42473, assuming none of the enhancements are required.

Change-Id: I9006a7a361dc3bb3c11f78b05ff84981813035a0
Reviewed-on: https://go-review.googlesource.com/c/tools/+/275253
Trust: Heschi Kreinick <[email protected]>
Run-TryBot: Heschi Kreinick <[email protected]>
gopls-CI: kokoro <[email protected]>
TryBot-Result: Go Bot <[email protected]>
Reviewed-by: Rebecca Stambler <[email protected]>
Reviewed-by: Robert Findley <[email protected]>

Author: heschi

gopls/doc/settings.md

@@ -175,6 +175,21 @@ just "Foo.Field".
 
 
 Default: `"Dynamic"`.
+### **directoryFilters** *[]string*
+directoryFilters can be used to exclude unwanted directories from the
+workspace. By default, all directories are included. Filters are an
+operator, `+` to include and `-` to exclude, followed by a path prefix
+relative to the workspace folder. They are evaluated in order, and
+the last filter that applies to a path controls whether it is included.
+The path prefix can be empty, so an initial `-` excludes everything.
+
+Examples:
+Exclude node_modules: `-node_modules`
+Include only project_a: `-` (exclude everything), `+project_a`
+Include only project_a, but not node_modules inside it: `-`, `+project_a`, `-project_a/node_modules`
+
+
+Default: `[]`.
 <!-- END User: DO NOT MANUALLY EDIT THIS SECTION -->
 
 ## Experimental

gopls/internal/regtest/workspace_test.go

@@ -645,3 +645,96 @@ func main() {
 		}
 	})
 }
+
+func TestDirectoryFiltersLoads(t *testing.T) {
+	// exclude, and its error, should be excluded from the workspace.
+	const files = `
+-- go.mod --
+module example.com
+
+go 1.12
+-- exclude/exclude.go --
+package exclude
+
+const _ = Nonexistant
+`
+	cfg := EditorConfig{
+		DirectoryFilters: []string{"-exclude"},
+	}
+	withOptions(cfg).run(t, files, func(t *testing.T, env *Env) {
+		env.Await(NoDiagnostics("exclude/x.go"))
+	})
+}
+
+func TestDirectoryFiltersTransitiveDep(t *testing.T) {
+	// Even though exclude is excluded from the workspace, it should
+	// still be importable as a non-workspace package.
+	const files = `
+-- go.mod --
+module example.com
+
+go 1.12
+-- include/include.go --
+package include
+import "example.com/exclude"
+
+const _ = exclude.X
+-- exclude/exclude.go --
+package exclude
+
+const _ = Nonexistant // should be ignored, since this is a non-workspace package
+const X = 1
+`
+
+	cfg := EditorConfig{
+		DirectoryFilters: []string{"-exclude"},
+	}
+	withOptions(cfg).run(t, files, func(t *testing.T, env *Env) {
+		env.Await(
+			NoDiagnostics("exclude/exclude.go"), // filtered out
+			NoDiagnostics("include/include.go"), // successfully builds
+		)
+	})
+}
+
+func TestDirectoryFiltersWorkspaceModules(t *testing.T) {
+	// Define a module include.com which should be in the workspace, plus a
+	// module exclude.com which should be excluded and therefore come from
+	// the proxy.
+	const files = `
+-- include/go.mod --
+module include.com
+
+go 1.12
+
+require exclude.com v1.0.0
+-- include/include.go --
+package include
+
+import "exclude.com"
+
+var _ = exclude.X // satisfied only by the workspace version
+-- exclude/go.mod --
+module exclude.com
+
+go 1.12
+-- exclude/exclude.go --
+package exclude
+
+const X = 1
+`
+	const proxy = `
+-- [email protected]/go.mod --
+module exclude.com
+
+go 1.12
+-- [email protected]/exclude.go --
+package exclude
+`
+	cfg := EditorConfig{
+		DirectoryFilters: []string{"-exclude"},
+	}
+	withOptions(cfg, WithModes(Experimental), WithProxyFiles(proxy)).run(t, files, func(t *testing.T, env *Env) {
+		env.Await(env.DiagnosticAtRegexp("include/include.go", `exclude.(X)`))
+	})
+}

internal/lsp/cache/load.go

@@ -159,6 +159,11 @@ func (s *snapshot) load(ctx context.Context, allowNetwork bool, scopes ...interf
 		if isTestMain(pkg, s.view.gocache) {
 			continue
 		}
+		// Skip filtered packages. They may be added anyway if they're
+		// dependencies of non-filtered packages.
+		if s.view.allFilesExcluded(pkg) {
+			continue
+		}
 		// Set the metadata for this package.
 		m, err := s.setMetadata(ctx, packagePath(pkg.PkgPath), pkg, cfg, map[packageID]struct{}{})
 		if err != nil {

internal/lsp/cache/session.go

@@ -175,14 +175,14 @@ func (s *Session) createView(ctx context.Context, name string, folder, tempWorks
 	}
 	root := folder
 	if options.ExpandWorkspaceToModule {
-		root, err = findWorkspaceRoot(ctx, root, s, options.ExperimentalWorkspaceModule)
+		root, err = findWorkspaceRoot(ctx, root, s, pathExcludedByFilterFunc(options), options.ExperimentalWorkspaceModule)
 		if err != nil {
 			return nil, nil, func() {}, err
 		}
 	}
 
 	// Build the gopls workspace, collecting active modules in the view.
-	workspace, err := newWorkspace(ctx, root, s, ws.userGo111Module == off, options.ExperimentalWorkspaceModule)
+	workspace, err := newWorkspace(ctx, root, s, pathExcludedByFilterFunc(options), ws.userGo111Module == off, options.ExperimentalWorkspaceModule)
 	if err != nil {
 		return nil, nil, func() {}, err
 	}

internal/lsp/cache/snapshot.go

@@ -1608,12 +1608,12 @@ func (s *snapshot) buildBuiltinPackage(ctx context.Context, goFiles []string) er
 
 // BuildGoplsMod generates a go.mod file for all modules in the workspace. It
 // bypasses any existing gopls.mod.
-func BuildGoplsMod(ctx context.Context, root span.URI, fs source.FileSource) (*modfile.File, error) {
-	allModules, err := findModules(ctx, root, 0)
+func BuildGoplsMod(ctx context.Context, root span.URI, s source.Snapshot) (*modfile.File, error) {
+	allModules, err := findModules(ctx, root, pathExcludedByFilterFunc(s.View().Options()), 0)
 	if err != nil {
 		return nil, err
 	}
-	return buildWorkspaceModFile(ctx, allModules, fs)
+	return buildWorkspaceModFile(ctx, allModules, s)
 }
 
 // TODO(rfindley): move this to workspacemodule.go

internal/lsp/cache/view.go

@@ -23,6 +23,7 @@ import (
 
 	"golang.org/x/mod/modfile"
 	"golang.org/x/mod/semver"
+	"golang.org/x/tools/go/packages"
 	"golang.org/x/tools/internal/event"
 	"golang.org/x/tools/internal/gocommand"
 	"golang.org/x/tools/internal/imports"
@@ -242,6 +243,9 @@ func minorOptionsChange(a, b *source.Options) bool {
 	if !reflect.DeepEqual(a.Env, b.Env) {
 		return false
 	}
+	if !reflect.DeepEqual(a.DirectoryFilters, b.DirectoryFilters) {
+		return false
+	}
 	aBuildFlags := make([]string, len(a.BuildFlags))
 	bBuildFlags := make([]string, len(b.BuildFlags))
 	copy(aBuildFlags, a.BuildFlags)
@@ -323,7 +327,16 @@ func (s *snapshot) RunProcessEnvFunc(ctx context.Context, fn func(*imports.Optio
 }
 
 func (v *View) contains(uri span.URI) bool {
-	return source.InDir(v.rootURI.Filename(), uri.Filename()) || source.InDir(v.folder.Filename(), uri.Filename())
+	inRoot := source.InDir(v.rootURI.Filename(), uri.Filename())
+	inFolder := source.InDir(v.folder.Filename(), uri.Filename())
+	if !inRoot && !inFolder {
+		return false
+	}
+	// Filters are applied relative to the workspace folder.
+	if inFolder {
+		return !pathExcludedByFilter(strings.TrimPrefix(uri.Filename(), v.folder.Filename()), v.Options())
+	}
+	return true
 }
 
 func (v *View) mapFile(uri span.URI, f *fileBase) {
@@ -699,7 +712,7 @@ func go111moduleForVersion(go111module string, goversion int) go111module {
 // Otherwise, it returns folder.
 // TODO (rFindley): move this to workspace.go
 // TODO (rFindley): simplify this once workspace modules are enabled by default.
-func findWorkspaceRoot(ctx context.Context, folder span.URI, fs source.FileSource, experimental bool) (span.URI, error) {
+func findWorkspaceRoot(ctx context.Context, folder span.URI, fs source.FileSource, excludePath func(string) bool, experimental bool) (span.URI, error) {
 	patterns := []string{"go.mod"}
 	if experimental {
 		patterns = []string{"gopls.mod", "go.mod"}
@@ -720,7 +733,7 @@ func findWorkspaceRoot(ctx context.Context, folder span.URI, fs source.FileSourc
 	}
 
 	// ...else we should check if there's exactly one nested module.
-	all, err := findModules(ctx, folder, 2)
+	all, err := findModules(ctx, folder, excludePath, 2)
 	if err == errExhausted {
 		// Fall-back behavior: if we don't find any modules after searching 10000
 		// files, assume there are none.
@@ -915,3 +928,43 @@ func (s *snapshot) vendorEnabled(ctx context.Context, modURI span.URI, modConten
 	vendorEnabled := modFile.Go != nil && modFile.Go.Version != "" && semver.Compare("v"+modFile.Go.Version, "v1.14") >= 0
 	return vendorEnabled, nil
 }
+
+func (v *View) allFilesExcluded(pkg *packages.Package) bool {
+	opts := v.Options()
+	folder := filepath.ToSlash(v.folder.Filename())
+	for _, f := range pkg.GoFiles {
+		f = filepath.ToSlash(f)
+		if !strings.HasPrefix(f, folder) {
+			return false
+		}
+		if !pathExcludedByFilter(strings.TrimPrefix(f, folder), opts) {
+			return false
+		}
+	}
+	return true
+}
+
+func pathExcludedByFilterFunc(opts *source.Options) func(string) bool {
+	return func(path string) bool {
+		return pathExcludedByFilter(path, opts)
+	}
+}
+
+func pathExcludedByFilter(path string, opts *source.Options) bool {
+	path = strings.TrimPrefix(filepath.ToSlash(path), "/")
+
+	excluded := false
+	for _, filter := range opts.DirectoryFilters {
+		op, prefix := filter[0], filter[1:]
+		// Non-empty prefixes have to be precise directory matches.
+		if prefix != "" {
+			prefix = prefix + "/"
+			path = path + "/"
+		}
+		if !strings.HasPrefix(path, prefix) {
+			continue
+		}
+		excluded = op == '-'
+	}
+	return excluded
+}

internal/lsp/cache/view_test.go

@@ -95,7 +95,8 @@ module fg
 		ctx := context.Background()
 		rel := fake.RelativeTo(dir)
 		folderURI := span.URIFromPath(rel.AbsPath(test.folder))
-		got, err := findWorkspaceRoot(ctx, folderURI, osFileSource{}, test.experimental)
+		excludeNothing := func(string) bool { return false }
+		got, err := findWorkspaceRoot(ctx, folderURI, osFileSource{}, excludeNothing, test.experimental)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -210,3 +211,49 @@ func TestInVendor(t *testing.T) {
 		}
 	}
 }
+
+func TestFilters(t *testing.T) {
+	tests := []struct {
+		filters  []string
+		included []string
+		excluded []string
+	}{
+		{
+			included: []string{"x"},
+		},
+		{
+			filters:  []string{"-"},
+			excluded: []string{"x", "x/a"},
+		},
+		{
+			filters:  []string{"-x", "+y"},
+			included: []string{"y", "y/a", "z"},
+			excluded: []string{"x", "x/a"},
+		},
+		{
+			filters:  []string{"-x", "+x/y", "-x/y/z"},
+			included: []string{"x/y", "x/y/a", "a"},
+			excluded: []string{"x", "x/a", "x/y/z/a"},
+		},
+		{
+			filters:  []string{"+foobar", "-foo"},
+			included: []string{"foobar", "foobar/a"},
+			excluded: []string{"foo", "foo/a"},
+		},
+	}
+
+	for _, tt := range tests {
+		opts := &source.Options{}
+		opts.DirectoryFilters = tt.filters
+		for _, inc := range tt.included {
+			if pathExcludedByFilter(inc, opts) {
+				t.Errorf("filters %q excluded %v, wanted included", tt.filters, inc)
+			}
+		}
+		for _, exc := range tt.excluded {
+			if !pathExcludedByFilter(exc, opts) {
+				t.Errorf("filters %q included %v, wanted excluded", tt.filters, exc)
+			}
+		}
+	}
+}