cmd/pkgsite: support the module cache

With the -cache flag, fetch modules from the module cache.

For golang/go#47780

Change-Id: I2aa6467955cd90a80ccae559f27928cca6e06079
Reviewed-on: https://go-review.googlesource.com/c/pkgsite/+/345272
Trust: Jonathan Amsterdam <[email protected]>
Run-TryBot: Jonathan Amsterdam <[email protected]>
Reviewed-by: Jamal Carvalho <[email protected]>
Reviewed-by: Julie Qiu <[email protected]>

Author: jba

cmd/pkgsite/main.go

@@ -26,6 +26,8 @@ import (
 	"fmt"
 	"net/http"
 	"os"
+	"os/exec"
+	"path/filepath"
 	"strings"
 	"time"
 
@@ -47,6 +49,8 @@ var (
 	_          = flag.String("static", "static", "path to folder containing static files served")
 	gopathMode = flag.Bool("gopath_mode", false, "assume that local modules' paths are relative to GOPATH/src")
 	httpAddr   = flag.String("http", defaultAddr, "HTTP service address to listen for incoming requests on")
+	useCache   = flag.Bool("cache", false, "fetch from the module cache")
+	cacheDir   = flag.String("cachedir", "", "module cache directory (defaults to `go env GOMODCACHE`)")
 	useProxy   = flag.Bool("proxy", false, "fetch from GOPROXY if not found locally")
 )
 
@@ -64,6 +68,23 @@ func main() {
 		paths = []string{"."}
 	}
 
+	var downloadDir string
+	if *useCache {
+		downloadDir = *cacheDir
+		if downloadDir == "" {
+			var err error
+			downloadDir, err = defaultCacheDir()
+			if err != nil {
+				die("%v", err)
+			}
+			if downloadDir == "" {
+				die("empty value for GOMODCACHE")
+			}
+		}
+		// We actually serve from the download subdirectory.
+		downloadDir = filepath.Join(downloadDir, "cache", "download")
+	}
+
 	var prox *proxy.Client
 	if *useProxy {
 		fmt.Fprintf(os.Stderr, "BYPASSING LICENSE CHECKING: MAY DISPLAY NON-REDISTRIBUTABLE INFORMATION\n")
@@ -77,7 +98,7 @@ func main() {
 			die("connecting to proxy: %s", err)
 		}
 	}
-	server, err := newServer(ctx, paths, *gopathMode, prox)
+	server, err := newServer(ctx, paths, *gopathMode, downloadDir, prox)
 	if err != nil {
 		die("%s", err)
 	}
@@ -101,8 +122,11 @@ func collectPaths(args []string) []string {
 	return paths
 }
 
-func newServer(ctx context.Context, paths []string, gopathMode bool, prox *proxy.Client) (*frontend.Server, error) {
+func newServer(ctx context.Context, paths []string, gopathMode bool, downloadDir string, prox *proxy.Client) (*frontend.Server, error) {
 	getters := buildGetters(ctx, paths, gopathMode)
+	if downloadDir != "" {
+		getters = append(getters, fetch.NewFSProxyModuleGetter(downloadDir))
+	}
 	if prox != nil {
 		getters = append(getters, fetch.NewProxyModuleGetter(prox))
 	}
@@ -148,3 +172,11 @@ func buildGetters(ctx context.Context, paths []string, gopathMode bool) []fetch.
 	}
 	return getters
 }
+
+func defaultCacheDir() (string, error) {
+	out, err := exec.Command("go", "env", "GOMODCACHE").CombinedOutput()
+	if err != nil {
+		return "", fmt.Errorf("running 'go env GOMODCACHE': %v: %s", err, out)
+	}
+	return strings.TrimSpace(string(out)), nil
+}

cmd/pkgsite/main_test.go

@@ -10,6 +10,7 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"path/filepath"
+	"strings"
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
@@ -18,25 +19,41 @@ import (
 
 func Test(t *testing.T) {
 	repoPath := func(fn string) string { return filepath.Join("..", "..", fn) }
+
 	localModule := repoPath("internal/fetch/testdata/has_go_mod")
+	cacheDir := repoPath("internal/fetch/testdata/modcache")
 	flag.Set("static", repoPath("static"))
 	testModules := proxytest.LoadTestModules(repoPath("internal/proxy/testdata"))
 	prox, teardown := proxytest.SetupTestClient(t, testModules)
 	defer teardown()
 
-	server, err := newServer(context.Background(), []string{localModule}, false, prox)
+	server, err := newServer(context.Background(), []string{localModule}, false, cacheDir, prox)
 	if err != nil {
 		t.Fatal(err)
 	}
 	mux := http.NewServeMux()
 	server.Install(mux.Handle, nil, nil)
-	w := httptest.NewRecorder()
 
-	for _, url := range []string{"/example.com/testmod", "/example.com/single/pkg"} {
-		mux.ServeHTTP(w, httptest.NewRequest("GET", url, nil))
-		if w.Code != http.StatusOK {
-			t.Errorf("%q: got status code = %d, want %d", url, w.Code, http.StatusOK)
-		}
+	for _, test := range []struct {
+		name       string
+		url        string
+		wantInBody string
+	}{
+		{"local", "example.com/testmod", "There is no documentation for this package."},
+		{"modcache", "[email protected]", "var V = 1"},
+		{"proxy", "example.com/single/pkg", "G is new in v1.1.0"},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			w := httptest.NewRecorder()
+			mux.ServeHTTP(w, httptest.NewRequest("GET", "/"+test.url, nil))
+			if w.Code != http.StatusOK {
+				t.Fatalf("got status code = %d, want %d", w.Code, http.StatusOK)
+			}
+			body := w.Body.String()
+			if !strings.Contains(body, test.wantInBody) {
+				t.Fatalf("body is missing %q\n%s", test.wantInBody, body)
+			}
+		})
 	}
 }
 

internal/fetch/testdata/modcache/modcache.com/@v/README

@@ -0,0 +1 @@
+This module exists only in this module cache.

internal/fetch/testdata/modcache/modcache.com/@v/v1.0.0.info

@@ -0,0 +1 @@
+{"Version":"v1.0.0","Time":"2019-03-30T17:04:38Z"}

internal/fetch/testdata/modcache/modcache.com/@v/v1.0.0.mod

@@ -0,0 +1,3 @@
+module modcache.com
+
+go 1.12

internal/fetch/testdata/modcache/modcache.com/@v/v1.0.0.zip

@@ -251,7 +251,7 @@ func (ds *FetchDataSource) GetLatestInfo(ctx context.Context, unitPath, modulePa
 	}
 
 	if latestUnitMeta == nil {
-		latestUnitMeta, err = ds.GetUnitMeta(ctx, unitPath, internal.UnknownModulePath, version.Latest)
+		latestUnitMeta, err = ds.GetUnitMeta(ctx, unitPath, modulePath, version.Latest)
 		if err != nil {
 			return latest, err
 		}