wip: cache go/analysis facts

Author: jirfag

internal/cache/cache.go

@@ -0,0 +1,270 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package cache
+
+import (
+	"bytes"
+	"encoding/binary"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+func init() {
+	verify = false // even if GODEBUG is set
+}
+
+func TestBasic(t *testing.T) {
+	dir, err := ioutil.TempDir("", "cachetest-")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+	_, err = Open(filepath.Join(dir, "notexist"))
+	if err == nil {
+		t.Fatal(`Open("tmp/notexist") succeeded, want failure`)
+	}
+
+	cdir := filepath.Join(dir, "c1")
+	if err := os.Mkdir(cdir, 0777); err != nil {
+		t.Fatal(err)
+	}
+
+	c1, err := Open(cdir)
+	if err != nil {
+		t.Fatalf("Open(c1) (create): %v", err)
+	}
+	if err := c1.putIndexEntry(dummyID(1), dummyID(12), 13, true); err != nil {
+		t.Fatalf("addIndexEntry: %v", err)
+	}
+	if err := c1.putIndexEntry(dummyID(1), dummyID(2), 3, true); err != nil { // overwrite entry
+		t.Fatalf("addIndexEntry: %v", err)
+	}
+	if entry, err := c1.Get(dummyID(1)); err != nil || entry.OutputID != dummyID(2) || entry.Size != 3 {
+		t.Fatalf("c1.Get(1) = %x, %v, %v, want %x, %v, nil", entry.OutputID, entry.Size, err, dummyID(2), 3)
+	}
+
+	c2, err := Open(cdir)
+	if err != nil {
+		t.Fatalf("Open(c2) (reuse): %v", err)
+	}
+	if entry, err := c2.Get(dummyID(1)); err != nil || entry.OutputID != dummyID(2) || entry.Size != 3 {
+		t.Fatalf("c2.Get(1) = %x, %v, %v, want %x, %v, nil", entry.OutputID, entry.Size, err, dummyID(2), 3)
+	}
+	if err := c2.putIndexEntry(dummyID(2), dummyID(3), 4, true); err != nil {
+		t.Fatalf("addIndexEntry: %v", err)
+	}
+	if entry, err := c1.Get(dummyID(2)); err != nil || entry.OutputID != dummyID(3) || entry.Size != 4 {
+		t.Fatalf("c1.Get(2) = %x, %v, %v, want %x, %v, nil", entry.OutputID, entry.Size, err, dummyID(3), 4)
+	}
+}
+
+func TestGrowth(t *testing.T) {
+	dir, err := ioutil.TempDir("", "cachetest-")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+
+	c, err := Open(dir)
+	if err != nil {
+		t.Fatalf("Open: %v", err)
+	}
+
+	n := 10000
+	if testing.Short() {
+		n = 1000
+	}
+
+	for i := 0; i < n; i++ {
+		if err := c.putIndexEntry(dummyID(i), dummyID(i*99), int64(i)*101, true); err != nil {
+			t.Fatalf("addIndexEntry: %v", err)
+		}
+		id := ActionID(dummyID(i))
+		entry, err := c.Get(id)
+		if err != nil {
+			t.Fatalf("Get(%x): %v", id, err)
+		}
+		if entry.OutputID != dummyID(i*99) || entry.Size != int64(i)*101 {
+			t.Errorf("Get(%x) = %x, %d, want %x, %d", id, entry.OutputID, entry.Size, dummyID(i*99), int64(i)*101)
+		}
+	}
+	for i := 0; i < n; i++ {
+		id := ActionID(dummyID(i))
+		entry, err := c.Get(id)
+		if err != nil {
+			t.Fatalf("Get2(%x): %v", id, err)
+		}
+		if entry.OutputID != dummyID(i*99) || entry.Size != int64(i)*101 {
+			t.Errorf("Get2(%x) = %x, %d, want %x, %d", id, entry.OutputID, entry.Size, dummyID(i*99), int64(i)*101)
+		}
+	}
+}
+
+func TestVerifyPanic(t *testing.T) {
+	os.Setenv("GODEBUG", "gocacheverify=1")
+	initEnv()
+	defer func() {
+		os.Unsetenv("GODEBUG")
+		verify = false
+	}()
+
+	if !verify {
+		t.Fatal("initEnv did not set verify")
+	}
+
+	dir, err := ioutil.TempDir("", "cachetest-")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+
+	c, err := Open(dir)
+	if err != nil {
+		t.Fatalf("Open: %v", err)
+	}
+
+	id := ActionID(dummyID(1))
+	if err := c.PutBytes(id, []byte("abc")); err != nil {
+		t.Fatal(err)
+	}
+
+	defer func() {
+		if err := recover(); err != nil {
+			t.Log(err)
+			return
+		}
+	}()
+	c.PutBytes(id, []byte("def"))
+	t.Fatal("mismatched Put did not panic in verify mode")
+}
+
+func dummyID(x int) [HashSize]byte {
+	var out [HashSize]byte
+	binary.LittleEndian.PutUint64(out[:], uint64(x))
+	return out
+}
+
+func TestCacheTrim(t *testing.T) {
+	dir, err := ioutil.TempDir("", "cachetest-")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+
+	c, err := Open(dir)
+	if err != nil {
+		t.Fatalf("Open: %v", err)
+	}
+	const start = 1000000000
+	now := int64(start)
+	c.now = func() time.Time { return time.Unix(now, 0) }
+
+	checkTime := func(name string, mtime int64) {
+		t.Helper()
+		file := filepath.Join(c.dir, name[:2], name)
+		info, err := os.Stat(file)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if info.ModTime().Unix() != mtime {
+			t.Fatalf("%s mtime = %d, want %d", name, info.ModTime().Unix(), mtime)
+		}
+	}
+
+	id := ActionID(dummyID(1))
+	c.PutBytes(id, []byte("abc"))
+	entry, _ := c.Get(id)
+	c.PutBytes(ActionID(dummyID(2)), []byte("def"))
+	mtime := now
+	checkTime(fmt.Sprintf("%x-a", id), mtime)
+	checkTime(fmt.Sprintf("%x-d", entry.OutputID), mtime)
+
+	// Get should not change recent mtimes.
+	now = start + 10
+	c.Get(id)
+	checkTime(fmt.Sprintf("%x-a", id), mtime)
+	checkTime(fmt.Sprintf("%x-d", entry.OutputID), mtime)
+
+	// Get should change distant mtimes.
+	now = start + 5000
+	mtime2 := now
+	if _, err := c.Get(id); err != nil {
+		t.Fatal(err)
+	}
+	c.OutputFile(entry.OutputID)
+	checkTime(fmt.Sprintf("%x-a", id), mtime2)
+	checkTime(fmt.Sprintf("%x-d", entry.OutputID), mtime2)
+
+	// Trim should leave everything alone: it's all too new.
+	c.Trim()
+	if _, err := c.Get(id); err != nil {
+		t.Fatal(err)
+	}
+	c.OutputFile(entry.OutputID)
+	data, err := ioutil.ReadFile(filepath.Join(dir, "trim.txt"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	checkTime(fmt.Sprintf("%x-a", dummyID(2)), start)
+
+	// Trim less than a day later should not do any work at all.
+	now = start + 80000
+	c.Trim()
+	if _, err := c.Get(id); err != nil {
+		t.Fatal(err)
+	}
+	c.OutputFile(entry.OutputID)
+	data2, err := ioutil.ReadFile(filepath.Join(dir, "trim.txt"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !bytes.Equal(data, data2) {
+		t.Fatalf("second trim did work: %q -> %q", data, data2)
+	}
+
+	// Fast forward and do another trim just before the 5 day cutoff.
+	// Note that because of usedQuantum the cutoff is actually 5 days + 1 hour.
+	// We used c.Get(id) just now, so 5 days later it should still be kept.
+	// On the other hand almost a full day has gone by since we wrote dummyID(2)
+	// and we haven't looked at it since, so 5 days later it should be gone.
+	now += 5 * 86400
+	checkTime(fmt.Sprintf("%x-a", dummyID(2)), start)
+	c.Trim()
+	if _, err := c.Get(id); err != nil {
+		t.Fatal(err)
+	}
+	c.OutputFile(entry.OutputID)
+	mtime3 := now
+	if _, err := c.Get(dummyID(2)); err == nil { // haven't done a Get for this since original write above
+		t.Fatalf("Trim did not remove dummyID(2)")
+	}
+
+	// The c.Get(id) refreshed id's mtime again.
+	// Check that another 5 days later it is still not gone,
+	// but check by using checkTime, which doesn't bring mtime forward.
+	now += 5 * 86400
+	c.Trim()
+	checkTime(fmt.Sprintf("%x-a", id), mtime3)
+	checkTime(fmt.Sprintf("%x-d", entry.OutputID), mtime3)
+
+	// Half a day later Trim should still be a no-op, because there was a Trim recently.
+	// Even though the entry for id is now old enough to be trimmed,
+	// it gets a reprieve until the time comes for a new Trim scan.
+	now += 86400 / 2
+	c.Trim()
+	checkTime(fmt.Sprintf("%x-a", id), mtime3)
+	checkTime(fmt.Sprintf("%x-d", entry.OutputID), mtime3)
+
+	// Another half a day later, Trim should actually run, and it should remove id.
+	now += 86400/2 + 1
+	c.Trim()
+	if _, err := c.Get(dummyID(1)); err == nil {
+		t.Fatal("Trim did not remove dummyID(1)")
+	}
+}

internal/cache/cache_test.go

@@ -0,0 +1,85 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package cache
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+	"sync"
+)
+
+// Default returns the default cache to use.
+func Default() (*Cache, error) {
+	defaultOnce.Do(initDefaultCache)
+	return defaultCache, defaultDirErr
+}
+
+var (
+	defaultOnce  sync.Once
+	defaultCache *Cache
+)
+
+// cacheREADME is a message stored in a README in the cache directory.
+// Because the cache lives outside the normal Go trees, we leave the
+// README as a courtesy to explain where it came from.
+const cacheREADME = `This directory holds cached build artifacts from golangci-lint.
+`
+
+// initDefaultCache does the work of finding the default cache
+// the first time Default is called.
+func initDefaultCache() {
+	dir := DefaultDir()
+	if err := os.MkdirAll(dir, 0777); err != nil {
+		log.Fatalf("failed to initialize build cache at %s: %s\n", dir, err)
+	}
+	if _, err := os.Stat(filepath.Join(dir, "README")); err != nil {
+		// Best effort.
+		ioutil.WriteFile(filepath.Join(dir, "README"), []byte(cacheREADME), 0666)
+	}
+
+	c, err := Open(dir)
+	if err != nil {
+		log.Fatalf("failed to initialize build cache at %s: %s\n", dir, err)
+	}
+	defaultCache = c
+}
+
+var (
+	defaultDirOnce sync.Once
+	defaultDir     string
+	defaultDirErr  error
+)
+
+// DefaultDir returns the effective GOLANGCI_LINT_CACHE setting.
+func DefaultDir() string {
+	// Save the result of the first call to DefaultDir for later use in
+	// initDefaultCache. cmd/go/main.go explicitly sets GOCACHE so that
+	// subprocesses will inherit it, but that means initDefaultCache can't
+	// otherwise distinguish between an explicit "off" and a UserCacheDir error.
+
+	defaultDirOnce.Do(func() {
+		defaultDir = os.Getenv("GOLANGCI_LINT_CACHE")
+		if filepath.IsAbs(defaultDir) {
+			return
+		}
+		if defaultDir != "" {
+			defaultDirErr = fmt.Errorf("GOLANGCI_LINT_CACHE is not an absolute path")
+			return
+		}
+
+		// Compute default location.
+		dir, err := os.UserCacheDir()
+		if err != nil {
+			defaultDirErr = fmt.Errorf("GOLANGCI_LINT_CACHE is not defined and %v", err)
+			return
+		}
+		defaultDir = filepath.Join(dir, "golangci-lint")
+	})
+
+	return defaultDir
+}