Skip to content

Commit 24c9f17

Browse files
hyangahgopherbot
authored andcommitted
internal/configstore: add a package for upload config download
Telemetry upload config is a Go module (golang.org/x/telemetry/config). that can be downloaded with `go mod download`. This allows telemetry configs to be cacheable, and verifiable like other Go modules. Moreover, 'go mod download' can download the config directly from the source repository, so we don't need a separate config serving infra. internal/proxy is a helper that builds a file-system based Go module proxy used for testing. Change-Id: I299946943fce05561879dfb05addec47404d6a32 Reviewed-on: https://go-review.googlesource.com/c/telemetry/+/499255 Reviewed-by: Jamal Carvalho <[email protected]> Run-TryBot: Hyang-Ah Hana Kim <[email protected]> TryBot-Result: Gopher Robot <[email protected]> Auto-Submit: Hyang-Ah Hana Kim <[email protected]>
1 parent ead61ca commit 24c9f17

File tree

6 files changed

+429
-0
lines changed

6 files changed

+429
-0
lines changed

go.mod

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
11
module golang.org/x/telemetry
22

33
go 1.20
4+
5+
require golang.org/x/mod v0.10.0

go.sum

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk=
2+
golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=

internal/configstore/download.go

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
// Copyright 2023 The Go Authors. All rights reserved.
2+
// Use of this source code is governed by a BSD-style
3+
// license that can be found in the LICENSE file.
4+
5+
// Package configstore abstracts interaction with the telemetry config server.
6+
// Telemetry config (golang.org/x/telemetry/config) is distributed as a go
7+
// module containing go.mod and config.json. Programs that upload collected
8+
// counters download the latest config using `go mod download`. This provides
9+
// verification of downloaded configuration and cacheability.
10+
package configstore
11+
12+
import (
13+
"bytes"
14+
"encoding/json"
15+
"fmt"
16+
"os"
17+
"os/exec"
18+
"path/filepath"
19+
20+
"golang.org/x/telemetry"
21+
)
22+
23+
const (
24+
configModulePath = "golang.org/x/telemetry/config"
25+
configFileName = "config.json"
26+
)
27+
28+
// DownloadOption is an option for Download.
29+
type DownloadOption struct {
30+
// Env holds the environment variables used when downloading the configuration.
31+
// If nil, the process's environment variables are used.
32+
Env []string
33+
}
34+
35+
// Download fetches the requested telemetry UploadConfig using "go mod download".
36+
func Download(version string, opts *DownloadOption) (telemetry.UploadConfig, error) {
37+
if version == "" {
38+
version = "latest"
39+
}
40+
if opts == nil {
41+
opts = &DownloadOption{}
42+
}
43+
modVer := configModulePath + "@" + version
44+
var stdout, stderr bytes.Buffer
45+
cmd := exec.Command("go", "mod", "download", "-json", modVer)
46+
cmd.Env = opts.Env
47+
cmd.Stdout = &stdout
48+
cmd.Stderr = &stderr
49+
if err := cmd.Run(); err != nil {
50+
return telemetry.UploadConfig{}, fmt.Errorf("failed to download config module: %w\n%s", err, &stderr)
51+
}
52+
53+
var info struct {
54+
Dir string
55+
Version string
56+
}
57+
if err := json.Unmarshal(stdout.Bytes(), &info); err != nil || info.Dir == "" {
58+
return telemetry.UploadConfig{}, fmt.Errorf("failed to download config module (invalid JSON): %w", err)
59+
}
60+
data, err := os.ReadFile(filepath.Join(info.Dir, configFileName))
61+
if err != nil {
62+
return telemetry.UploadConfig{}, fmt.Errorf("invalid config module: %w", err)
63+
}
64+
var cfg telemetry.UploadConfig
65+
if err := json.Unmarshal(data, &cfg); err != nil {
66+
return telemetry.UploadConfig{}, fmt.Errorf("invalid config: %w", err)
67+
}
68+
cfg.Version = info.Version
69+
return cfg, nil
70+
}

internal/configstore/download_test.go

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
// Copyright 2023 The Go Authors. All rights reserved.
2+
// Use of this source code is governed by a BSD-style
3+
// license that can be found in the LICENSE file.
4+
5+
package configstore
6+
7+
import (
8+
"encoding/json"
9+
"fmt"
10+
"os"
11+
"os/exec"
12+
"reflect"
13+
"testing"
14+
15+
"golang.org/x/telemetry"
16+
"golang.org/x/telemetry/internal/proxy"
17+
"golang.org/x/telemetry/internal/testenv"
18+
)
19+
20+
func TestDownload(t *testing.T) {
21+
testenv.NeedsGo(t)
22+
tmpdir := t.TempDir()
23+
defer cleanModuleCache(t, tmpdir)
24+
25+
configVersion := "v0.0.0-20230526221463-e8d11ddaba41"
26+
in := telemetry.UploadConfig{
27+
GOOS: []string{"darwin"},
28+
GOARCH: []string{"amd64", "arm64"},
29+
GoVersion: []string{"1.20.3", "1.20.4"},
30+
Programs: []*telemetry.ProgramConfig{{
31+
Name: "gopls",
32+
Versions: []string{"v0.11.0"},
33+
Counters: []telemetry.CounterConfig{{
34+
Name: "foobar",
35+
Rate: 2,
36+
}},
37+
}},
38+
}
39+
40+
proxyURI, err := writeConfig(tmpdir, in, configVersion)
41+
if err != nil {
42+
t.Fatal("failed to prepare proxy:", err)
43+
}
44+
45+
opts := testDownloadOption(proxyURI, tmpdir)
46+
47+
got, err := Download("latest", opts)
48+
if err != nil {
49+
t.Fatal("failed to download the latest version:", err)
50+
}
51+
52+
want := in
53+
want.Version = configVersion // want the Version field to be populated with the module version.
54+
if !reflect.DeepEqual(got, want) {
55+
t.Errorf("Download(latest, _) = %v\nwant %v", stringify(got), stringify(want))
56+
}
57+
}
58+
59+
func stringify(x any) string {
60+
ret, err := json.MarshalIndent(x, "", " ")
61+
if err != nil {
62+
return fmt.Sprintf("json.Marshal failed - %v", err)
63+
}
64+
return string(ret)
65+
}
66+
67+
// writeConfig adds cfg to the module proxy used for testing.
68+
func writeConfig(dir string, cfg telemetry.UploadConfig, version string) (proxyURI string, _ error) {
69+
encoded, err := json.Marshal(cfg)
70+
if err != nil {
71+
return "", err
72+
}
73+
dirPath := fmt.Sprintf("%v@%v/", configModulePath, version)
74+
files := map[string][]byte{
75+
dirPath + "go.mod": []byte("module " + configModulePath + "\n\ngo 1.20\n"),
76+
dirPath + "config.json": encoded,
77+
}
78+
return proxy.WriteProxy(dir, files)
79+
}
80+
81+
func testDownloadOption(proxyURI, tmpDir string) *DownloadOption {
82+
var env []string
83+
env = append(env, os.Environ()...)
84+
env = append(env,
85+
"GOPROXY="+proxyURI, // Use the fake proxy.
86+
"GONOSUMDB=*", // Skip verifying checksum against sum.golang.org.
87+
"GOMODCACHE="+tmpDir, // Don't pollute system module cache.
88+
)
89+
return &DownloadOption{
90+
Env: env,
91+
}
92+
}
93+
94+
func cleanModuleCache(t *testing.T, tmpDir string) {
95+
t.Helper()
96+
cmd := exec.Command("go", "clean", "-modcache")
97+
cmd.Env = append(cmd.Environ(), "GOMODCACHE="+tmpDir)
98+
out, err := cmd.CombinedOutput()
99+
if err != nil {
100+
t.Errorf("go clean -modcache failed: %v\n%s", err, out)
101+
}
102+
}

internal/proxy/proxy.go

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
// Copyright 2023 The Go Authors. All rights reserved.
2+
// Use of this source code is governed by a BSD-style
3+
// license that can be found in the LICENSE file.
4+
5+
// Package proxy provides functions for writing module data to a directory
6+
// in proxy format, so that it can be used as a module proxy by setting
7+
// GOPROXY="file://<dir>".
8+
// This is copied from golang.org/x/tools/gopls/internal/{proxydir,proxy}.
9+
package proxy
10+
11+
import (
12+
"archive/zip"
13+
"fmt"
14+
"io"
15+
"os"
16+
"path/filepath"
17+
"strings"
18+
19+
"golang.org/x/mod/module"
20+
)
21+
22+
// WriteProxy creates a new proxy file tree using the txtar-encoded content,
23+
// and returns its URL.
24+
func WriteProxy(tmpdir string, files map[string][]byte) (string, error) {
25+
type moduleVersion struct {
26+
modulePath, version string
27+
}
28+
// Transform into the format expected by the proxydir package.
29+
filesByModule := make(map[moduleVersion]map[string][]byte)
30+
for name, data := range files {
31+
modulePath, version, suffix := splitModuleVersionPath(name)
32+
mv := moduleVersion{modulePath, version}
33+
if _, ok := filesByModule[mv]; !ok {
34+
filesByModule[mv] = make(map[string][]byte)
35+
}
36+
filesByModule[mv][suffix] = data
37+
}
38+
for mv, files := range filesByModule {
39+
if err := writeModuleVersion(tmpdir, mv.modulePath, mv.version, files); err != nil {
40+
return "", fmt.Errorf("error writing %s@%s: %v", mv.modulePath, mv.version, err)
41+
}
42+
}
43+
return toURL(tmpdir), nil
44+
}
45+
46+
// splitModuleVersionPath extracts module information from files stored in the
47+
// directory structure modulePath@version/suffix.
48+
// For example:
49+
//
50+
// splitModuleVersionPath("[email protected]/package") = ("mod.com", "v1.2.3", "package")
51+
func splitModuleVersionPath(path string) (modulePath, version, suffix string) {
52+
parts := strings.Split(path, "/")
53+
var modulePathParts []string
54+
for i, p := range parts {
55+
if strings.Contains(p, "@") {
56+
mv := strings.SplitN(p, "@", 2)
57+
modulePathParts = append(modulePathParts, mv[0])
58+
return strings.Join(modulePathParts, "/"), mv[1], strings.Join(parts[i+1:], "/")
59+
}
60+
modulePathParts = append(modulePathParts, p)
61+
}
62+
// Default behavior: this is just a module path.
63+
return path, "", ""
64+
}
65+
66+
// writeModuleVersion creates a directory in the proxy dir for a module.
67+
func writeModuleVersion(rootDir, mod, ver string, files map[string][]byte) (rerr error) {
68+
dir := filepath.Join(rootDir, mod, "@v")
69+
if err := os.MkdirAll(dir, 0755); err != nil {
70+
return err
71+
}
72+
73+
// The go command checks for versions by looking at the "list" file. Since
74+
// we are supporting multiple versions, create this file if it does not exist
75+
// or append the version number to the preexisting file.
76+
77+
f, err := os.OpenFile(filepath.Join(dir, "list"), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
78+
if err != nil {
79+
return err
80+
}
81+
defer checkClose("list file", f, &rerr)
82+
if _, err := f.WriteString(ver + "\n"); err != nil {
83+
return err
84+
}
85+
86+
// Serve the go.mod file on the <version>.mod url, if it exists. Otherwise,
87+
// serve a stub.
88+
modContents, ok := files["go.mod"]
89+
if !ok {
90+
modContents = []byte("module " + mod)
91+
}
92+
if err := os.WriteFile(filepath.Join(dir, ver+".mod"), modContents, 0644); err != nil {
93+
return err
94+
}
95+
96+
// info file, just the bare bones.
97+
infoContents := []byte(fmt.Sprintf(`{"Version": "%v", "Time":"2017-12-14T13:08:43Z"}`, ver))
98+
if err := os.WriteFile(filepath.Join(dir, ver+".info"), infoContents, 0644); err != nil {
99+
return err
100+
}
101+
102+
// zip of all the source files.
103+
f, err = os.OpenFile(filepath.Join(dir, ver+".zip"), os.O_CREATE|os.O_WRONLY, 0644)
104+
if err != nil {
105+
return err
106+
}
107+
defer checkClose("zip file", f, &rerr)
108+
z := zip.NewWriter(f)
109+
defer checkClose("zip writer", z, &rerr)
110+
for name, contents := range files {
111+
zf, err := z.Create(mod + "@" + ver + "/" + name)
112+
if err != nil {
113+
return err
114+
}
115+
if _, err := zf.Write(contents); err != nil {
116+
return err
117+
}
118+
}
119+
120+
// Populate the /module/path/@latest that is used by @latest query.
121+
if module.IsPseudoVersion(ver) {
122+
latestFile := filepath.Join(rootDir, mod, "@latest")
123+
if err := os.WriteFile(latestFile, infoContents, 0644); err != nil {
124+
return err
125+
}
126+
}
127+
return nil
128+
}
129+
130+
func checkClose(name string, closer io.Closer, err *error) {
131+
if cerr := closer.Close(); cerr != nil && *err == nil {
132+
*err = fmt.Errorf("closing %s: %v", name, cerr)
133+
}
134+
}
135+
136+
// toURL returns the file uri for a proxy directory.
137+
func toURL(dir string) string {
138+
// file URLs on Windows must start with file:///. See golang.org/issue/6027.
139+
path := filepath.ToSlash(dir)
140+
if !strings.HasPrefix(path, "/") {
141+
path = "/" + path
142+
}
143+
return "file://" + path
144+
}

0 commit comments

Comments
 (0)