Changes from statik to go:embed for examples serving

This reduces build complexity by eliminating a generation step with go:embed. This implicitly reduces tech debt even more because not only were we using a stalled project, statik, but also a fork of it.

go:embed is not perfect, as it disallows the file name go.mod. The workaround is to rename it to go.mod_ per golang/go#45197. While this is imperfect an if-statement is a far better punch than a dependency on a fork of a stalled project which also requires a codegen step.

Author: Adrian Cole

.circleci/config.yml

@@ -39,7 +39,6 @@ jobs:
     resource_class: medium+
     steps:
       - checkout
-      - run: make init
       # prefetch implicit version needed by pkg/binary/envoy/controlplane/istio_test.go until #136. This avoids:
       # Unable to start Envoy process: fork/exec /home/circleci/.getenvoy/builds/standard/1.11.0/linux_glibc/bin/envoy: text file busy
       - run: go run cmd/getenvoy/main.go fetch standard:1.11.0

.github/workflows/commit.yaml

@@ -30,9 +30,6 @@ jobs:
         with:
           go-version: '1.16.2'
 
-      - name: "Init on first use"
-        run: make init
-
       - name: "Build the `getenvoy` binary"
         run: make bin
 

.gitignore

@@ -3,6 +3,3 @@ dist
 /getenvoy
 /build/
 .idea
-
-# code generated by `github.com/rakyll/statik`
-statik.go

.golangci.yml

@@ -134,11 +134,6 @@ issues:
         - gosec
         - lll
 
-    # Exclude lll issues for long lines with go:generate
-    - linters:
-        - lll
-      source: "^//go:generate "
-
   # Independently from option `exclude` we use default exclude patterns,
   # it can be disabled by this option. To list all
   # excluded by default patterns execute `golangci-lint run --help`.

.goreleaser.yaml

@@ -15,9 +15,6 @@
 project_name: getenvoy
 env:
   - GO111MODULE=on
-before:
-  hooks:
-    - make init
 builds:
 - binary: getenvoy
   ldflags: "-s -w -X github.com/tetratelabs/getenvoy/pkg/version.version={{.Version}}"

Makefile

@@ -68,22 +68,15 @@ GETENVOY_OUT_PATH = $(BIN_DIR)/$(1)/$(2)/getenvoy
 
 define GEN_GETENVOY_BUILD_TARGET
 .PHONY: $(call GETENVOY_OUT_PATH,$(1),$(2))
-$(call GETENVOY_OUT_PATH,$(1),$(2)): generate
+$(call GETENVOY_OUT_PATH,$(1),$(2)):
 	CGO_ENABLED=0 GOOS=$(1) GOARCH=$(2) go build $(GO_LD_FLAGS) -o $(call GETENVOY_OUT_PATH,$(1),$(2)) ./cmd/getenvoy/main.go
 endef
 $(foreach os,$(GOOSES),$(foreach arch,$(GOARCHS),$(eval $(call GEN_GETENVOY_BUILD_TARGET,$(os),$(arch)))))
 
-.PHONY: init
-init: generate
-
 .PHONY: deps
 deps:
 	go mod download
 
-.PHONY: generate
-generate: deps
-	go generate ./pkg/...
-
 .PHONY: build
 build: $(call GETENVOY_OUT_PATH,$(GOOS),$(GOARCH))
 
@@ -96,12 +89,12 @@ release.dryrun:
 	goreleaser release --skip-publish --snapshot --rm-dist
 
 .PHONY: test
-test: generate
+test:
 	docker-compose up -d
 	go test $(GO_TEST_OPTS) $(GO_TEST_EXTRA_OPTS) $(TEST_PKG_LIST)
 
 .PHONY: test.ci
-test.ci: generate
+test.ci:
 	go test $(GO_TEST_OPTS) $(GO_TEST_EXTRA_OPTS) $(TEST_PKG_LIST)
 
 .PHONY: e2e
@@ -125,7 +118,7 @@ endef
 $(foreach os,$(GOOSES),$(foreach arch,$(GOARCHS),$(eval $(call GEN_BIN_GOOS_GOARCH_TARGET,$(os),$(arch)))))
 
 .PHONY: coverage
-coverage: generate
+coverage:
 	mkdir -p "$(shell dirname "$(COVERAGE_PROFILE)")"
 	go test $(GO_COVERAGE_OPTS) $(GO_COVERAGE_EXTRA_OPTS) -coverprofile="$(COVERAGE_PROFILE)" $(COVERAGE_PKG_LIST)
 	go tool cover -html="$(COVERAGE_PROFILE)" -o "$(COVERAGE_REPORT)"
@@ -182,8 +175,7 @@ builders.pull: $(foreach lang,$(BUILDERS_LANGS), pull/builder/$(lang))
 
 LINT_OPTS ?= --timeout 5m
 .PHONY: lint
-# generate must be called while generated source is still used
-lint: generate $(GOLANGCI_LINT) $(SHFMT) $(LICENSER) .golangci.yml  ## Run the linters
+lint: $(GOLANGCI_LINT) $(SHFMT) $(LICENSER) .golangci.yml  ## Run the linters
 	@echo "--- lint ---"
 	@$(SHFMT) -d .
 	@$(LICENSER) verify -r .

data/example/init/templates.go

@@ -0,0 +1,36 @@
+// Copyright 2021 Tetrate
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package init
+
+import (
+	"embed"
+	"io/fs"
+)
+
+// templatesFs includes only the relative path of "templates".
+//
+// Assets must be in this directory because go:embed doesn't support navigation outside (ex. ../)
+// See https://pkg.go.dev/embed#hdr-Directives
+//go:embed templates/*
+var templatesFs embed.FS
+
+// GetTemplates returns the templates directory as a filesystem
+func GetTemplates() fs.FS {
+	f, err := fs.Sub(templatesFs, "templates")
+	if err != nil {
+		panic(err) // unexpected or a typo
+	}
+	return f
+}

data/extension/init/templates.go

@@ -0,0 +1,36 @@
+// Copyright 2021 Tetrate
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package init
+
+import (
+	"embed"
+	"io/fs"
+)
+
+// templatesFs includes only the relative path of "templates".
+//
+// Assets must be in this directory because go:embed doesn't support navigation outside (ex. ../)
+// See https://pkg.go.dev/embed#hdr-Directives
+//go:embed templates/*
+var templatesFs embed.FS
+
+// GetTemplates returns the templates directory as a filesystem
+func GetTemplates() fs.FS {
+	f, err := fs.Sub(templatesFs, "templates")
+	if err != nil {
+		panic(err) // unexpected or a typo
+	}
+	return f
+}

data/extension/init/templates/rust/.gitignore

@@ -1,2 +1,4 @@
+# We rename .cargo to cargo because go:embed doesn't recurse hidden directories
+.cargo
 Cargo.lock
 target/

data/extension/init/templates/rust/envoy.access_loggers/default/.cargo/config renamed to data/extension/init/templates/rust/envoy.access_loggers/default/cargo/config.toml

@@ -0,0 +1,2 @@
+# We rename go.mod to go.mod_ to workaround https://github.com/golang/go/issues/45197
+go.mod

data/extension/init/templates/rust/envoy.filters.http/default/.cargo/config renamed to data/extension/init/templates/rust/envoy.filters.http/default/cargo/config.toml

@@ -1,5 +1,6 @@
 module github.com/tetratelabs/getenvoy
 
+// This project uses go:embed, so requires minimally go 1.16
 go 1.16
 
 require (
@@ -16,7 +17,7 @@ require (
 	github.com/go-ole/go-ole v1.2.4 // indirect
 	github.com/golang/protobuf v1.3.5
 	github.com/gotestyourself/gotestyourself v2.2.0+incompatible // indirect
-	github.com/manifoldco/promptui v0.0.0-00010101000000-000000000000
+	github.com/manifoldco/promptui v0.8.0
 	github.com/mattn/go-isatty v0.0.12
 	github.com/mattn/go-shellwords v1.0.10
 	github.com/mholt/archiver v3.1.1+incompatible
@@ -27,7 +28,6 @@ require (
 	github.com/opencontainers/selinux v1.8.0 // indirect
 	github.com/otiai10/copy v1.2.0
 	github.com/pkg/errors v0.9.1
-	github.com/rakyll/statik v0.0.0-00010101000000-000000000000
 	github.com/schollz/progressbar/v2 v2.13.2
 	github.com/shirou/gopsutil v0.0.0-20190731134726-d80c43f9c984
 	github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749
@@ -36,6 +36,7 @@ require (
 	github.com/tetratelabs/getenvoy-package v0.0.0-20190730071641-da31aed4333e
 	github.com/tetratelabs/log v0.0.0-20190710134534-eb04d1e84fb8
 	github.com/tetratelabs/multierror v1.1.0
+	github.com/tetratelabs/proxy-wasm-go-sdk v0.12.0
 	gotest.tools v2.2.0+incompatible
 	istio.io/api v0.0.0-20200227213531-891bf31f3c32
 	istio.io/istio v0.0.0-20200304114959-c3c353285578
@@ -47,7 +48,3 @@ replace github.com/Azure/go-autorest/autorest => github.com/Azure/go-autorest/au
 replace github.com/docker/docker => github.com/docker/docker v17.12.1-ce+incompatible
 
 replace github.com/hashicorp/consul => github.com/hashicorp/consul v1.3.1
-
-replace github.com/manifoldco/promptui => github.com/yskopets/promptui v0.7.1-0.20200429230902-361491009c11
-
-replace github.com/rakyll/statik => github.com/yskopets/statik v0.1.8-0.20200501213002-c2d8dcc79889

data/extension/init/templates/rust/envoy.filters.network/default/.cargo/config renamed to data/extension/init/templates/rust/envoy.filters.network/default/cargo/config.toml

@@ -514,6 +514,8 @@ github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
+github.com/manifoldco/promptui v0.8.0 h1:R95mMF+McvXZQ7j1g8ucVZE1gLP3Sv6j9vlF9kyRqQo=
+github.com/manifoldco/promptui v0.8.0/go.mod h1:n4zTdgP0vr0S3w7/O/g98U+e0gwLScEXGwov2nIKuGQ=
 github.com/marstr/guid v1.1.0/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho=
 github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
@@ -747,6 +749,8 @@ github.com/tetratelabs/log v0.0.0-20190710134534-eb04d1e84fb8 h1:a7FN/XPymdzttMa
 github.com/tetratelabs/log v0.0.0-20190710134534-eb04d1e84fb8/go.mod h1:w+dEBsxcYEFg0I6whrgkMzjD8GBBQgmDq9hykB30pt8=
 github.com/tetratelabs/multierror v1.1.0 h1:cKmV/Pbf42K5wp8glxa2YIausbxIraPN8fzru9Pn1Cg=
 github.com/tetratelabs/multierror v1.1.0/go.mod h1:kH3SzI/z+FwEbV9bxQDx4GiIgE2djuyb8wiB2DaUBnY=
+github.com/tetratelabs/proxy-wasm-go-sdk v0.12.0 h1:n2foXJoViPNPz5Jr3bHMc8a4WXBuXwQ3bKJYpfXZUPE=
+github.com/tetratelabs/proxy-wasm-go-sdk v0.12.0/go.mod h1:y1ZQT4bQEBnR8Do4nSOzb3roczzPvcAp8UrF6NEYWNY=
 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
 github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
@@ -775,10 +779,6 @@ github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:
 github.com/yashtewari/glob-intersection v0.0.0-20180206001645-7af743e8ec84/go.mod h1:HptNXiXVDcJjXe9SqMd0v2FsL9f8dz4GnXgltU6q/co=
 github.com/yl2chen/cidranger v0.0.0-20180214081945-928b519e5268 h1:lkoOjizoHqOcEFsvYGE5c8Ykdijjnd0R3r1yDYHzLno=
 github.com/yl2chen/cidranger v0.0.0-20180214081945-928b519e5268/go.mod h1:mq0zhomp/G6rRTb0dvHWXRHr/2+Qgeq5hMXfJ670+i4=
-github.com/yskopets/promptui v0.7.1-0.20200429230902-361491009c11 h1:MlzMpHq1fRfH1RYzfQ7Ch7JjdGnBq/m29jJtPOExWuw=
-github.com/yskopets/promptui v0.7.1-0.20200429230902-361491009c11/go.mod h1:n4zTdgP0vr0S3w7/O/g98U+e0gwLScEXGwov2nIKuGQ=
-github.com/yskopets/statik v0.1.8-0.20200501213002-c2d8dcc79889 h1:f62aKW+gryXYYtNut+3b6i5n1ioXCXJWpDgA11l6Pak=
-github.com/yskopets/statik v0.1.8-0.20200501213002-c2d8dcc79889/go.mod h1:AlZONWzMtEnMs7W4e/1LURLiI49pIMmp6V9Unghqrcc=
 github.com/yuin/gopher-lua v0.0.0-20180316054350-84ea3a3c79b3/go.mod h1:aEV29XrmTYFr3CiRxZeGHpkvbwq+prZduBqMaascyCU=
 github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43 h1:+lm10QQTNSBd8DVTNGHx7o/IKu9HYDvLMffDhbyLccI=
 github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs=

data/extension/init/templates/tinygo/.gitignore

@@ -14,26 +14,19 @@
 
 package registry
 
-//go:generate go run github.com/rakyll/statik -p=templates -m -ns=example/init/templates -src=../../../../../data/example/init/templates -a -include=* -f
-
 import (
+	"net/http"
 	"path"
 
-	"github.com/rakyll/statik/fs"
-
-	// force execution of auto generated code
-	_ "github.com/tetratelabs/getenvoy/pkg/extension/example/init/registry/templates"
+	exampleTemplates "github.com/tetratelabs/getenvoy/data/example/init"
 	"github.com/tetratelabs/getenvoy/pkg/extension/workspace/config/extension"
 )
 
+var templatesFs = exampleTemplates.GetTemplates()
+
 func newDefaultRegistry() registry {
-	fileSystem, err := fs.NewWithNamespace("example/init/templates")
-	if err != nil {
-		// must be caught by unit tests
-		panic(err)
-	}
 	return &fsRegistry{
-		fs: fileSystem,
+		fs: http.FS(templatesFs),
 		namingScheme: func(category extension.Category, example string) string {
 			return "/" + path.Join(category.String(), example)
 		},

data/extension/init/templates/tinygo/envoy.access_loggers/default/go.mod renamed to data/extension/init/templates/tinygo/envoy.access_loggers/default/go.mod_

@@ -106,6 +106,10 @@ func (s *scaffolder) visit(sourceDirName, destinationDirName string, sourceFileI
 	if baseOutputFileName == "go.mod_" {
 		baseOutputFileName = "go.mod"
 	}
+	// We rename .cargo to cargo because go:embed doesn't recurse hidden directories
+	if baseOutputFileName == "cargo" {
+		baseOutputFileName = ".cargo"
+	}
 	relOutputFileName := filepath.Join(destinationDirName, baseOutputFileName)
 	outputFileName := filepath.Join(s.opts.OutputDir, relOutputFileName)
 	if err := osutil.EnsureDirExists(filepath.Dir(outputFileName)); err != nil {

data/extension/init/templates/tinygo/envoy.filters.http/default/go.mod renamed to data/extension/init/templates/tinygo/envoy.filters.http/default/go.mod_

@@ -14,18 +14,13 @@
 
 package init
 
-//go:generate go run github.com/rakyll/statik -p=templates -m -ns=extension/init/templates -src=../../../data/extension/init/templates -a -include=* -f
-
 import (
+	"net/http"
 	"path"
 	"sync"
 
-	"github.com/rakyll/statik/fs"
-
+	extensionTemplates "github.com/tetratelabs/getenvoy/data/extension/init"
 	"github.com/tetratelabs/getenvoy/pkg/extension/workspace/config/extension"
-
-	// force execution of auto generated code
-	_ "github.com/tetratelabs/getenvoy/pkg/extension/init/templates"
 )
 
 var (
@@ -34,17 +29,14 @@ var (
 	templatesOnce sync.Once
 )
 
+var templatesFs = extensionTemplates.GetTemplates()
+
 // getTemplateSource returns a source of extension templates.
 func getTemplateSource() templateSource {
 	// do lazy init to avoid zip decompression unless absolutely necessary
 	templatesOnce.Do(func() {
-		fileSystem, err := fs.NewWithNamespace("extension/init/templates")
-		if err != nil {
-			// must be caught by unit tests
-			panic(err)
-		}
 		templates = &fsTemplateSource{
-			fs: fileSystem,
+			fs: http.FS(templatesFs),
 			namingScheme: func(language extension.Language, category extension.Category, template string) string {
 				return "/" + path.Join(language.String(), category.String(), template)
 			},