feat(elastic-agent-services): add another tag to docker image for agentless use (#6874)

* feat(elastic-agent-services): add another tag to docker image for agentless use

Author: olegsu

dev-tools/mage/dockerbuilder.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"io"
 	"io/fs"
+	"maps"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -53,13 +54,13 @@ func (b *dockerBuilder) Build() error {
 		return fmt.Errorf("failed to prepare build: %w", err)
 	}
 
-	tag, err := b.dockerBuild()
+	tag, additionalTags, err := b.dockerBuild()
 	tries := 3
 	for err != nil && tries != 0 {
 		fmt.Println(">> Building docker images again (after 10 s)")
 		// This sleep is to avoid hitting the docker build issues when resources are not available.
 		time.Sleep(time.Second * 10)
-		tag, err = b.dockerBuild()
+		tag, additionalTags, err = b.dockerBuild()
 		tries--
 	}
 	if err != nil {
@@ -70,6 +71,16 @@ func (b *dockerBuilder) Build() error {
 		return fmt.Errorf("failed to save docker as artifact: %w", err)
 	}
 
+	// additional tags should not be created with
+	for _, tag := range additionalTags {
+		if err := b.dockerSave(tag, map[string]interface{}{
+			// effectively override the name used from b.ImageName() to the tag
+			"Name": strings.ReplaceAll(tag, ":", "-"),
+		}); err != nil {
+			return fmt.Errorf("failed to save docker with tag %s as artifact: %w", tag, err)
+		}
+	}
+
 	return nil
 }
 
@@ -173,25 +184,43 @@ func (b *dockerBuilder) expandDockerfile(templatesDir string, data map[string]in
 	return nil
 }
 
-func (b *dockerBuilder) dockerBuild() (string, error) {
-	tag := fmt.Sprintf("%s:%s", b.imageName, b.Version)
+// dockerBuild runs "docker build -t t1 -t t2 ... buildDir"
+// returns the main tag additional tags if specified as part of extra_tags property
+// the extra tags are not push to the registry from b.ExtraVars["repository"]
+// returns an error if the command fails
+func (b *dockerBuilder) dockerBuild() (string, []string, error) {
+	mainTag := fmt.Sprintf("%s:%s", b.imageName, b.Version)
 	// For Independent Agent releases, replace the "+" with a "." since the "+" character
 	// currently isn't allowed in a tag in Docker
 	// E.g., 8.13.0+build202402191057 -> 8.13.0.build202402191057
-	tag = strings.Replace(tag, "+", ".", 1)
+	mainTag = strings.Replace(mainTag, "+", ".", 1)
 	if b.Snapshot {
-		tag = tag + "-SNAPSHOT"
+		mainTag = mainTag + "-SNAPSHOT"
 	}
 	if b.FIPS {
-		tag = tag + "-fips"
+		mainTag = mainTag + "-fips"
 	}
 	if repository := b.ExtraVars["repository"]; repository != "" {
-		tag = fmt.Sprintf("%s/%s", repository, tag)
+		mainTag = fmt.Sprintf("%s/%s", repository, mainTag)
+	}
+
+	args := []string{
+		"build",
+		"-t", mainTag,
 	}
-	return tag, sh.Run("docker", "build", "-t", tag, b.buildDir)
+	extraTags := []string{}
+	for _, tag := range b.ExtraTags {
+		extraTags = append(extraTags, fmt.Sprintf("%s:%s", b.imageName, tag))
+	}
+	for _, t := range extraTags {
+		args = append(args, "-t", t)
+	}
+	args = append(args, b.buildDir)
+
+	return mainTag, extraTags, sh.Run("docker", args...)
 }
 
-func (b *dockerBuilder) dockerSave(tag string) error {
+func (b *dockerBuilder) dockerSave(tag string, templateExtraArgs ...map[string]interface{}) error {
 	if _, err := os.Stat(distributionsDir); os.IsNotExist(err) {
 		err := os.MkdirAll(distributionsDir, 0750)
 		if err != nil {
@@ -201,9 +230,13 @@ func (b *dockerBuilder) dockerSave(tag string) error {
 	// Save the container as artifact
 	outputFile := b.OutputFile
 	if outputFile == "" {
-		outputTar, err := b.Expand(defaultBinaryName+".docker.tar.gz", map[string]interface{}{
+		args := map[string]interface{}{
 			"Name": b.imageName,
-		})
+		}
+		for _, extraArgs := range templateExtraArgs {
+			maps.Copy(args, extraArgs)
+		}
+		outputTar, err := b.Expand(defaultBinaryName+".docker.tar.gz", args)
 		if err != nil {
 			return err
 		}

dev-tools/mage/pkg_test.go

@@ -20,6 +20,9 @@ func testPackageSpec() PackageSpec {
 		Snapshot: true,
 		OS:       "windows",
 		Arch:     "x86_64",
+		ExtraTags: []string{
+			"git-{{ substring commit 0 12 }}",
+		},
 		Files: map[string]PackageFile{
 			"brewbeat.yml": PackageFile{
 				Source: "./testdata/config.yml",
@@ -66,6 +69,10 @@ func testPackage(t testing.TB, pack func(PackageSpec) error) {
 	readmePath := filepath.ToSlash(filepath.Clean(readme.Source))
 	assert.True(t, strings.HasPrefix(readmePath, packageStagingDir))
 
+	commit := spec.ExtraTags[0]
+	expected := "git-" + commitHash[:12]
+	assert.Equal(t, expected, commit)
+
 	if err := pack(spec); err != nil {
 		t.Fatal(err)
 	}

dev-tools/mage/pkgtypes.go

@@ -14,6 +14,7 @@ import (
 	"io"
 	"io/fs"
 	"log"
+	"math"
 	"os"
 	"path/filepath"
 	"reflect"
@@ -105,6 +106,7 @@ type PackageSpec struct {
 	Qualifier         string                 `yaml:"qualifier,omitempty"`   // Optional
 	OutputFile        string                 `yaml:"output_file,omitempty"` // Optional
 	ExtraVars         map[string]string      `yaml:"extra_vars,omitempty"`  // Optional
+	ExtraTags         []string               `yaml:"extra_tags,omitempty"`  // Optional
 
 	evalContext            map[string]interface{}
 	packageDir             string
@@ -385,6 +387,12 @@ func (s PackageSpec) Evaluate(args ...map[string]interface{}) PackageSpec {
 		s.evalContext[k] = mustExpand(v)
 	}
 
+	if s.ExtraTags != nil {
+		for i, tag := range s.ExtraTags {
+			s.ExtraTags[i] = mustExpand(tag)
+		}
+	}
+
 	s.Name = mustExpand(s.Name)
 	s.ServiceName = mustExpand(s.ServiceName)
 	s.OS = mustExpand(s.OS)
@@ -963,7 +971,7 @@ func addFileToTar(ar *tar.Writer, baseDir string, pkgFile PackageFile) error {
 		}
 
 		if mg.Verbose() {
-			log.Println("Adding", os.FileMode(header.Mode), header.Name) //nolint:gosec // we don't care about an int overflow in a log line
+			log.Println("Adding", os.FileMode(mustConvertToUnit32(header.Mode)), header.Name)
 		}
 		if err := ar.WriteHeader(header); err != nil {
 			return err
@@ -1031,7 +1039,7 @@ func addSymlinkToTar(tmpdir string, ar *tar.Writer, baseDir string, pkgFile Pack
 		header.Typeflag = tar.TypeSymlink
 
 		if mg.Verbose() {
-			log.Println("Adding", os.FileMode(header.Mode), header.Name) //nolint:gosec // we don't care about an int overflow in a log line
+			log.Println("Adding", os.FileMode(mustConvertToUnit32(header.Mode)), header.Name)
 		}
 		if err := ar.WriteHeader(header); err != nil {
 			return err
@@ -1053,3 +1061,10 @@ func PackageDocker(spec PackageSpec) error {
 	}
 	return b.Build()
 }
+
+func mustConvertToUnit32(i int64) uint32 {
+	if i > math.MaxUint32 {
+		panic(fmt.Sprintf("%d is bigger than math.MaxUint32", i))
+	}
+	return uint32(i) // #nosec
+}

dev-tools/mage/settings.go

@@ -113,6 +113,7 @@ var (
 		"title":                          func(s string) string { return cases.Title(language.English, cases.NoLower).String(s) },
 		"tolower":                        strings.ToLower,
 		"contains":                       strings.Contains,
+		"substring":                      Substring,
 		agentPackageVersionMappedFunc:    AgentPackageVersion,
 		agentManifestGeneratorMappedFunc: PackageManifest,
 		snapshotSuffix:                   SnapshotSuffix,
@@ -385,6 +386,17 @@ func SnapshotSuffix() string {
 	return GenerateSnapshotSuffix(Snapshot)
 }
 
+func Substring(s string, start, length int) string {
+	if start < 0 || start >= len(s) {
+		return ""
+	}
+	end := start + length
+	if end > len(s) {
+		end = len(s)
+	}
+	return s[start:end]
+}
+
 func GenerateSnapshotSuffix(snapshot bool) string {
 	if !snapshot {
 		return ""

dev-tools/packaging/packages.yml

@@ -288,6 +288,8 @@ shared:
   # service build is based on previous cloud variant
   - &agent_docker_service_spec
     docker_variant: 'service'
+    extra_tags:
+      - 'git-{{ substring commit 0 12 }}'
     files:
       'data/service/connectors-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}.zip':
         source: '{{.AgentDropPath}}/archives/{{.GOOS}}-{{.AgentArchName}}.tar.gz/connectors-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}.zip'