Export Prometheus telemetry in daemon mode

cli/board/list.go

@@ -42,7 +42,7 @@ func initListCommand() *cobra.Command {
 	}
 
 	listCommand.Flags().StringVar(&listFlags.timeout, "timeout", "0s",
-		"The timeout of the search of connected devices, try to increase it if your board is not found (e.g. to 10s).")
+		"The connected devices search timeout, raise it if your board doesn't show up (e.g. to 10s).")
 	return listCommand
 }
 

cli/cli.go

@@ -39,6 +39,7 @@ import (
 	"github.com/arduino/arduino-cli/cli/upload"
 	"github.com/arduino/arduino-cli/cli/version"
 	"github.com/arduino/arduino-cli/configuration"
+	"github.com/arduino/arduino-cli/inventory"
 	"github.com/mattn/go-colorable"
 	"github.com/rifflock/lfshook"
 	"github.com/sirupsen/logrus"
@@ -167,6 +168,9 @@ func preRun(cmd *cobra.Command, args []string) {
 	configuration.Init(configPath)
 	configFile := viper.ConfigFileUsed()
 
+	// initialize inventory
+	inventory.Init(viper.GetString("directories.Data"))
+
 	//
 	// Prepare logging
 	//

cli/daemon/daemon.go

@@ -34,6 +34,8 @@ import (
 	srv_debug "github.com/arduino/arduino-cli/rpc/debug"
 	srv_monitor "github.com/arduino/arduino-cli/rpc/monitor"
 	srv_settings "github.com/arduino/arduino-cli/rpc/settings"
+	"github.com/arduino/arduino-cli/telemetry"
+	"github.com/segmentio/stats/v4"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
@@ -59,16 +61,29 @@ func NewCommand() *cobra.Command {
 var daemonize bool
 
 func runDaemonCommand(cmd *cobra.Command, args []string) {
+
+	if viper.GetBool("telemetry.enabled") {
+		telemetry.Activate("daemon")
+		stats.Incr("daemon", stats.T("success", "true"))
+		defer stats.Flush()
+	}
+
 	port := viper.GetString("daemon.port")
 	s := grpc.NewServer()
 
-	// register the commands service
-	headers := http.Header{"User-Agent": []string{
-		fmt.Sprintf("%s/%s daemon (%s; %s; %s) Commit:%s",
-			globals.VersionInfo.Application,
-			globals.VersionInfo.VersionString,
-			runtime.GOARCH, runtime.GOOS,
-			runtime.Version(), globals.VersionInfo.Commit)}}
+	// Compose user agent header
+	headers := http.Header{
+		"User-Agent": []string{
+			fmt.Sprintf("%s/%s daemon (%s; %s; %s) Commit:%s",
+				globals.VersionInfo.Application,
+				globals.VersionInfo.VersionString,
+				runtime.GOARCH,
+				runtime.GOOS,
+				runtime.Version(),
+				globals.VersionInfo.Commit),
+		},
+	}
+	// Register the commands service
 	srv_commands.RegisterArduinoCoreServer(s, &daemon.ArduinoCoreServerImpl{
 		DownloaderHeaders: headers,
 		VersionString:     globals.VersionInfo.VersionString,
@@ -88,6 +103,8 @@ func runDaemonCommand(cmd *cobra.Command, args []string) {
 		go func() {
 			// Stdin is closed when the controlling parent process ends
 			_, _ = io.Copy(ioutil.Discard, os.Stdin)
+			// Flush telemetry stats (this is a no-op if telemetry is disabled)
+			stats.Flush()
 			os.Exit(0)
 		}()
 	}

commands/board/attach.go

@@ -28,13 +28,12 @@ import (
 	"github.com/arduino/arduino-cli/arduino/sketches"
 	"github.com/arduino/arduino-cli/commands"
 	rpc "github.com/arduino/arduino-cli/rpc/commands"
-	discovery "github.com/arduino/board-discovery"
-	paths "github.com/arduino/go-paths-helper"
+	"github.com/arduino/board-discovery"
+	"github.com/arduino/go-paths-helper"
 )
 
 // Attach FIXMEDOC
 func Attach(ctx context.Context, req *rpc.BoardAttachReq, taskCB commands.TaskProgressCB) (*rpc.BoardAttachResp, error) {
-
 	pm := commands.GetPackageManager(req.GetInstance().GetId())
 	if pm == nil {
 		return nil, errors.New("invalid instance")

commands/board/list.go

@@ -27,6 +27,7 @@ import (
 	"github.com/arduino/arduino-cli/commands"
 	rpc "github.com/arduino/arduino-cli/rpc/commands"
 	"github.com/pkg/errors"
+	"github.com/segmentio/stats/v4"
 	"github.com/sirupsen/logrus"
 )
 
@@ -104,13 +105,19 @@ func List(instanceID int32) ([]*rpc.DetectedPort, error) {
 	m.Lock()
 	defer m.Unlock()
 
+	tags := map[string]string{}
+	// Use defer func() to evaluate tags map when function returns
+	defer func() { stats.Incr("board.list", stats.M(tags)...) }()
+
 	pm := commands.GetPackageManager(instanceID)
 	if pm == nil {
+		tags["success"] = "false"
 		return nil, errors.New("invalid instance")
 	}
 
 	ports, err := commands.ListBoards(pm)
 	if err != nil {
+		tags["success"] = "false"
 		return nil, errors.Wrap(err, "error getting port list from serial-discovery")
 	}
 
@@ -136,6 +143,7 @@ func List(instanceID int32) ([]*rpc.DetectedPort, error) {
 				logrus.Debug("Board not recognized")
 			} else if err != nil {
 				// this is bad, bail out
+				tags["success"] = "false"
 				return nil, errors.Wrap(err, "error getting board info from Arduino Cloud")
 			}
 
@@ -156,5 +164,6 @@ func List(instanceID int32) ([]*rpc.DetectedPort, error) {
 		retVal = append(retVal, p)
 	}
 
+	tags["success"] = "true"
 	return retVal, nil
 }

commands/compile/compile.go

@@ -22,6 +22,7 @@ import (
 	"io"
 	"path/filepath"
 	"sort"
+	"strconv"
 	"strings"
 
 	"github.com/arduino/arduino-cli/arduino/cores"
@@ -33,26 +34,50 @@ import (
 	"github.com/arduino/arduino-cli/legacy/builder/i18n"
 	"github.com/arduino/arduino-cli/legacy/builder/types"
 	rpc "github.com/arduino/arduino-cli/rpc/commands"
+	"github.com/arduino/arduino-cli/telemetry"
 	paths "github.com/arduino/go-paths-helper"
 	properties "github.com/arduino/go-properties-orderedmap"
+	"github.com/segmentio/stats/v4"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/viper"
 )
 
 // Compile FIXMEDOC
 func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.Writer, debug bool) (*rpc.CompileResp, error) {
+
+	tags := map[string]string{
+		"fqbn":            req.Fqbn,
+		"sketchPath":      telemetry.Sanitize(req.SketchPath),
+		"showProperties":  strconv.FormatBool(req.ShowProperties),
+		"preprocess":      strconv.FormatBool(req.Preprocess),
+		"buildProperties": strings.Join(req.BuildProperties, ","),
+		"warnings":        req.Warnings,
+		"verbose":         strconv.FormatBool(req.Verbose),
+		"quiet":           strconv.FormatBool(req.Quiet),
+		"vidPid":          req.VidPid,
+		"exportFile":      telemetry.Sanitize(req.ExportFile),
+		"jobs":            strconv.FormatInt(int64(req.Jobs), 10),
+		"libraries":       strings.Join(req.Libraries, ","),
+	}
+
+	// Use defer func() to evaluate tags map when function returns
+	defer func() { stats.Incr("compile", stats.M(tags)...) }()
+
 	pm := commands.GetPackageManager(req.GetInstance().GetId())
 	if pm == nil {
+		tags["success"] = "false"
 		return nil, errors.New("invalid instance")
 	}
 
 	logrus.Tracef("Compile %s for %s started", req.GetSketchPath(), req.GetFqbn())
 	if req.GetSketchPath() == "" {
+		tags["success"] = "false"
 		return nil, fmt.Errorf("missing sketchPath")
 	}
 	sketchPath := paths.New(req.GetSketchPath())
 	sketch, err := sketches.NewSketchFromPath(sketchPath)
 	if err != nil {
+		tags["success"] = "false"
 		return nil, fmt.Errorf("opening sketch: %s", err)
 	}
 
@@ -61,10 +86,12 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W
 		fqbnIn = sketch.Metadata.CPU.Fqbn
 	}
 	if fqbnIn == "" {
+		tags["success"] = "false"
 		return nil, fmt.Errorf("no FQBN provided")
 	}
 	fqbn, err := cores.ParseFQBN(fqbnIn)
 	if err != nil {
+		tags["success"] = "false"
 		return nil, fmt.Errorf("incorrect FQBN: %s", err)
 	}
 
@@ -78,6 +105,7 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W
 		// 	"\"%[1]s:%[2]s\" platform is not installed, please install it by running \""+
 		// 		version.GetAppName()+" core install %[1]s:%[2]s\".", fqbn.Package, fqbn.PlatformArch)
 		// feedback.Error(errorMessage)
+		tags["success"] = "false"
 		return nil, fmt.Errorf("platform not installed")
 	}
 
@@ -97,6 +125,7 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W
 		builderCtx.BuildPath = paths.New(req.GetBuildPath())
 		err = builderCtx.BuildPath.MkdirAll()
 		if err != nil {
+			tags["success"] = "false"
 			return nil, fmt.Errorf("cannot create build directory: %s", err)
 		}
 	}
@@ -125,6 +154,7 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W
 		builderCtx.BuildCachePath = paths.New(req.GetBuildCachePath())
 		err = builderCtx.BuildCachePath.MkdirAll()
 		if err != nil {
+			tags["success"] = "false"
 			return nil, fmt.Errorf("cannot create build cache directory: %s", err)
 		}
 	}
@@ -157,13 +187,16 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W
 
 	// if --preprocess or --show-properties were passed, we can stop here
 	if req.GetShowProperties() {
+		tags["success"] = "true"
 		return &rpc.CompileResp{}, builder.RunParseHardwareAndDumpBuildProperties(builderCtx)
 	} else if req.GetPreprocess() {
+		tags["success"] = "true"
 		return &rpc.CompileResp{}, builder.RunPreprocess(builderCtx)
 	}
 
 	// if it's a regular build, go on...
 	if err := builder.RunBuilder(builderCtx); err != nil {
+		tags["success"] = "false"
 		return nil, err
 	}
 
@@ -199,6 +232,7 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W
 	// Copy "sketch.ino.*.hex" / "sketch.ino.*.bin" artifacts to sketch directory
 	srcDir, err := outputPath.Parent().ReadDir() // read "/build/path/*"
 	if err != nil {
+		tags["success"] = "false"
 		return nil, fmt.Errorf("reading build directory: %s", err)
 	}
 	srcDir.FilterPrefix(base + ".")
@@ -209,6 +243,7 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W
 		dstOutput := exportPath.Join(exportFile + srcFilename)
 		logrus.WithField("from", srcOutput).WithField("to", dstOutput).Debug("copying sketch build output")
 		if err = srcOutput.CopyTo(dstOutput); err != nil {
+			tags["success"] = "false"
 			return nil, fmt.Errorf("copying output file: %s", err)
 		}
 	}
@@ -219,11 +254,12 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W
 		dstElf := exportPath.Join(exportFile + ".elf")
 		logrus.WithField("from", srcElf).WithField("to", dstElf).Debug("copying sketch build output")
 		if err = srcElf.CopyTo(dstElf); err != nil {
+			tags["success"] = "false"
 			return nil, fmt.Errorf("copying elf file: %s", err)
 		}
 	}
 
 	logrus.Tracef("Compile %s for %s successful", sketch.Name, fqbnIn)
-
+	tags["success"] = "true"
 	return &rpc.CompileResp{}, nil
 }

configuration/defaults.go

@@ -35,4 +35,8 @@ func setDefaults(dataDir, userDir string) {
 
 	// daemon settings
 	viper.SetDefault("daemon.port", "50051")
+
+	//telemetry settings
+	viper.SetDefault("telemetry.enabled", true)
+	viper.SetDefault("telemetry.addr", ":9090")
 }

docs/getting-started.md

@@ -309,3 +309,40 @@ [email protected] downloaded
 Installing [email protected]...
 Installed [email protected]
 ```
+
+
+Using the ``daemon`` mode and the gRPC interface
+------------------------------------------------
+
+Arduino CLI can be launched as a gRPC server via the `daemon` command.
+
+The [client_example] folder contains a sample client code that shows how to
+interact with the gRPC server. Available services and messages are detailed
+in the [gRPC reference] pages.
+
+
+To provide observability for the gRPC server activities besides logs,
+the `daemon` mode activates and exposes by default a [Prometheus](https://prometheus.io/)
+endpoint (http://localhost:9090/metrics) that can be fetched for
+telemetry data like:
+
+```text
+# TYPE daemon_compile counter
+daemon_compile{buildProperties="",exportFile="",fqbn="arduino:samd:mkr1000",installationID="ed6f1f22-1fbe-4b1f-84be-84d035b6369c",jobs="0",libraries="",preprocess="false",quiet="false",showProperties="false",sketchPath="5ff767c6fa5a91230f5cb4e267c889aa61489ab2c4f70f35f921f934c1462cb6",success="true",verbose="true",vidPid="",warnings=""} 1 1580385724726
+
+# TYPE daemon_board_list counter
+daemon_board_list{installationID="ed6f1f22-1fbe-4b1f-84be-84d035b6369c",success="true"} 1 1580385724833
+```
+
+The telemetry settings are exposed via the ``telemetry`` section
+in the CLI configuration:
+
+```yaml
+telemetry:
+  enabled: true
+  addr: :9090
+```
+
+[client_example]: https://github.com/arduino/arduino-cli/blob/master/client_example
+[gRPC reference]: /rpc/commands
+[Prometheus]: https://prometheus.io/

go.mod

@@ -17,12 +17,12 @@ require (
 	github.com/fluxio/multierror v0.0.0-20160419044231-9c68d39025e5 // indirect
 	github.com/fsnotify/fsnotify v1.4.7
 	github.com/go-errors/errors v1.0.1
+	github.com/gofrs/uuid v3.2.0+incompatible
 	github.com/golang/protobuf v1.3.3
 	github.com/h2non/filetype v1.0.8 // indirect
 	github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5 // indirect
 	github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8 // indirect
 	github.com/juju/testing v0.0.0-20190429233213-dfc56b8c09fc // indirect
-	github.com/kr/pretty v0.1.0 // indirect
 	github.com/mattn/go-colorable v0.1.2
 	github.com/mattn/go-runewidth v0.0.2 // indirect
 	github.com/miekg/dns v1.0.5 // indirect
@@ -31,11 +31,11 @@ require (
 	github.com/pmylund/sortutil v0.0.0-20120526081524-abeda66eb583
 	github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5
 	github.com/schollz/closestmatch v2.1.0+incompatible
+	github.com/segmentio/stats/v4 v4.5.3
 	github.com/sirupsen/logrus v1.4.2
-	github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a // indirect
 	github.com/spf13/cobra v0.0.5
 	github.com/spf13/jwalterweatherman v1.0.0
-	github.com/spf13/viper v1.3.2
+	github.com/spf13/viper v1.6.2
 	github.com/stretchr/testify v1.4.0
 	go.bug.st/cleanup v1.0.0
 	go.bug.st/downloader v1.1.0
@@ -49,5 +49,5 @@ require (
 	google.golang.org/grpc v1.27.0
 	gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
 	gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce // indirect
-	gopkg.in/yaml.v2 v2.2.2
+	gopkg.in/yaml.v2 v2.2.4
 )