Skip to content

Add remote execution for codegen #2214

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Apr 19, 2023
Merged

Add remote execution for codegen #2214

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,3 +46,9 @@ mysqlsh:

proto:
buf generate

remote-proto:
protoc \
--go_out=. --go_opt="Minternal/remote/gen.proto=github.com/kyleconroy/sqlc/internal/remote" --go_opt=module=github.com/kyleconroy/sqlc \
--go-grpc_out=. --go-grpc_opt="Minternal/remote/gen.proto=github.com/kyleconroy/sqlc/internal/remote" --go-grpc_opt=module=github.com/kyleconroy/sqlc \
internal/remote/gen.proto
Comment on lines +49 to +54
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should get this working using buf generate to ensure that we're consistently using the same code for gRPC. This isn't blocking.

14 changes: 8 additions & 6 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,20 +15,17 @@ require (
github.com/lib/pq v1.10.7
github.com/mattn/go-sqlite3 v1.14.16
github.com/pganalyze/pg_query_go/v4 v4.2.0
github.com/riza-io/grpc-go v0.1.0
github.com/spf13/cobra v1.7.0
github.com/spf13/pflag v1.0.5
golang.org/x/sync v0.1.0
google.golang.org/grpc v1.54.0
google.golang.org/protobuf v1.30.0
gopkg.in/yaml.v3 v3.0.1
)

require (
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect
)

require (
github.com/golang/protobuf v1.5.2 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/jackc/chunkreader/v2 v2.0.1 // indirect
Expand All @@ -41,10 +38,15 @@ require (
github.com/pingcap/log v0.0.0-20210906054005-afc726e70354 // indirect
github.com/pingcap/tidb/parser v0.0.0-20220725134311-c80026e61f00
github.com/pkg/errors v0.9.1 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
go.uber.org/atomic v1.9.0 // indirect
go.uber.org/multierr v1.7.0 // indirect
go.uber.org/zap v1.19.1 // indirect
golang.org/x/crypto v0.6.0 // indirect
golang.org/x/text v0.7.0 // indirect
golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect
golang.org/x/net v0.8.0 // indirect
golang.org/x/sys v0.6.0 // indirect
golang.org/x/text v0.8.0 // indirect
google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect
gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
)
13 changes: 12 additions & 1 deletion go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk=
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/riza-io/grpc-go v0.1.0 h1:qm0j1YT0mqvtaGQ4A+sFe5odQSEE2OGnXGjTJAzQdUM=
github.com/riza-io/grpc-go v0.1.0/go.mod h1:2bDvR9KkKC3KhtlSHfR3dAXjUMT86kg4UfWFyVGWqi8=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU=
Expand Down Expand Up @@ -216,6 +218,8 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ=
golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
Expand All @@ -238,6 +242,8 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
Expand All @@ -248,8 +254,9 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68=
golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
Expand All @@ -268,6 +275,10 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f h1:BWUVssLB0HVOSY78gIdvk1dTVYtT1y8SBWtPYuTJ/6w=
google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM=
google.golang.org/grpc v1.54.0 h1:EhTqbhiYeixwWQtAEZAxmV9MGqcjEU2mFx52xCzNyag=
google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
Expand Down
12 changes: 8 additions & 4 deletions internal/cmd/cmd.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ func Do(args []string, stdin io.Reader, stdout io.Writer, stderr io.Writer) int
rootCmd := &cobra.Command{Use: "sqlc", SilenceUsage: true}
rootCmd.PersistentFlags().StringP("file", "f", "", "specify an alternate config file (default: sqlc.yaml)")
rootCmd.PersistentFlags().BoolP("experimental", "x", false, "DEPRECATED: enable experimental features (default: false)")
rootCmd.PersistentFlags().BoolP("no-remote", "nr", false, "disable remote execution (default: false)")

rootCmd.AddCommand(checkCmd)
rootCmd.AddCommand(diffCmd)
Expand Down Expand Up @@ -107,15 +108,18 @@ var initCmd = &cobra.Command{
}

type Env struct {
DryRun bool
Debug opts.Debug
DryRun bool
Debug opts.Debug
NoRemote bool
}

func ParseEnv(c *cobra.Command) Env {
dr := c.Flag("dry-run")
nr := c.Flag("no-remote")
return Env{
DryRun: dr != nil && dr.Changed,
Debug: opts.DebugFromEnv(),
DryRun: dr != nil && dr.Changed,
Debug: opts.DebugFromEnv(),
NoRemote: nr != nil && nr.Value.String() == "true",
}
}

Expand Down
71 changes: 71 additions & 0 deletions internal/cmd/generate.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (
"sync"

"golang.org/x/sync/errgroup"
"google.golang.org/grpc/status"

"github.com/kyleconroy/sqlc/internal/codegen/golang"
"github.com/kyleconroy/sqlc/internal/codegen/json"
Expand All @@ -23,9 +24,12 @@ import (
"github.com/kyleconroy/sqlc/internal/ext"
"github.com/kyleconroy/sqlc/internal/ext/process"
"github.com/kyleconroy/sqlc/internal/ext/wasm"
"github.com/kyleconroy/sqlc/internal/info"
"github.com/kyleconroy/sqlc/internal/multierr"
"github.com/kyleconroy/sqlc/internal/opts"
"github.com/kyleconroy/sqlc/internal/plugin"
"github.com/kyleconroy/sqlc/internal/remote"
"github.com/kyleconroy/sqlc/internal/sql/sqlpath"
)

const errMessageNoVersion = `The configuration file must have a version number.
Expand Down Expand Up @@ -140,6 +144,10 @@ func Generate(ctx context.Context, e Env, dir, filename string, stderr io.Writer
return nil, err
}

if conf.Cloud.Hostname != "" && !e.NoRemote {
return remoteGenerate(ctx, configPath, conf, dir, stderr)
}

output := map[string]string{}
errored := false

Expand Down Expand Up @@ -258,6 +266,69 @@ func Generate(ctx context.Context, e Env, dir, filename string, stderr io.Writer
return output, nil
}

func remoteGenerate(ctx context.Context, configPath string, conf *config.Config, dir string, stderr io.Writer) (map[string]string, error) {
rpcClient, err := remote.NewClient(conf.Cloud)
if err != nil {
fmt.Fprintf(stderr, "error creating rpc client: %s\n", err)
return nil, err
}

configBytes, err := os.ReadFile(configPath)
if err != nil {
fmt.Fprintf(stderr, "error reading config file %s: %s\n", configPath, err)
return nil, err
}

rpcReq := remote.GenerateRequest{
Version: info.Version,
Inputs: []*remote.File{{Path: filepath.Base(configPath), Bytes: configBytes}},
}

for _, pkg := range conf.SQL {
for _, paths := range []config.Paths{pkg.Schema, pkg.Queries} {
for i, relFilePath := range paths {
paths[i] = filepath.Join(dir, relFilePath)
}
files, err := sqlpath.Glob(paths)
if err != nil {
fmt.Fprintf(stderr, "error globbing paths: %s\n", err)
return nil, err
}
for _, filePath := range files {
fileBytes, err := os.ReadFile(filePath)
if err != nil {
fmt.Fprintf(stderr, "error reading file %s: %s\n", filePath, err)
return nil, err
}
fileRelPath, _ := filepath.Rel(dir, filePath)
rpcReq.Inputs = append(rpcReq.Inputs, &remote.File{Path: fileRelPath, Bytes: fileBytes})
}
}
}

rpcResp, err := rpcClient.Generate(ctx, &rpcReq)
if err != nil {
rpcStatus, ok := status.FromError(err)
if !ok {
return nil, err
}
fmt.Fprintf(stderr, "rpc error: %s", rpcStatus.Message())
return nil, rpcStatus.Err()
}

if rpcResp.ExitCode != 0 {
fmt.Fprintf(stderr, "%s", rpcResp.Stderr)
return nil, errors.New("remote execution returned with non-zero exit code")
}

output := map[string]string{}
for _, file := range rpcResp.Outputs {
output[filepath.Join(dir, file.Path)] = string(file.Bytes)
}

return output, nil
}

func parse(ctx context.Context, name, dir string, sql config.SQL, combo config.CombinedSettings, parserOpts opts.Parser, stderr io.Writer) (*compiler.Result, bool) {
defer trace.StartRegion(ctx, "parse").End()
c := compiler.NewCompiler(sql, combo)
Expand Down
3 changes: 2 additions & 1 deletion internal/endtoend/endtoend_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,8 @@ func TestReplay(t *testing.T) {
}

env := cmd.Env{
Debug: opts.DebugFromString(args.Env["SQLCDEBUG"]),
Debug: opts.DebugFromString(args.Env["SQLCDEBUG"]),
NoRemote: true,
}
switch args.Command {
case "diff":
Expand Down
31 changes: 31 additions & 0 deletions internal/endtoend/testdata/remote_missing_semicolon/mysql/go/db.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

15 changes: 15 additions & 0 deletions internal/endtoend/testdata/remote_missing_semicolon/mysql/go/models.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

26 changes: 26 additions & 0 deletions internal/endtoend/testdata/remote_missing_semicolon/mysql/go/query.sql.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

11 changes: 11 additions & 0 deletions internal/endtoend/testdata/remote_missing_semicolon/mysql/query.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
-- https://github.com/kyleconroy/sqlc/issues/1198
CREATE TABLE authors (
id INT PRIMARY KEY,
name VARCHAR(255) NOT NULL,
bio text
);

-- name: SetAuthor :exec
UPDATE authors
SET name = ?
WHERE id = ?
17 changes: 17 additions & 0 deletions internal/endtoend/testdata/remote_missing_semicolon/mysql/sqlc.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
{
"version": "1",
"cloud": {
"hostname": "localhost",
"organization": "foo",
"project": "bar"
},
"packages": [
{
"path": "go",
"engine": "mysql",
"name": "querytest",
"schema": "query.sql",
"queries": "query.sql"
}
]
}
7 changes: 7 additions & 0 deletions internal/endtoend/testdata/remote_missing_semicolon/pgx/v4/query.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
CREATE TABLE foo (email text not null);

-- name: FirstQuery :many
SELECT * FROM foo;

-- name: SecondQuery :many
SELECT * FROM foo WHERE email = $1
18 changes: 18 additions & 0 deletions internal/endtoend/testdata/remote_missing_semicolon/pgx/v4/sqlc.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
{
"version": "1",
"cloud": {
"hostname": "localhost",
"organization": "foo",
"project": "bar"
},
"packages": [
{
"path": "go",
"engine": "postgresql",
"sql_package": "pgx/v4",
"name": "querytest",
"schema": "query.sql",
"queries": "query.sql"
}
]
}
2 changes: 2 additions & 0 deletions internal/endtoend/testdata/remote_missing_semicolon/pgx/v4/stderr.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
# package querytest
query.sql:7:1: missing semicolon at end of file
7 changes: 7 additions & 0 deletions internal/endtoend/testdata/remote_missing_semicolon/pgx/v5/query.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
CREATE TABLE foo (email text not null);

-- name: FirstQuery :many
SELECT * FROM foo;

-- name: SecondQuery :many
SELECT * FROM foo WHERE email = $1
Loading