From 8861a4e0d6a32652d7931cb164ae15633e369860 Mon Sep 17 00:00:00 2001 From: Kyle Conroy Date: Wed, 15 Jun 2022 19:10:51 -0700 Subject: [PATCH 1/8] wasm: WASM-based codegen plugins --- go.mod | 1 + go.sum | 2 + internal/ext/wasm/nowasm.go | 13 ++++ internal/ext/wasm/wasm.go | 120 ++++++++++++++++++++++++++++++++++++ 4 files changed, 136 insertions(+) create mode 100644 internal/ext/wasm/nowasm.go create mode 100644 internal/ext/wasm/wasm.go diff --git a/go.mod b/go.mod index a0782bec66..c61e8d981a 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( ) require ( + github.com/bytecodealliance/wasmtime-go v0.37.0 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jackc/chunkreader/v2 v2.0.1 // indirect diff --git a/go.sum b/go.sum index 7c62716fcb..066f7f4f10 100644 --- a/go.sum +++ b/go.sum @@ -5,6 +5,8 @@ github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20220209173558-ad29539cd2e9 h1:z github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20220209173558-ad29539cd2e9/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/bytecodealliance/wasmtime-go v0.37.0 h1:eNP2Snp5UFMuGuunRPxwVETJ/WpC8LhWonZAklXJfjk= +github.com/bytecodealliance/wasmtime-go v0.37.0/go.mod h1:q320gUxqyI8yB+ZqRuaJOEnGkAnHh6WtJjMaT2CW4wI= github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= diff --git a/internal/ext/wasm/nowasm.go b/internal/ext/wasm/nowasm.go new file mode 100644 index 0000000000..42ce0e807d --- /dev/null +++ b/internal/ext/wasm/nowasm.go @@ -0,0 +1,13 @@ +//go:build !(cgo && ((linux && amd64) || (linux && arm64) || (darwin && amd64) || (darwin && arm64) || (windows && amd64))) + +package wasm + +import ( + "fmt" + + "github.com/kyleconroy/sqlc/internal/plugin" +) + +func Generate(req *plugin.CodeGenRequest) (*plugin.CodeGenResponse, error) { + return nil, fmt.Errorf("sqlc built without wasmtime support") +} diff --git a/internal/ext/wasm/wasm.go b/internal/ext/wasm/wasm.go new file mode 100644 index 0000000000..c823a79723 --- /dev/null +++ b/internal/ext/wasm/wasm.go @@ -0,0 +1,120 @@ +//go:build cgo && ((linux && amd64) || (linux && arm64) || (darwin && amd64) || (darwin && arm64) || (windows && amd64)) + +// The above build constraint is based of the cgo directives in this file: +// https://github.com/bytecodealliance/wasmtime-go/blob/main/ffi.go +package wasm + +import ( + "context" + _ "embed" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "runtime/trace" + + wasmtime "github.com/bytecodealliance/wasmtime-go" + + "github.com/kyleconroy/sqlc/internal/plugin" +) + +func Generate(req *plugin.CodeGenRequest) (*plugin.CodeGenResponse, error) { + ctx := context.Background() // XXX + + engine := wasmtime.NewEngine() + // module, err = wasmtime.NewModuleDeserialize(engine, pythonModule) + // if err != nil { + // panic(err) + // } + + // out, err := module.Serialize() + // if err != nil { + // panic(err) + // } + + // err = os.WriteFile("sqlc-codegen-python.module", out, 0644) + // if err != nil { + // panic(err) + // } + + linker := wasmtime.NewLinker(engine) + if err := linker.DefineWasi(); err != nil { + return nil, err + } + + stdinBlob, err := req.MarshalVT() + if err != nil { + return nil, err + } + + dir, err := ioutil.TempDir("", "out") + if err != nil { + return nil, fmt.Errorf("temp dir: %w", err) + } + + defer os.RemoveAll(dir) + stdinPath := filepath.Join(dir, "stdin") + stderrPath := filepath.Join(dir, "stderr") + stdoutPath := filepath.Join(dir, "stdout") + + if err := os.WriteFile(stdinPath, stdinBlob, 0755); err != nil { + return nil, fmt.Errorf("write file: %w", err) + } + + // Configure WASI imports to write stdout into a file. + wasiConfig := wasmtime.NewWasiConfig() + wasiConfig.SetStdinFile(stdinPath) + wasiConfig.SetStdoutFile(stdoutPath) + wasiConfig.SetStderrFile(stderrPath) + + store := wasmtime.NewStore(engine) + store.SetWasi(wasiConfig) + + // Set the version to the same as in the WAT. + // wasi, err := wasmtime.NewWasiInstance(store, wasiConfig, "wasi_snapshot_preview1") + // if err != nil { + // return fmt.Errorf("new wasi instances: %w", err) + // } + + // Create our module + // + // Compiling modules requires WebAssembly binary input, but the wasmtime + // package also supports converting the WebAssembly text format to the + // binary format. + wasm, err := os.ReadFile("foo.wasm") // req.Settings.Wasm.Path) + if err != nil { + return nil, fmt.Errorf("read file: %w", err) + } + + moduRegion := trace.StartRegion(ctx, "wasmtime.NewModule") + module, err := wasmtime.NewModule(store.Engine, wasm) + moduRegion.End() + if err != nil { + return nil, fmt.Errorf("define wasi: %w", err) + } + + linkRegion := trace.StartRegion(ctx, "linker.Instantiate") + instance, err := linker.Instantiate(store, module) + linkRegion.End() + if err != nil { + return nil, fmt.Errorf("define wasi: %w", err) + } + + // Run the function + + callRegion := trace.StartRegion(ctx, "call _start") + nom := instance.GetExport(store, "_start").Func() + _, err = nom.Call(store) + callRegion.End() + if err != nil { + return nil, fmt.Errorf("call: %w", err) + } + + // Print WASM stdout + stdoutBlob, err := os.ReadFile(stdoutPath) + if err != nil { + return nil, fmt.Errorf("read file: %w", err) + } + var resp plugin.CodeGenResponse + return &resp, resp.UnmarshalVT(stdoutBlob) +} From 1c5b33dbca8e2975ddbeea77b7b16f81cc53f8d5 Mon Sep 17 00:00:00 2001 From: Kyle Conroy Date: Wed, 15 Jun 2022 22:24:48 -0700 Subject: [PATCH 2/8] Download from a URL --- internal/cmd/generate.go | 32 +++++++++++++++++++++++++++++--- internal/ext/process/gen.go | 26 +++----------------------- internal/ext/wasm/nowasm.go | 5 ++++- internal/ext/wasm/wasm.go | 22 ++++++++++++++++++---- 4 files changed, 54 insertions(+), 31 deletions(-) diff --git a/internal/cmd/generate.go b/internal/cmd/generate.go index 3afd79220f..5825f14178 100644 --- a/internal/cmd/generate.go +++ b/internal/cmd/generate.go @@ -20,6 +20,7 @@ import ( "github.com/kyleconroy/sqlc/internal/debug" "github.com/kyleconroy/sqlc/internal/ext" "github.com/kyleconroy/sqlc/internal/ext/process" + "github.com/kyleconroy/sqlc/internal/ext/wasm" "github.com/kyleconroy/sqlc/internal/multierr" "github.com/kyleconroy/sqlc/internal/opts" ) @@ -51,6 +52,15 @@ type outPair struct { config.SQL } +func findPlugin(conf config.Config, name string) (*config.Plugin, error) { + for _, plug := range conf.Plugins { + if plug.Name == name { + return &plug, nil + } + } + return nil, fmt.Errorf("plugin not found") +} + func readConfig(stderr io.Writer, dir, filename string) (string, *config.Config, error) { configPath := "" if filename != "" { @@ -241,12 +251,28 @@ func Generate(ctx context.Context, e Env, dir, filename string, stderr io.Writer case sql.Plugin != nil: out = sql.Plugin.Out - handler = &process.Runner{ - Config: combo.Global, - Plugin: sql.Plugin.Plugin, + plug, err := findPlugin(combo.Global, sql.Plugin.Plugin) + if err != nil { + // TODO: Return a real error + panic("plugin not found") + } + + switch { + case plug.Process != nil: + handler = &process.Runner{ + Cmd: plug.Process.Cmd, + } + case plug.WASM != nil: + handler = &wasm.Runner{ + URL: plug.WASM.URL, + } + default: + // TODO: Return a real error + panic("unsupported plugin type") } default: + // TODO: Return a real error panic("missing language backend") } resp, err := handler.Generate(codeGenRequest(result, combo)) diff --git a/internal/ext/process/gen.go b/internal/ext/process/gen.go index 09e68038ef..a7fbd4a8f5 100644 --- a/internal/ext/process/gen.go +++ b/internal/ext/process/gen.go @@ -9,26 +9,11 @@ import ( "google.golang.org/protobuf/proto" - "github.com/kyleconroy/sqlc/internal/config" "github.com/kyleconroy/sqlc/internal/plugin" ) type Runner struct { - Config config.Config - Plugin string -} - -func (r Runner) pluginCmd() (string, error) { - for _, plug := range r.Config.Plugins { - if plug.Name != r.Plugin { - continue - } - if plug.Process == nil { - continue - } - return plug.Process.Cmd, nil - } - return "", fmt.Errorf("plugin not found") + Cmd string } // TODO: Update the gen func signature to take a ctx @@ -38,15 +23,10 @@ func (r Runner) Generate(req *plugin.CodeGenRequest) (*plugin.CodeGenResponse, e return nil, fmt.Errorf("failed to encode codegen request: %s", err) } - name, err := r.pluginCmd() - if err != nil { - return nil, fmt.Errorf("process: unknown plugin %s", r.Plugin) - } - // Check if the output plugin exists - path, err := exec.LookPath(name) + path, err := exec.LookPath(r.Cmd) if err != nil { - return nil, fmt.Errorf("process: %s not found", name) + return nil, fmt.Errorf("process: %s not found", r.Cmd) } ctx := context.Background() diff --git a/internal/ext/wasm/nowasm.go b/internal/ext/wasm/nowasm.go index 42ce0e807d..e32100fcad 100644 --- a/internal/ext/wasm/nowasm.go +++ b/internal/ext/wasm/nowasm.go @@ -8,6 +8,9 @@ import ( "github.com/kyleconroy/sqlc/internal/plugin" ) -func Generate(req *plugin.CodeGenRequest) (*plugin.CodeGenResponse, error) { +type Runner struct { +} + +func (r *Runner) Generate(req *plugin.CodeGenRequest) (*plugin.CodeGenResponse, error) { return nil, fmt.Errorf("sqlc built without wasmtime support") } diff --git a/internal/ext/wasm/wasm.go b/internal/ext/wasm/wasm.go index c823a79723..b74f42b0ac 100644 --- a/internal/ext/wasm/wasm.go +++ b/internal/ext/wasm/wasm.go @@ -8,7 +8,9 @@ import ( "context" _ "embed" "fmt" + "io" "io/ioutil" + "net/http" "os" "path/filepath" "runtime/trace" @@ -18,7 +20,11 @@ import ( "github.com/kyleconroy/sqlc/internal/plugin" ) -func Generate(req *plugin.CodeGenRequest) (*plugin.CodeGenResponse, error) { +type Runner struct { + URL string +} + +func (r *Runner) Generate(req *plugin.CodeGenRequest) (*plugin.CodeGenResponse, error) { ctx := context.Background() // XXX engine := wasmtime.NewEngine() @@ -81,13 +87,21 @@ func Generate(req *plugin.CodeGenRequest) (*plugin.CodeGenResponse, error) { // Compiling modules requires WebAssembly binary input, but the wasmtime // package also supports converting the WebAssembly text format to the // binary format. - wasm, err := os.ReadFile("foo.wasm") // req.Settings.Wasm.Path) + // + hresp, err := http.Get(r.URL) if err != nil { - return nil, fmt.Errorf("read file: %w", err) + return nil, fmt.Errorf("http get: %w", err) + } + + defer hresp.Body.Close() + + wmod, err := io.ReadAll(hresp.Body) + if err != nil { + return nil, fmt.Errorf("readall: %w", err) } moduRegion := trace.StartRegion(ctx, "wasmtime.NewModule") - module, err := wasmtime.NewModule(store.Engine, wasm) + module, err := wasmtime.NewModule(store.Engine, wmod) moduRegion.End() if err != nil { return nil, fmt.Errorf("define wasi: %w", err) From 7143d69e656a5bfdf79923185c1f573df5cd750e Mon Sep 17 00:00:00 2001 From: Kyle Conroy Date: Tue, 21 Jun 2022 15:26:02 -0700 Subject: [PATCH 3/8] feat: Cache downloaded and compiled plugins Only do the compilcation once --- internal/cmd/generate.go | 3 +- internal/config/config.go | 3 +- .../wasm_plugin_sqlc_gen_json/gen/hello.txt | 1 + .../wasm_plugin_sqlc_gen_json/query.sql | 19 ++ .../wasm_plugin_sqlc_gen_json/schema.sql | 5 + .../wasm_plugin_sqlc_gen_json/sqlc.json | 29 +++ internal/ext/wasm/wasm.go | 201 +++++++++++++----- 7 files changed, 209 insertions(+), 52 deletions(-) create mode 100644 internal/endtoend/testdata/wasm_plugin_sqlc_gen_json/gen/hello.txt create mode 100644 internal/endtoend/testdata/wasm_plugin_sqlc_gen_json/query.sql create mode 100644 internal/endtoend/testdata/wasm_plugin_sqlc_gen_json/schema.sql create mode 100644 internal/endtoend/testdata/wasm_plugin_sqlc_gen_json/sqlc.json diff --git a/internal/cmd/generate.go b/internal/cmd/generate.go index 5825f14178..e743e927b3 100644 --- a/internal/cmd/generate.go +++ b/internal/cmd/generate.go @@ -264,7 +264,8 @@ func Generate(ctx context.Context, e Env, dir, filename string, stderr io.Writer } case plug.WASM != nil: handler = &wasm.Runner{ - URL: plug.WASM.URL, + URL: plug.WASM.URL, + Checksum: plug.WASM.Checksum, } default: // TODO: Return a real error diff --git a/internal/config/config.go b/internal/config/config.go index f12df00685..a9f07da646 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -90,7 +90,8 @@ type Plugin struct { Cmd string `json:"cmd" yaml:"cmd"` } `json:"process" yaml:"process"` WASM *struct { - URL string `json:"url" yaml:"url"` + URL string `json:"url" yaml:"url"` + Checksum string `json:"checksum" yaml:"checksum"` } `json:"wasm" yaml:"wasm"` } diff --git a/internal/endtoend/testdata/wasm_plugin_sqlc_gen_json/gen/hello.txt b/internal/endtoend/testdata/wasm_plugin_sqlc_gen_json/gen/hello.txt new file mode 100644 index 0000000000..5e1c309dae --- /dev/null +++ b/internal/endtoend/testdata/wasm_plugin_sqlc_gen_json/gen/hello.txt @@ -0,0 +1 @@ +Hello World \ No newline at end of file diff --git a/internal/endtoend/testdata/wasm_plugin_sqlc_gen_json/query.sql b/internal/endtoend/testdata/wasm_plugin_sqlc_gen_json/query.sql new file mode 100644 index 0000000000..75e38b2caf --- /dev/null +++ b/internal/endtoend/testdata/wasm_plugin_sqlc_gen_json/query.sql @@ -0,0 +1,19 @@ +-- name: GetAuthor :one +SELECT * FROM authors +WHERE id = $1 LIMIT 1; + +-- name: ListAuthors :many +SELECT * FROM authors +ORDER BY name; + +-- name: CreateAuthor :one +INSERT INTO authors ( + name, bio +) VALUES ( + $1, $2 +) +RETURNING *; + +-- name: DeleteAuthor :exec +DELETE FROM authors +WHERE id = $1; diff --git a/internal/endtoend/testdata/wasm_plugin_sqlc_gen_json/schema.sql b/internal/endtoend/testdata/wasm_plugin_sqlc_gen_json/schema.sql new file mode 100644 index 0000000000..b4fad78497 --- /dev/null +++ b/internal/endtoend/testdata/wasm_plugin_sqlc_gen_json/schema.sql @@ -0,0 +1,5 @@ +CREATE TABLE authors ( + id BIGSERIAL PRIMARY KEY, + name text NOT NULL, + bio text +); diff --git a/internal/endtoend/testdata/wasm_plugin_sqlc_gen_json/sqlc.json b/internal/endtoend/testdata/wasm_plugin_sqlc_gen_json/sqlc.json new file mode 100644 index 0000000000..06e3392053 --- /dev/null +++ b/internal/endtoend/testdata/wasm_plugin_sqlc_gen_json/sqlc.json @@ -0,0 +1,29 @@ +{ + "version": "2", + "sql": [ + { + "schema": "schema.sql", + "queries": "query.sql", + "engine": "postgresql", + "codegen": [ + { + "out": "gen", + "plugin": "jsonb", + "options": { + "indent": " ", + "filename": "codegen.json" + } + } + ] + } + ], + "plugins": [ + { + "name": "jsonb", + "wasm": { + "url": "file:///Users/kyle/Downloads/sqlc-gen-json-wasm.wasm", + "checksum": "sha256/afc486dac2068d741d7a4110146559d12a013fd0286f42a2fc7dcd802424ad07" + } + } + ] +} diff --git a/internal/ext/wasm/wasm.go b/internal/ext/wasm/wasm.go index b74f42b0ac..1a6ac9ac07 100644 --- a/internal/ext/wasm/wasm.go +++ b/internal/ext/wasm/wasm.go @@ -6,6 +6,7 @@ package wasm import ( "context" + "crypto/sha256" _ "embed" "fmt" "io" @@ -14,45 +15,177 @@ import ( "os" "path/filepath" "runtime/trace" + "strings" wasmtime "github.com/bytecodealliance/wasmtime-go" "github.com/kyleconroy/sqlc/internal/plugin" ) +func cacheDir() (string, error) { + // Use the checksum to see if it already existsin the modcache + cache := os.Getenv("SQLCCACHE") + if cache != "" { + return cache, nil + } + home, err := os.UserHomeDir() + if err != nil { + return "", err + } + return filepath.Join(home, "sqlc", "mod"), nil +} + type Runner struct { - URL string + URL string + Checksum string } -func (r *Runner) Generate(req *plugin.CodeGenRequest) (*plugin.CodeGenResponse, error) { - ctx := context.Background() // XXX +// Verify the provided sha256 is valid. +func (r *Runner) parseChecksum() (string, error) { + if r.Checksum == "" { + return "", fmt.Errorf("missing checksum") + } + if !strings.HasPrefix(r.Checksum, "sha256/") { + return "", fmt.Errorf("invalid checksum algo: %s", r.Checksum) + } + return strings.TrimPrefix(r.Checksum, "sha256/"), nil +} - engine := wasmtime.NewEngine() - // module, err = wasmtime.NewModuleDeserialize(engine, pythonModule) - // if err != nil { - // panic(err) - // } +func (r *Runner) loadModule(ctx context.Context, engine *wasmtime.Engine) (*wasmtime.Module, error) { + expected, err := r.parseChecksum() + if err != nil { + return nil, err + } - // out, err := module.Serialize() - // if err != nil { - // panic(err) - // } + cache, err := cacheDir() + if err != nil { + return nil, err + } - // err = os.WriteFile("sqlc-codegen-python.module", out, 0644) - // if err != nil { - // panic(err) - // } + pluginDir := filepath.Join(cache, expected) + // TODO: Include os / arch in module name + modPath := filepath.Join(pluginDir, "plugin.module") + _, staterr := os.Stat(modPath) + + if staterr == nil { + data, err := os.ReadFile(modPath) + if err != nil { + return nil, err + } + return wasmtime.NewModuleDeserialize(engine, data) + } - linker := wasmtime.NewLinker(engine) - if err := linker.DefineWasi(); err != nil { + wmod, err := r.loadWASM(ctx, cache, expected) + if err != nil { return nil, err } + moduRegion := trace.StartRegion(ctx, "wasmtime.NewModule") + module, err := wasmtime.NewModule(engine, wmod) + moduRegion.End() + if err != nil { + return nil, fmt.Errorf("define wasi: %w", err) + } + + if staterr != nil { + // TODO: What permissions to use? + err := os.MkdirAll(pluginDir, 0750) + if err != nil && !os.IsExist(err) { + return nil, fmt.Errorf("mkdirall: %w", err) + } + out, err := module.Serialize() + if err != nil { + return nil, fmt.Errorf("serialize: %w", err) + } + // TODO: What permissions to use? + if err := os.WriteFile(modPath, out, 0666); err != nil { + return nil, fmt.Errorf("cache wasm: %w", err) + } + } + + return module, nil +} + +func (r *Runner) loadWASM(ctx context.Context, cache string, expected string) ([]byte, error) { + pluginDir := filepath.Join(cache, expected) + pluginPath := filepath.Join(pluginDir, "plugin.wasm") + _, staterr := os.Stat(pluginPath) + + var body io.ReadCloser + switch { + case staterr == nil: + file, err := os.Open(pluginPath) + if err != nil { + return nil, fmt.Errorf("os.Open: %s %w", pluginPath, err) + } + body = file + + case strings.HasPrefix(r.URL, "file://"): + file, err := os.Open(strings.TrimPrefix(r.URL, "file://")) + if err != nil { + return nil, fmt.Errorf("os.Open: %s %w", r.URL, err) + } + body = file + + case strings.HasPrefix(r.URL, "https://"): + resp, err := http.Get(r.URL) + if err != nil { + return nil, fmt.Errorf("http.Get: %s %w", r.URL, err) + } + body = resp.Body + + default: + return nil, fmt.Errorf("unknown scheme: %s", r.URL) + } + + defer body.Close() + + wmod, err := io.ReadAll(body) + if err != nil { + return nil, fmt.Errorf("readall: %w", err) + } + + sum := sha256.Sum256(wmod) + actual := fmt.Sprintf("%x", sum) + + if expected != actual { + return nil, fmt.Errorf("invalid checksum: expected %s, got %s", expected, actual) + } + + if staterr != nil { + // TODO: What permissions to use? + err := os.MkdirAll(pluginDir, 0750) + if err != nil && !os.IsExist(err) { + return nil, fmt.Errorf("mkdirall: %w", err) + } + // TODO: What permissions to use? + if err := os.WriteFile(pluginPath, wmod, 0666); err != nil { + return nil, fmt.Errorf("cache wasm: %w", err) + } + } + + return wmod, nil +} + +func (r *Runner) Generate(req *plugin.CodeGenRequest) (*plugin.CodeGenResponse, error) { + ctx := context.Background() // XXX + stdinBlob, err := req.MarshalVT() if err != nil { return nil, err } + engine := wasmtime.NewEngine() + module, err := r.loadModule(ctx, engine) + if err != nil { + return nil, fmt.Errorf("loadModule: %w", err) + } + + linker := wasmtime.NewLinker(engine) + if err := linker.DefineWasi(); err != nil { + return nil, err + } + dir, err := ioutil.TempDir("", "out") if err != nil { return nil, fmt.Errorf("temp dir: %w", err) @@ -76,37 +209,6 @@ func (r *Runner) Generate(req *plugin.CodeGenRequest) (*plugin.CodeGenResponse, store := wasmtime.NewStore(engine) store.SetWasi(wasiConfig) - // Set the version to the same as in the WAT. - // wasi, err := wasmtime.NewWasiInstance(store, wasiConfig, "wasi_snapshot_preview1") - // if err != nil { - // return fmt.Errorf("new wasi instances: %w", err) - // } - - // Create our module - // - // Compiling modules requires WebAssembly binary input, but the wasmtime - // package also supports converting the WebAssembly text format to the - // binary format. - // - hresp, err := http.Get(r.URL) - if err != nil { - return nil, fmt.Errorf("http get: %w", err) - } - - defer hresp.Body.Close() - - wmod, err := io.ReadAll(hresp.Body) - if err != nil { - return nil, fmt.Errorf("readall: %w", err) - } - - moduRegion := trace.StartRegion(ctx, "wasmtime.NewModule") - module, err := wasmtime.NewModule(store.Engine, wmod) - moduRegion.End() - if err != nil { - return nil, fmt.Errorf("define wasi: %w", err) - } - linkRegion := trace.StartRegion(ctx, "linker.Instantiate") instance, err := linker.Instantiate(store, module) linkRegion.End() @@ -115,7 +217,6 @@ func (r *Runner) Generate(req *plugin.CodeGenRequest) (*plugin.CodeGenResponse, } // Run the function - callRegion := trace.StartRegion(ctx, "call _start") nom := instance.GetExport(store, "_start").Func() _, err = nom.Call(store) From 685363b2b517343df7714d8f4b5dbc4410fadd13 Mon Sep 17 00:00:00 2001 From: Kyle Conroy Date: Tue, 21 Jun 2022 15:29:21 -0700 Subject: [PATCH 4/8] Include GOOS and GOARCH in module filename Make sure we don't try to share these modules across os / arch --- internal/ext/wasm/wasm.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/ext/wasm/wasm.go b/internal/ext/wasm/wasm.go index 1a6ac9ac07..cac764f19b 100644 --- a/internal/ext/wasm/wasm.go +++ b/internal/ext/wasm/wasm.go @@ -14,6 +14,7 @@ import ( "net/http" "os" "path/filepath" + "runtime" "runtime/trace" "strings" @@ -64,7 +65,7 @@ func (r *Runner) loadModule(ctx context.Context, engine *wasmtime.Engine) (*wasm pluginDir := filepath.Join(cache, expected) // TODO: Include os / arch in module name - modPath := filepath.Join(pluginDir, "plugin.module") + modPath := filepath.Join(pluginDir, fmt.Sprintf("plugin_%s_%s.module", runtime.GOOS, runtime.GOARCH)) _, staterr := os.Stat(modPath) if staterr == nil { From 0092807767ca900be63958d4d64c638b9a0f8a59 Mon Sep 17 00:00:00 2001 From: Kyle Conroy Date: Tue, 21 Jun 2022 15:46:54 -0700 Subject: [PATCH 5/8] No more panics --- internal/cmd/generate.go | 111 ++++++++++++++++++++------------------- 1 file changed, 57 insertions(+), 54 deletions(-) diff --git a/internal/cmd/generate.go b/internal/cmd/generate.go index e743e927b3..f33a14fe40 100644 --- a/internal/cmd/generate.go +++ b/internal/cmd/generate.go @@ -23,6 +23,7 @@ import ( "github.com/kyleconroy/sqlc/internal/ext/wasm" "github.com/kyleconroy/sqlc/internal/multierr" "github.com/kyleconroy/sqlc/internal/opts" + "github.com/kyleconroy/sqlc/internal/plugin" ) const errMessageNoVersion = `The configuration file must have a version number. @@ -226,60 +227,7 @@ func Generate(ctx context.Context, e Env, dir, filename string, stderr io.Writer break } - var region *trace.Region - if debug.Traced { - region = trace.StartRegion(ctx, "codegen") - } - var handler ext.Handler - var out string - switch { - case sql.Gen.Go != nil: - out = combo.Go.Out - handler = ext.HandleFunc(golang.Generate) - - case sql.Gen.Kotlin != nil: - out = combo.Kotlin.Out - handler = ext.HandleFunc(kotlin.Generate) - - case sql.Gen.Python != nil: - out = combo.Python.Out - handler = ext.HandleFunc(python.Generate) - - case sql.Gen.JSON != nil: - out = combo.JSON.Out - handler = ext.HandleFunc(json.Generate) - - case sql.Plugin != nil: - out = sql.Plugin.Out - plug, err := findPlugin(combo.Global, sql.Plugin.Plugin) - if err != nil { - // TODO: Return a real error - panic("plugin not found") - } - - switch { - case plug.Process != nil: - handler = &process.Runner{ - Cmd: plug.Process.Cmd, - } - case plug.WASM != nil: - handler = &wasm.Runner{ - URL: plug.WASM.URL, - Checksum: plug.WASM.Checksum, - } - default: - // TODO: Return a real error - panic("unsupported plugin type") - } - - default: - // TODO: Return a real error - panic("missing language backend") - } - resp, err := handler.Generate(codeGenRequest(result, combo)) - if region != nil { - region.End() - } + out, resp, err := codegen(ctx, combo, sql, result) if err != nil { fmt.Fprintf(stderr, "# package %s\n", name) fmt.Fprintf(stderr, "error generating code: %s\n", err) @@ -340,3 +288,58 @@ func parse(ctx context.Context, e Env, name, dir string, sql config.SQL, combo c } return c.Result(), false } + +func codegen(ctx context.Context, combo config.CombinedSettings, sql outPair, result *compiler.Result) (string, *plugin.CodeGenResponse, error) { + var region *trace.Region + if debug.Traced { + region = trace.StartRegion(ctx, "codegen") + } + var handler ext.Handler + var out string + switch { + case sql.Gen.Go != nil: + out = combo.Go.Out + handler = ext.HandleFunc(golang.Generate) + + case sql.Gen.Kotlin != nil: + out = combo.Kotlin.Out + handler = ext.HandleFunc(kotlin.Generate) + + case sql.Gen.Python != nil: + out = combo.Python.Out + handler = ext.HandleFunc(python.Generate) + + case sql.Gen.JSON != nil: + out = combo.JSON.Out + handler = ext.HandleFunc(json.Generate) + + case sql.Plugin != nil: + out = sql.Plugin.Out + plug, err := findPlugin(combo.Global, sql.Plugin.Plugin) + if err != nil { + return "", nil, fmt.Errorf("plugin not found: %s", err) + } + + switch { + case plug.Process != nil: + handler = &process.Runner{ + Cmd: plug.Process.Cmd, + } + case plug.WASM != nil: + handler = &wasm.Runner{ + URL: plug.WASM.URL, + Checksum: plug.WASM.Checksum, + } + default: + return "", nil, fmt.Errorf("unsupported plugin type") + } + + default: + return "", nil, fmt.Errorf("missing language backend") + } + resp, err := handler.Generate(codeGenRequest(result, combo)) + if region != nil { + region.End() + } + return out, resp, err +} From 55701803e1520b1461b7ce9f109ced31d33097be Mon Sep 17 00:00:00 2001 From: Kyle Conroy Date: Tue, 21 Jun 2022 19:18:48 -0700 Subject: [PATCH 6/8] feat: Add context.Context to handler interface --- cmd/sqlc-gen-json/main.go | 3 ++- internal/cmd/generate.go | 3 ++- internal/codegen/golang/gen.go | 3 ++- internal/codegen/json/gen.go | 3 ++- internal/codegen/kotlin/gen.go | 3 ++- internal/codegen/python/gen.go | 3 ++- internal/ext/handler.go | 12 +++++++----- internal/ext/process/gen.go | 3 +-- internal/ext/wasm/wasm.go | 6 +++--- 9 files changed, 23 insertions(+), 16 deletions(-) diff --git a/cmd/sqlc-gen-json/main.go b/cmd/sqlc-gen-json/main.go index 6e776290db..a66188280a 100644 --- a/cmd/sqlc-gen-json/main.go +++ b/cmd/sqlc-gen-json/main.go @@ -2,6 +2,7 @@ package main import ( "bufio" + "context" "fmt" "io" "os" @@ -26,7 +27,7 @@ func run() error { if err := req.UnmarshalVT(reqBlob); err != nil { return err } - resp, err := json.Generate(&req) + resp, err := json.Generate(context.Background(), &req) if err != nil { return err } diff --git a/internal/cmd/generate.go b/internal/cmd/generate.go index f33a14fe40..567e549783 100644 --- a/internal/cmd/generate.go +++ b/internal/cmd/generate.go @@ -237,6 +237,7 @@ func Generate(ctx context.Context, e Env, dir, filename string, stderr io.Writer } continue } + files := map[string]string{} for _, file := range resp.Files { files[file.Name] = string(file.Contents) @@ -337,7 +338,7 @@ func codegen(ctx context.Context, combo config.CombinedSettings, sql outPair, re default: return "", nil, fmt.Errorf("missing language backend") } - resp, err := handler.Generate(codeGenRequest(result, combo)) + resp, err := handler.Generate(ctx, codeGenRequest(result, combo)) if region != nil { region.End() } diff --git a/internal/codegen/golang/gen.go b/internal/codegen/golang/gen.go index eb21c06875..0b896e7c41 100644 --- a/internal/codegen/golang/gen.go +++ b/internal/codegen/golang/gen.go @@ -3,6 +3,7 @@ package golang import ( "bufio" "bytes" + "context" "errors" "fmt" "go/format" @@ -42,7 +43,7 @@ func (t *tmplCtx) OutputQuery(sourceName string) bool { return t.SourceName == sourceName } -func Generate(req *plugin.CodeGenRequest) (*plugin.CodeGenResponse, error) { +func Generate(ctx context.Context, req *plugin.CodeGenRequest) (*plugin.CodeGenResponse, error) { enums := buildEnums(req) structs := buildStructs(req) queries, err := buildQueries(req, structs) diff --git a/internal/codegen/json/gen.go b/internal/codegen/json/gen.go index c862909535..f481d009c6 100644 --- a/internal/codegen/json/gen.go +++ b/internal/codegen/json/gen.go @@ -2,6 +2,7 @@ package json import ( "bytes" + "context" ejson "encoding/json" "fmt" @@ -31,7 +32,7 @@ func parseOptions(req *plugin.CodeGenRequest) (plugin.JSONCode, error) { return plugin.JSONCode{}, nil } -func Generate(req *plugin.CodeGenRequest) (*plugin.CodeGenResponse, error) { +func Generate(ctx context.Context, req *plugin.CodeGenRequest) (*plugin.CodeGenResponse, error) { options, err := parseOptions(req) if err != nil { return nil, err diff --git a/internal/codegen/kotlin/gen.go b/internal/codegen/kotlin/gen.go index 275b76c2b2..1c4a9daa15 100644 --- a/internal/codegen/kotlin/gen.go +++ b/internal/codegen/kotlin/gen.go @@ -3,6 +3,7 @@ package kotlin import ( "bufio" "bytes" + "context" "errors" "fmt" "regexp" @@ -759,7 +760,7 @@ func ktFormat(s string) string { return o } -func Generate(req *plugin.CodeGenRequest) (*plugin.CodeGenResponse, error) { +func Generate(ctx context.Context, req *plugin.CodeGenRequest) (*plugin.CodeGenResponse, error) { enums := buildEnums(req) structs := buildDataClasses(req) queries, err := buildQueries(req, structs) diff --git a/internal/codegen/python/gen.go b/internal/codegen/python/gen.go index 6bdb5491b1..b9f9b1d84c 100644 --- a/internal/codegen/python/gen.go +++ b/internal/codegen/python/gen.go @@ -1,6 +1,7 @@ package python import ( + "context" "errors" "fmt" "log" @@ -1080,7 +1081,7 @@ func HashComment(s string) string { return "# " + strings.ReplaceAll(s, "\n", "\n# ") } -func Generate(req *plugin.CodeGenRequest) (*plugin.CodeGenResponse, error) { +func Generate(_ context.Context, req *plugin.CodeGenRequest) (*plugin.CodeGenResponse, error) { enums := buildEnums(req) models := buildModels(req) queries, err := buildQueries(req, models) diff --git a/internal/ext/handler.go b/internal/ext/handler.go index a65f7d0398..0f586039c0 100644 --- a/internal/ext/handler.go +++ b/internal/ext/handler.go @@ -1,21 +1,23 @@ package ext import ( + "context" + "github.com/kyleconroy/sqlc/internal/plugin" ) type Handler interface { - Generate(*plugin.CodeGenRequest) (*plugin.CodeGenResponse, error) + Generate(context.Context, *plugin.CodeGenRequest) (*plugin.CodeGenResponse, error) } type wrapper struct { - fn func(*plugin.CodeGenRequest) (*plugin.CodeGenResponse, error) + fn func(context.Context, *plugin.CodeGenRequest) (*plugin.CodeGenResponse, error) } -func (w *wrapper) Generate(req *plugin.CodeGenRequest) (*plugin.CodeGenResponse, error) { - return w.fn(req) +func (w *wrapper) Generate(ctx context.Context, req *plugin.CodeGenRequest) (*plugin.CodeGenResponse, error) { + return w.fn(ctx, req) } -func HandleFunc(fn func(*plugin.CodeGenRequest) (*plugin.CodeGenResponse, error)) Handler { +func HandleFunc(fn func(context.Context, *plugin.CodeGenRequest) (*plugin.CodeGenResponse, error)) Handler { return &wrapper{fn} } diff --git a/internal/ext/process/gen.go b/internal/ext/process/gen.go index a7fbd4a8f5..0c0295ad8f 100644 --- a/internal/ext/process/gen.go +++ b/internal/ext/process/gen.go @@ -17,7 +17,7 @@ type Runner struct { } // TODO: Update the gen func signature to take a ctx -func (r Runner) Generate(req *plugin.CodeGenRequest) (*plugin.CodeGenResponse, error) { +func (r Runner) Generate(ctx context.Context, req *plugin.CodeGenRequest) (*plugin.CodeGenResponse, error) { stdin, err := proto.Marshal(req) if err != nil { return nil, fmt.Errorf("failed to encode codegen request: %s", err) @@ -29,7 +29,6 @@ func (r Runner) Generate(req *plugin.CodeGenRequest) (*plugin.CodeGenResponse, e return nil, fmt.Errorf("process: %s not found", r.Cmd) } - ctx := context.Background() cmd := exec.CommandContext(ctx, path) cmd.Stdin = bytes.NewReader(stdin) cmd.Env = []string{ diff --git a/internal/ext/wasm/wasm.go b/internal/ext/wasm/wasm.go index cac764f19b..058d6cfbb1 100644 --- a/internal/ext/wasm/wasm.go +++ b/internal/ext/wasm/wasm.go @@ -129,6 +129,8 @@ func (r *Runner) loadWASM(ctx context.Context, cache string, expected string) ([ body = file case strings.HasPrefix(r.URL, "https://"): + // TODO: Set User-agent + // TODO: Set ETag resp, err := http.Get(r.URL) if err != nil { return nil, fmt.Errorf("http.Get: %s %w", r.URL, err) @@ -168,9 +170,7 @@ func (r *Runner) loadWASM(ctx context.Context, cache string, expected string) ([ return wmod, nil } -func (r *Runner) Generate(req *plugin.CodeGenRequest) (*plugin.CodeGenResponse, error) { - ctx := context.Background() // XXX - +func (r *Runner) Generate(ctx context.Context, req *plugin.CodeGenRequest) (*plugin.CodeGenResponse, error) { stdinBlob, err := req.MarshalVT() if err != nil { return nil, err From 59962c278f861f0ab9c82847a373cb4589e15371 Mon Sep 17 00:00:00 2001 From: Kyle Conroy Date: Sat, 25 Jun 2022 16:54:37 +0800 Subject: [PATCH 7/8] Fix WASM test --- .../gen/hello.txt | 0 .../query.sql | 0 .../schema.sql | 0 .../sqlc.json | 10 +++------- 4 files changed, 3 insertions(+), 7 deletions(-) rename internal/endtoend/testdata/{wasm_plugin_sqlc_gen_json => wasm_plugin_sqlc_gen_greeter}/gen/hello.txt (100%) rename internal/endtoend/testdata/{wasm_plugin_sqlc_gen_json => wasm_plugin_sqlc_gen_greeter}/query.sql (100%) rename internal/endtoend/testdata/{wasm_plugin_sqlc_gen_json => wasm_plugin_sqlc_gen_greeter}/schema.sql (100%) rename internal/endtoend/testdata/{wasm_plugin_sqlc_gen_json => wasm_plugin_sqlc_gen_greeter}/sqlc.json (61%) diff --git a/internal/endtoend/testdata/wasm_plugin_sqlc_gen_json/gen/hello.txt b/internal/endtoend/testdata/wasm_plugin_sqlc_gen_greeter/gen/hello.txt similarity index 100% rename from internal/endtoend/testdata/wasm_plugin_sqlc_gen_json/gen/hello.txt rename to internal/endtoend/testdata/wasm_plugin_sqlc_gen_greeter/gen/hello.txt diff --git a/internal/endtoend/testdata/wasm_plugin_sqlc_gen_json/query.sql b/internal/endtoend/testdata/wasm_plugin_sqlc_gen_greeter/query.sql similarity index 100% rename from internal/endtoend/testdata/wasm_plugin_sqlc_gen_json/query.sql rename to internal/endtoend/testdata/wasm_plugin_sqlc_gen_greeter/query.sql diff --git a/internal/endtoend/testdata/wasm_plugin_sqlc_gen_json/schema.sql b/internal/endtoend/testdata/wasm_plugin_sqlc_gen_greeter/schema.sql similarity index 100% rename from internal/endtoend/testdata/wasm_plugin_sqlc_gen_json/schema.sql rename to internal/endtoend/testdata/wasm_plugin_sqlc_gen_greeter/schema.sql diff --git a/internal/endtoend/testdata/wasm_plugin_sqlc_gen_json/sqlc.json b/internal/endtoend/testdata/wasm_plugin_sqlc_gen_greeter/sqlc.json similarity index 61% rename from internal/endtoend/testdata/wasm_plugin_sqlc_gen_json/sqlc.json rename to internal/endtoend/testdata/wasm_plugin_sqlc_gen_greeter/sqlc.json index 06e3392053..67cb5ab86c 100644 --- a/internal/endtoend/testdata/wasm_plugin_sqlc_gen_json/sqlc.json +++ b/internal/endtoend/testdata/wasm_plugin_sqlc_gen_greeter/sqlc.json @@ -8,20 +8,16 @@ "codegen": [ { "out": "gen", - "plugin": "jsonb", - "options": { - "indent": " ", - "filename": "codegen.json" - } + "plugin": "greeter" } ] } ], "plugins": [ { - "name": "jsonb", + "name": "greeter", "wasm": { - "url": "file:///Users/kyle/Downloads/sqlc-gen-json-wasm.wasm", + "url": "https://github.com/kyleconroy/sqlc-gen-greeter/releases/download/v0.1.0/sqlc-gen-greeter.wasm", "checksum": "sha256/afc486dac2068d741d7a4110146559d12a013fd0286f42a2fc7dcd802424ad07" } } From 22a18d3a2bd857a24914b98d4a16ffdadc38fb5a Mon Sep 17 00:00:00 2001 From: Kyle Conroy Date: Sat, 25 Jun 2022 17:07:29 +0800 Subject: [PATCH 8/8] fix: Hard-code exception for hello.txt --- internal/endtoend/endtoend_test.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/internal/endtoend/endtoend_test.go b/internal/endtoend/endtoend_test.go index e3989ffcad..3a3e759fba 100644 --- a/internal/endtoend/endtoend_test.go +++ b/internal/endtoend/endtoend_test.go @@ -116,7 +116,11 @@ func cmpDirectory(t *testing.T, dir string, actual map[string]string) { if file.IsDir() { return nil } - if !strings.HasSuffix(path, ".go") && !strings.HasSuffix(path, ".kt") && !strings.HasSuffix(path, ".py") && !strings.HasSuffix(path, ".json") { + if !strings.HasSuffix(path, ".go") && !strings.HasSuffix(path, ".kt") && !strings.HasSuffix(path, ".py") && !strings.HasSuffix(path, ".json") && !strings.HasSuffix(path, ".txt") { + return nil + } + // TODO: Figure out a better way to ignore certain files + if strings.HasSuffix(path, ".txt") && filepath.Base(path) != "hello.txt" { return nil } if filepath.Base(path) == "sqlc.json" {