From 3fde94747aa85f3d58f62324877c11b03579258e Mon Sep 17 00:00:00 2001 From: Kyle Conroy Date: Thu, 19 Oct 2023 10:57:33 -0700 Subject: [PATCH 1/5] feat(createdb): Create ephemeral databases --- internal/cmd/cmd.go | 2 + internal/cmd/createdb.go | 122 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 124 insertions(+) create mode 100644 internal/cmd/createdb.go diff --git a/internal/cmd/cmd.go b/internal/cmd/cmd.go index 6e31be3ab8..f305953aed 100644 --- a/internal/cmd/cmd.go +++ b/internal/cmd/cmd.go @@ -25,6 +25,7 @@ import ( ) func init() { + createDBCmd.Flags().StringP("env", "e", "DATABASE_URL", "environment variable to set (default: DATABASE_URL)") uploadCmd.Flags().BoolP("dry-run", "", false, "dump upload request (default: false)") initCmd.Flags().BoolP("v1", "", false, "generate v1 config yaml file") initCmd.Flags().BoolP("v2", "", true, "generate v2 config yaml file") @@ -41,6 +42,7 @@ func Do(args []string, stdin io.Reader, stdout io.Writer, stderr io.Writer) int rootCmd.PersistentFlags().Bool("no-database", false, "disable database connections (default: false)") rootCmd.AddCommand(checkCmd) + rootCmd.AddCommand(createDBCmd) rootCmd.AddCommand(diffCmd) rootCmd.AddCommand(genCmd) rootCmd.AddCommand(initCmd) diff --git a/internal/cmd/createdb.go b/internal/cmd/createdb.go new file mode 100644 index 0000000000..2061075a09 --- /dev/null +++ b/internal/cmd/createdb.go @@ -0,0 +1,122 @@ +package cmd + +import ( + "context" + "errors" + "fmt" + "os" + "os/exec" + "runtime/trace" + "strings" + + "github.com/spf13/cobra" + "github.com/sqlc-dev/sqlc/internal/config" + "github.com/sqlc-dev/sqlc/internal/opts" + "github.com/sqlc-dev/sqlc/internal/quickdb" + pb "github.com/sqlc-dev/sqlc/internal/quickdb/v1" + "github.com/sqlc-dev/sqlc/internal/sql/sqlpath" +) + +var createDBCmd = &cobra.Command{ + Use: "createdb", + Short: "Create an ephemeral database", + Args: cobra.MinimumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + defer trace.StartRegion(cmd.Context(), "createdb").End() + stderr := cmd.ErrOrStderr() + dir, name := getConfigPath(stderr, cmd.Flag("file")) + env, err := cmd.Flags().GetString("env") + if err != nil { + return err + } + code, err := CreateDB(cmd.Context(), dir, name, args, env, &Options{ + Env: ParseEnv(cmd), + Stderr: stderr, + }) + if err != nil { + fmt.Fprintln(stderr, err.Error()) + os.Exit(code) + } + return nil + }, +} + +func CreateDB(ctx context.Context, dir, filename string, args []string, env string, o *Options) (int, error) { + dbg := opts.DebugFromEnv() + if !dbg.ProcessPlugins { + return 1, fmt.Errorf("process-plugins disabled") + } + _, conf, err := o.ReadConfig(dir, filename) + if err != nil { + return 1, err + } + // Find the first SQL with a managed database + var pkg *config.SQL + for _, sql := range conf.SQL { + sql := sql + if sql.Database != nil && sql.Database.Managed { + pkg = &sql + break + } + } + if pkg == nil { + return 1, fmt.Errorf("no managed database found") + } + if pkg.Engine != config.EnginePostgreSQL { + return 1, fmt.Errorf("managed: only PostgreSQL currently") + } + + var migrations []string + files, err := sqlpath.Glob(pkg.Schema) + if err != nil { + return 1, err + } + for _, schema := range files { + contents, err := os.ReadFile(schema) + if err != nil { + return 1, fmt.Errorf("read file: %w", err) + } + migrations = append(migrations, string(contents)) + } + client, err := quickdb.NewClientFromConfig(conf.Cloud) + if err != nil { + return 1, fmt.Errorf("client error: %w", err) + } + + resp, err := client.CreateEphemeralDatabase(ctx, &pb.CreateEphemeralDatabaseRequest{ + Engine: "postgresql", + Region: quickdb.GetClosestRegion(), + Migrations: migrations, + }) + if err != nil { + return 1, fmt.Errorf("managed: create database: %w", err) + } + + defer func() { + client.DropEphemeralDatabase(ctx, &pb.DropEphemeralDatabaseRequest{ + DatabaseId: resp.DatabaseId, + }) + }() + + cmd := exec.Command(args[0], args[1:]...) + cmd.Env = append(os.Environ(), fmt.Sprintf("%s=%s", env, resp.Uri)) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Env = []string{fmt.Sprintf("%s=%s", env, resp.Uri)} + for _, val := range os.Environ() { + if strings.HasPrefix(val, "SQLC_AUTH_TOKEN") { + continue + } + cmd.Env = append(cmd.Env, val) + } + + if err := cmd.Run(); err != nil { + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + return exitErr.ExitCode(), err + } + return 1, err + } + + return 0, nil +} From 2c1120224e6ee41dbdef341ce53308a55061c38d Mon Sep 17 00:00:00 2001 From: Kyle Conroy Date: Fri, 20 Oct 2023 12:28:31 -0700 Subject: [PATCH 2/5] Just print to stdout --- internal/cmd/cmd.go | 2 +- internal/cmd/createdb.go | 90 +++++++++++++--------------------------- 2 files changed, 30 insertions(+), 62 deletions(-) diff --git a/internal/cmd/cmd.go b/internal/cmd/cmd.go index f305953aed..c4c08e161c 100644 --- a/internal/cmd/cmd.go +++ b/internal/cmd/cmd.go @@ -25,7 +25,7 @@ import ( ) func init() { - createDBCmd.Flags().StringP("env", "e", "DATABASE_URL", "environment variable to set (default: DATABASE_URL)") + createDBCmd.Flags().StringP("queryset", "q", "", "queryset to use") uploadCmd.Flags().BoolP("dry-run", "", false, "dump upload request (default: false)") initCmd.Flags().BoolP("v1", "", false, "generate v1 config yaml file") initCmd.Flags().BoolP("v2", "", true, "generate v2 config yaml file") diff --git a/internal/cmd/createdb.go b/internal/cmd/createdb.go index 2061075a09..2239733466 100644 --- a/internal/cmd/createdb.go +++ b/internal/cmd/createdb.go @@ -2,16 +2,13 @@ package cmd import ( "context" - "errors" "fmt" "os" - "os/exec" "runtime/trace" - "strings" "github.com/spf13/cobra" "github.com/sqlc-dev/sqlc/internal/config" - "github.com/sqlc-dev/sqlc/internal/opts" + "github.com/sqlc-dev/sqlc/internal/migrations" "github.com/sqlc-dev/sqlc/internal/quickdb" pb "github.com/sqlc-dev/sqlc/internal/quickdb/v1" "github.com/sqlc-dev/sqlc/internal/sql/sqlpath" @@ -20,103 +17,74 @@ import ( var createDBCmd = &cobra.Command{ Use: "createdb", Short: "Create an ephemeral database", - Args: cobra.MinimumNArgs(1), + Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { defer trace.StartRegion(cmd.Context(), "createdb").End() stderr := cmd.ErrOrStderr() dir, name := getConfigPath(stderr, cmd.Flag("file")) - env, err := cmd.Flags().GetString("env") - if err != nil { - return err - } - code, err := CreateDB(cmd.Context(), dir, name, args, env, &Options{ + err := CreateDB(cmd.Context(), dir, name, &Options{ Env: ParseEnv(cmd), Stderr: stderr, }) if err != nil { fmt.Fprintln(stderr, err.Error()) - os.Exit(code) + os.Exit(1) } return nil }, } -func CreateDB(ctx context.Context, dir, filename string, args []string, env string, o *Options) (int, error) { - dbg := opts.DebugFromEnv() - if !dbg.ProcessPlugins { - return 1, fmt.Errorf("process-plugins disabled") - } +func CreateDB(ctx context.Context, dir, filename string, o *Options) error { _, conf, err := o.ReadConfig(dir, filename) if err != nil { - return 1, err + return err } - // Find the first SQL with a managed database - var pkg *config.SQL + // Find the first queryset with a managed database + var queryset *config.SQL + var count int for _, sql := range conf.SQL { sql := sql if sql.Database != nil && sql.Database.Managed { - pkg = &sql - break + queryset = &sql + count += 1 } } - if pkg == nil { - return 1, fmt.Errorf("no managed database found") + if queryset == nil { + return fmt.Errorf("no querysets configured to use a managed database") + } + if count > 1 { + return fmt.Errorf("multiple querysets configured to use managed databases") } - if pkg.Engine != config.EnginePostgreSQL { - return 1, fmt.Errorf("managed: only PostgreSQL currently") + if queryset.Engine != config.EnginePostgreSQL { + return fmt.Errorf("managed databases currently only support PostgreSQL") } - var migrations []string - files, err := sqlpath.Glob(pkg.Schema) + var ddl []string + files, err := sqlpath.Glob(queryset.Schema) if err != nil { - return 1, err + return err } for _, schema := range files { contents, err := os.ReadFile(schema) if err != nil { - return 1, fmt.Errorf("read file: %w", err) + return fmt.Errorf("read file: %w", err) } - migrations = append(migrations, string(contents)) + ddl = append(ddl, migrations.RemoveRollbackStatements(string(contents))) } + client, err := quickdb.NewClientFromConfig(conf.Cloud) if err != nil { - return 1, fmt.Errorf("client error: %w", err) + return fmt.Errorf("client error: %w", err) } resp, err := client.CreateEphemeralDatabase(ctx, &pb.CreateEphemeralDatabaseRequest{ Engine: "postgresql", Region: quickdb.GetClosestRegion(), - Migrations: migrations, + Migrations: ddl, }) if err != nil { - return 1, fmt.Errorf("managed: create database: %w", err) - } - - defer func() { - client.DropEphemeralDatabase(ctx, &pb.DropEphemeralDatabaseRequest{ - DatabaseId: resp.DatabaseId, - }) - }() - - cmd := exec.Command(args[0], args[1:]...) - cmd.Env = append(os.Environ(), fmt.Sprintf("%s=%s", env, resp.Uri)) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - cmd.Env = []string{fmt.Sprintf("%s=%s", env, resp.Uri)} - for _, val := range os.Environ() { - if strings.HasPrefix(val, "SQLC_AUTH_TOKEN") { - continue - } - cmd.Env = append(cmd.Env, val) + return fmt.Errorf("managed: create database: %w", err) } - - if err := cmd.Run(); err != nil { - var exitErr *exec.ExitError - if errors.As(err, &exitErr) { - return exitErr.ExitCode(), err - } - return 1, err - } - - return 0, nil + fmt.Println(resp.Uri) + return nil } From 162756b5077554715108c6f17c8b6b453ff3c2ea Mon Sep 17 00:00:00 2001 From: Kyle Conroy Date: Fri, 20 Oct 2023 12:48:31 -0700 Subject: [PATCH 3/5] Add names to querysets --- internal/cmd/cmd.go | 2 +- internal/cmd/createdb.go | 14 ++++++++++++-- internal/config/config.go | 1 + internal/config/v_one.go | 1 + 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/internal/cmd/cmd.go b/internal/cmd/cmd.go index c4c08e161c..8d7a64c8dd 100644 --- a/internal/cmd/cmd.go +++ b/internal/cmd/cmd.go @@ -25,7 +25,7 @@ import ( ) func init() { - createDBCmd.Flags().StringP("queryset", "q", "", "queryset to use") + createDBCmd.Flags().StringP("queryset", "", "", "name of the queryset to use") uploadCmd.Flags().BoolP("dry-run", "", false, "dump upload request (default: false)") initCmd.Flags().BoolP("v1", "", false, "generate v1 config yaml file") initCmd.Flags().BoolP("v2", "", true, "generate v2 config yaml file") diff --git a/internal/cmd/createdb.go b/internal/cmd/createdb.go index 2239733466..3660006113 100644 --- a/internal/cmd/createdb.go +++ b/internal/cmd/createdb.go @@ -22,7 +22,11 @@ var createDBCmd = &cobra.Command{ defer trace.StartRegion(cmd.Context(), "createdb").End() stderr := cmd.ErrOrStderr() dir, name := getConfigPath(stderr, cmd.Flag("file")) - err := CreateDB(cmd.Context(), dir, name, &Options{ + qs, err := cmd.Flags().GetString("queryset") + if err != nil { + return err + } + err = CreateDB(cmd.Context(), dir, name, qs, &Options{ Env: ParseEnv(cmd), Stderr: stderr, }) @@ -34,7 +38,7 @@ var createDBCmd = &cobra.Command{ }, } -func CreateDB(ctx context.Context, dir, filename string, o *Options) error { +func CreateDB(ctx context.Context, dir, filename, name string, o *Options) error { _, conf, err := o.ReadConfig(dir, filename) if err != nil { return err @@ -44,11 +48,17 @@ func CreateDB(ctx context.Context, dir, filename string, o *Options) error { var count int for _, sql := range conf.SQL { sql := sql + if name != "" && sql.Name != name { + continue + } if sql.Database != nil && sql.Database.Managed { queryset = &sql count += 1 } } + if queryset == nil && name != "" { + return fmt.Errorf("no queryset found with name %q", name) + } if queryset == nil { return fmt.Errorf("no querysets configured to use a managed database") } diff --git a/internal/config/config.go b/internal/config/config.go index 26aefc7e70..8bd1f004f4 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -103,6 +103,7 @@ type GenGo struct { } type SQL struct { + Name string `json:"name" yaml:"name"` Engine Engine `json:"engine,omitempty" yaml:"engine"` Schema Paths `json:"schema" yaml:"schema"` Queries Paths `json:"queries" yaml:"queries"` diff --git a/internal/config/v_one.go b/internal/config/v_one.go index c1a0a9f79e..c6568ebb53 100644 --- a/internal/config/v_one.go +++ b/internal/config/v_one.go @@ -143,6 +143,7 @@ func (c *V1GenerateSettings) Translate() Config { pkg.StrictOrderBy = &defaultValue } conf.SQL = append(conf.SQL, SQL{ + Name: pkg.Name, Engine: pkg.Engine, Database: pkg.Database, Schema: pkg.Schema, From aa6c3664b81daee158af9db15c5ef7b2141f6432 Mon Sep 17 00:00:00 2001 From: Kyle Conroy Date: Fri, 20 Oct 2023 12:56:39 -0700 Subject: [PATCH 4/5] Add a warning about the databse going away --- internal/cmd/createdb.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/cmd/createdb.go b/internal/cmd/createdb.go index 3660006113..51db0d4f79 100644 --- a/internal/cmd/createdb.go +++ b/internal/cmd/createdb.go @@ -95,6 +95,7 @@ func CreateDB(ctx context.Context, dir, filename, name string, o *Options) error if err != nil { return fmt.Errorf("managed: create database: %w", err) } + fmt.Fprintln(os.Stderr, "WARNING: This database will be removed in two minutes") fmt.Println(resp.Uri) return nil } From ab4db66e5295b3b98841eba1fcad3b59249f1234 Mon Sep 17 00:00:00 2001 From: Kyle Conroy Date: Fri, 20 Oct 2023 13:32:13 -0700 Subject: [PATCH 5/5] Last changes --- internal/cmd/createdb.go | 14 +++++++------- internal/config/v_two.json | 5 ++++- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/internal/cmd/createdb.go b/internal/cmd/createdb.go index 51db0d4f79..8eb3d26222 100644 --- a/internal/cmd/createdb.go +++ b/internal/cmd/createdb.go @@ -21,12 +21,12 @@ var createDBCmd = &cobra.Command{ RunE: func(cmd *cobra.Command, args []string) error { defer trace.StartRegion(cmd.Context(), "createdb").End() stderr := cmd.ErrOrStderr() - dir, name := getConfigPath(stderr, cmd.Flag("file")) - qs, err := cmd.Flags().GetString("queryset") + dir, filename := getConfigPath(stderr, cmd.Flag("file")) + querySetName, err := cmd.Flags().GetString("queryset") if err != nil { return err } - err = CreateDB(cmd.Context(), dir, name, qs, &Options{ + err = CreateDB(cmd.Context(), dir, filename, querySetName, &Options{ Env: ParseEnv(cmd), Stderr: stderr, }) @@ -38,7 +38,7 @@ var createDBCmd = &cobra.Command{ }, } -func CreateDB(ctx context.Context, dir, filename, name string, o *Options) error { +func CreateDB(ctx context.Context, dir, filename, querySetName string, o *Options) error { _, conf, err := o.ReadConfig(dir, filename) if err != nil { return err @@ -48,7 +48,7 @@ func CreateDB(ctx context.Context, dir, filename, name string, o *Options) error var count int for _, sql := range conf.SQL { sql := sql - if name != "" && sql.Name != name { + if querySetName != "" && sql.Name != querySetName { continue } if sql.Database != nil && sql.Database.Managed { @@ -56,8 +56,8 @@ func CreateDB(ctx context.Context, dir, filename, name string, o *Options) error count += 1 } } - if queryset == nil && name != "" { - return fmt.Errorf("no queryset found with name %q", name) + if queryset == nil && querySetName != "" { + return fmt.Errorf("no queryset found with name %q", querySetName) } if queryset == nil { return fmt.Errorf("no querysets configured to use a managed database") diff --git a/internal/config/v_two.json b/internal/config/v_two.json index c2f1b03142..dd39fddc10 100644 --- a/internal/config/v_two.json +++ b/internal/config/v_two.json @@ -31,6 +31,9 @@ "engine" ], "properties": { + "name": { + "type": "string" + }, "engine": { "enum": [ "postgresql", @@ -439,4 +442,4 @@ } } } -} \ No newline at end of file +}