From 384bffea1a516da6bc15c7803637fe028723f04f Mon Sep 17 00:00:00 2001 From: vl4deee11 Date: Tue, 8 Jun 2021 22:02:11 +0100 Subject: [PATCH 01/11] fix with pg13 Signed-off-by: vl4deee11 Signed-off-by: vl4deee11 Signed-off-by: vl4deee11 Signed-off-by: vl4deee11 --- cmd/postgres_exporter/pg_breaking_changes.go | 31 ++++++++++++++++++++ cmd/postgres_exporter/postgres_exporter.go | 31 +++++++++++++++----- queries.yaml | 8 +++++ 3 files changed, 63 insertions(+), 7 deletions(-) create mode 100644 cmd/postgres_exporter/pg_breaking_changes.go diff --git a/cmd/postgres_exporter/pg_breaking_changes.go b/cmd/postgres_exporter/pg_breaking_changes.go new file mode 100644 index 000000000..0e0de10a6 --- /dev/null +++ b/cmd/postgres_exporter/pg_breaking_changes.go @@ -0,0 +1,31 @@ +package main + +import ( + "github.com/blang/semver" + "strings" +) + +type BreakingChanges struct { + Version string `yaml:"version"` + ver semver.Version + Columns map[string]string `yaml:"columns"` +} + +func (bc *BreakingChanges) ParseVerTolerant() error { + bcVer, err := semver.ParseTolerant(bc.Version) + if err != nil { + return err + } + + bc.ver = bcVer + return nil +} + +func (bc *BreakingChanges) FixColumns(query string) string { + oldnew := make([]string, 0, 2*len(bc.Columns)) + for old := range bc.Columns { + oldnew = append(oldnew, old, bc.Columns[old]) + } + r := strings.NewReplacer(oldnew...) + return r.Replace(query) +} diff --git a/cmd/postgres_exporter/postgres_exporter.go b/cmd/postgres_exporter/postgres_exporter.go index 36c524c8e..a7d20705b 100644 --- a/cmd/postgres_exporter/postgres_exporter.go +++ b/cmd/postgres_exporter/postgres_exporter.go @@ -119,11 +119,12 @@ type Mapping map[string]MappingOptions // nolint: golint type UserQuery struct { - Query string `yaml:"query"` - Metrics []Mapping `yaml:"metrics"` - Master bool `yaml:"master"` // Querying only for master database - CacheSeconds uint64 `yaml:"cache_seconds"` // Number of seconds to cache the namespace result metrics for. - RunOnServer string `yaml:"runonserver"` // Querying to run on which server version + Query string `yaml:"query"` + Metrics []Mapping `yaml:"metrics"` + BreakingChanges []BreakingChanges `yaml:"breakingChanges"` + Master bool `yaml:"master"` // Querying only for master database + CacheSeconds uint64 `yaml:"cache_seconds"` // Number of seconds to cache the namespace result metrics for. + RunOnServer string `yaml:"runonserver"` // Querying to run on which server version } // nolint: golint @@ -517,7 +518,7 @@ func makeQueryOverrideMap(pgVersion semver.Version, queryOverrides map[string][] return resultMap } -func parseUserQueries(content []byte) (map[string]intermediateMetricMap, map[string]string, error) { +func parseUserQueries(content []byte, pgVersion semver.Version) (map[string]intermediateMetricMap, map[string]string, error) { var userQueries UserQueries err := yaml.Unmarshal(content, &userQueries) @@ -532,6 +533,16 @@ func parseUserQueries(content []byte) (map[string]intermediateMetricMap, map[str for metric, specs := range userQueries { level.Debug(logger).Log("msg", "New user metric namespace from YAML metric", "metric", metric, "cache_seconds", specs.CacheSeconds) newQueryOverrides[metric] = specs.Query + for i := range specs.BreakingChanges { + if err := specs.BreakingChanges[i].ParseVerTolerant(); err != nil { + return nil, nil, err + } + + if pgVersion.GE(specs.BreakingChanges[i].ver) { + newQueryOverrides[metric] = specs.BreakingChanges[i].FixColumns(specs.Query) + } + } + metricMap, ok := metricMaps[metric] if !ok { // Namespace for metric not found - add it. @@ -558,6 +569,8 @@ func parseUserQueries(content []byte) (map[string]intermediateMetricMap, map[str metricMap.columnMappings[name] = columnMapping } } + + } return metricMaps, newQueryOverrides, nil } @@ -571,7 +584,7 @@ func parseUserQueries(content []byte) (map[string]intermediateMetricMap, map[str // TODO: test code for all cu. // TODO: the YAML this supports is "non-standard" - we should move away from it. func addQueries(content []byte, pgVersion semver.Version, server *Server) error { - metricMaps, newQueryOverrides, err := parseUserQueries(content) + metricMaps, newQueryOverrides, err := parseUserQueries(content, pgVersion) if err != nil { return err } @@ -1317,6 +1330,10 @@ func queryNamespaceMapping(server *Server, namespace string, mapping MetricMapNa // an admin tool so you're not injecting SQL right? rows, err = server.db.Query(fmt.Sprintf("SELECT * FROM %s;", namespace)) // nolint: gas } else { + // make for pg13 breaking changes + if server.lastMapVersion.GE(semver.Version{Major: 13}) && namespace == "pg_stat_statements" { + fmt.Println(query) + } rows, err = server.db.Query(query) } if err != nil { diff --git a/queries.yaml b/queries.yaml index 35b754319..8acee91e3 100644 --- a/queries.yaml +++ b/queries.yaml @@ -161,6 +161,14 @@ pg_database: pg_stat_statements: query: "SELECT t2.rolname, t3.datname, queryid, calls, total_time / 1000 as total_time_seconds, min_time / 1000 as min_time_seconds, max_time / 1000 as max_time_seconds, mean_time / 1000 as mean_time_seconds, stddev_time / 1000 as stddev_time_seconds, rows, shared_blks_hit, shared_blks_read, shared_blks_dirtied, shared_blks_written, local_blks_hit, local_blks_read, local_blks_dirtied, local_blks_written, temp_blks_read, temp_blks_written, blk_read_time / 1000 as blk_read_time_seconds, blk_write_time / 1000 as blk_write_time_seconds FROM pg_stat_statements t1 JOIN pg_roles t2 ON (t1.userid=t2.oid) JOIN pg_database t3 ON (t1.dbid=t3.oid) WHERE t2.rolname != 'rdsadmin'" master: true + breakingChanges: + - version: '13.0.0' + columns: + total_time: total_exec_time + min_time: min_exec_time + max_time: max_exec_time + mean_time: mean_exec_time + stddev_time: stddev_exec_time metrics: - rolname: usage: "LABEL" From 3807ac86ad8af7cb3c91a4e87b0ac0a1314bf19e Mon Sep 17 00:00:00 2001 From: vl4deee11 <44677024+vl4deee11@users.noreply.github.com> Date: Wed, 9 Jun 2021 09:43:01 +0300 Subject: [PATCH 02/11] Update queries.yaml Signed-off-by: vl4deee11 Signed-off-by: vl4deee11 Signed-off-by: vl4deee11 Signed-off-by: vl4deee11 --- queries.yaml | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/queries.yaml b/queries.yaml index 8acee91e3..f4f49e609 100644 --- a/queries.yaml +++ b/queries.yaml @@ -159,16 +159,24 @@ pg_database: description: "Disk space used by the database" pg_stat_statements: - query: "SELECT t2.rolname, t3.datname, queryid, calls, total_time / 1000 as total_time_seconds, min_time / 1000 as min_time_seconds, max_time / 1000 as max_time_seconds, mean_time / 1000 as mean_time_seconds, stddev_time / 1000 as stddev_time_seconds, rows, shared_blks_hit, shared_blks_read, shared_blks_dirtied, shared_blks_written, local_blks_hit, local_blks_read, local_blks_dirtied, local_blks_written, temp_blks_read, temp_blks_written, blk_read_time / 1000 as blk_read_time_seconds, blk_write_time / 1000 as blk_write_time_seconds FROM pg_stat_statements t1 JOIN pg_roles t2 ON (t1.userid=t2.oid) JOIN pg_database t3 ON (t1.dbid=t3.oid) WHERE t2.rolname != 'rdsadmin'" + query: "SELECT t2.rolname, t3.datname, queryid, calls, total_time_T / 1000 as total_time_seconds, min_time_T / 1000 as min_time_seconds, max_time_T / 1000 as max_time_seconds, mean_time_T / 1000 as mean_time_seconds, stddev_time_T / 1000 as stddev_time_seconds, rows, shared_blks_hit, shared_blks_read, shared_blks_dirtied, shared_blks_written, local_blks_hit, local_blks_read, local_blks_dirtied, local_blks_written, temp_blks_read, temp_blks_written, blk_read_time / 1000 as blk_read_time_seconds, blk_write_time / 1000 as blk_write_time_seconds FROM pg_stat_statements t1 JOIN pg_roles t2 ON (t1.userid=t2.oid) JOIN pg_database t3 ON (t1.dbid=t3.oid) WHERE t2.rolname != 'rdsadmin'" master: true breakingChanges: + # should be in asc order by version + - version: '9.4.0' + columns: + total_time_T: total_time + min_time_T: min_time + max_time_T: max_time + mean_time_T: mean_time + stddev_time_T: stddev_time - version: '13.0.0' columns: - total_time: total_exec_time - min_time: min_exec_time - max_time: max_exec_time - mean_time: mean_exec_time - stddev_time: stddev_exec_time + total_time_T: total_exec_time + min_time_T: min_exec_time + max_time_T: max_exec_time + mean_time_T: mean_exec_time + stddev_time_T: stddev_exec_time metrics: - rolname: usage: "LABEL" From d40e499d152bd295c9750b191c85e8990cfb7e80 Mon Sep 17 00:00:00 2001 From: vl4deee11 <44677024+vl4deee11@users.noreply.github.com> Date: Wed, 9 Jun 2021 09:44:37 +0300 Subject: [PATCH 03/11] Update postgres_exporter.go Signed-off-by: vl4deee11 Signed-off-by: vl4deee11 Signed-off-by: vl4deee11 Signed-off-by: vl4deee11 --- cmd/postgres_exporter/postgres_exporter.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/postgres_exporter/postgres_exporter.go b/cmd/postgres_exporter/postgres_exporter.go index a7d20705b..198a170cc 100644 --- a/cmd/postgres_exporter/postgres_exporter.go +++ b/cmd/postgres_exporter/postgres_exporter.go @@ -533,13 +533,14 @@ func parseUserQueries(content []byte, pgVersion semver.Version) (map[string]inte for metric, specs := range userQueries { level.Debug(logger).Log("msg", "New user metric namespace from YAML metric", "metric", metric, "cache_seconds", specs.CacheSeconds) newQueryOverrides[metric] = specs.Query - for i := range specs.BreakingChanges { + for i := len(specs.BreakingChanges) - 1; i >= 0; i++ { if err := specs.BreakingChanges[i].ParseVerTolerant(); err != nil { return nil, nil, err } if pgVersion.GE(specs.BreakingChanges[i].ver) { newQueryOverrides[metric] = specs.BreakingChanges[i].FixColumns(specs.Query) + break } } From a044b77b2d4303fbe4de51fbe5773c83c565e644 Mon Sep 17 00:00:00 2001 From: vl4deee11 <44677024+vl4deee11@users.noreply.github.com> Date: Wed, 9 Jun 2021 09:56:16 +0300 Subject: [PATCH 04/11] Update postgres_exporter.go Signed-off-by: vl4deee11 Signed-off-by: vl4deee11 Signed-off-by: vl4deee11 Signed-off-by: vl4deee11 --- cmd/postgres_exporter/postgres_exporter.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cmd/postgres_exporter/postgres_exporter.go b/cmd/postgres_exporter/postgres_exporter.go index 198a170cc..9b2fc4dc9 100644 --- a/cmd/postgres_exporter/postgres_exporter.go +++ b/cmd/postgres_exporter/postgres_exporter.go @@ -533,7 +533,7 @@ func parseUserQueries(content []byte, pgVersion semver.Version) (map[string]inte for metric, specs := range userQueries { level.Debug(logger).Log("msg", "New user metric namespace from YAML metric", "metric", metric, "cache_seconds", specs.CacheSeconds) newQueryOverrides[metric] = specs.Query - for i := len(specs.BreakingChanges) - 1; i >= 0; i++ { + for i := len(specs.BreakingChanges) - 1; i >= 0; i-- { if err := specs.BreakingChanges[i].ParseVerTolerant(); err != nil { return nil, nil, err } @@ -570,8 +570,6 @@ func parseUserQueries(content []byte, pgVersion semver.Version) (map[string]inte metricMap.columnMappings[name] = columnMapping } } - - } return metricMaps, newQueryOverrides, nil } From d58ca96493cd142b29d24a0bd331d0dcf2c473f2 Mon Sep 17 00:00:00 2001 From: vl4deee11 <44677024+vl4deee11@users.noreply.github.com> Date: Wed, 9 Jun 2021 09:58:10 +0300 Subject: [PATCH 05/11] Update postgres_exporter.go Signed-off-by: vl4deee11 Signed-off-by: vl4deee11 Signed-off-by: vl4deee11 Signed-off-by: vl4deee11 --- cmd/postgres_exporter/postgres_exporter.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/postgres_exporter/postgres_exporter.go b/cmd/postgres_exporter/postgres_exporter.go index 9b2fc4dc9..ecaed8541 100644 --- a/cmd/postgres_exporter/postgres_exporter.go +++ b/cmd/postgres_exporter/postgres_exporter.go @@ -119,12 +119,12 @@ type Mapping map[string]MappingOptions // nolint: golint type UserQuery struct { - Query string `yaml:"query"` - Metrics []Mapping `yaml:"metrics"` + Query string `yaml:"query"` + Metrics []Mapping `yaml:"metrics"` BreakingChanges []BreakingChanges `yaml:"breakingChanges"` - Master bool `yaml:"master"` // Querying only for master database - CacheSeconds uint64 `yaml:"cache_seconds"` // Number of seconds to cache the namespace result metrics for. - RunOnServer string `yaml:"runonserver"` // Querying to run on which server version + Master bool `yaml:"master"` // Querying only for master database + CacheSeconds uint64 `yaml:"cache_seconds"` // Number of seconds to cache the namespace result metrics for. + RunOnServer string `yaml:"runonserver"` // Querying to run on which server version } // nolint: golint From f16170311cac384dc0b91506bfa75a022fcee792 Mon Sep 17 00:00:00 2001 From: vl4deee11 <44677024+vl4deee11@users.noreply.github.com> Date: Wed, 9 Jun 2021 10:01:42 +0300 Subject: [PATCH 06/11] Update pg_breaking_changes.go Signed-off-by: vl4deee11 Signed-off-by: vl4deee11 Signed-off-by: vl4deee11 Signed-off-by: vl4deee11 --- cmd/postgres_exporter/pg_breaking_changes.go | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/cmd/postgres_exporter/pg_breaking_changes.go b/cmd/postgres_exporter/pg_breaking_changes.go index 0e0de10a6..e5e6c73a0 100644 --- a/cmd/postgres_exporter/pg_breaking_changes.go +++ b/cmd/postgres_exporter/pg_breaking_changes.go @@ -1,8 +1,22 @@ +// Copyright 2021 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package main import ( - "github.com/blang/semver" "strings" + + "github.com/blang/semver" ) type BreakingChanges struct { @@ -22,6 +36,8 @@ func (bc *BreakingChanges) ParseVerTolerant() error { } func (bc *BreakingChanges) FixColumns(query string) string { + // nolint: golint + // 2 because old - new oldnew := make([]string, 0, 2*len(bc.Columns)) for old := range bc.Columns { oldnew = append(oldnew, old, bc.Columns[old]) From 421f4a4739beea256fbf63bd400e01034b525991 Mon Sep 17 00:00:00 2001 From: vl4deee11 <44677024+vl4deee11@users.noreply.github.com> Date: Wed, 9 Jun 2021 10:07:10 +0300 Subject: [PATCH 07/11] Update postgres_exporter_test.go Signed-off-by: vl4deee11 Signed-off-by: vl4deee11 Signed-off-by: vl4deee11 Signed-off-by: vl4deee11 --- cmd/postgres_exporter/postgres_exporter_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/postgres_exporter/postgres_exporter_test.go b/cmd/postgres_exporter/postgres_exporter_test.go index 5747f0b07..5645a8ee8 100644 --- a/cmd/postgres_exporter/postgres_exporter_test.go +++ b/cmd/postgres_exporter/postgres_exporter_test.go @@ -410,7 +410,7 @@ func (s *FunctionalSuite) TestBooleanConversionToValueAndString(c *C) { func (s *FunctionalSuite) TestParseUserQueries(c *C) { userQueriesData, err := ioutil.ReadFile("./tests/user_queries_ok.yaml") if err == nil { - metricMaps, newQueryOverrides, err := parseUserQueries(userQueriesData) + metricMaps, newQueryOverrides, err := parseUserQueries(userQueriesData, semver.Version{Major: 13}) c.Assert(err, Equals, nil) c.Assert(metricMaps, NotNil) c.Assert(newQueryOverrides, NotNil) From 37f16e2b53e63d34dbabfe2174e40b7bac623fbd Mon Sep 17 00:00:00 2001 From: vl4deee11 <44677024+vl4deee11@users.noreply.github.com> Date: Wed, 9 Jun 2021 10:37:49 +0300 Subject: [PATCH 08/11] Update postgres_exporter.go Signed-off-by: vl4deee11 Signed-off-by: vl4deee11 Signed-off-by: vl4deee11 Signed-off-by: vl4deee11 --- cmd/postgres_exporter/postgres_exporter.go | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/cmd/postgres_exporter/postgres_exporter.go b/cmd/postgres_exporter/postgres_exporter.go index ecaed8541..6618071c4 100644 --- a/cmd/postgres_exporter/postgres_exporter.go +++ b/cmd/postgres_exporter/postgres_exporter.go @@ -533,17 +533,30 @@ func parseUserQueries(content []byte, pgVersion semver.Version) (map[string]inte for metric, specs := range userQueries { level.Debug(logger).Log("msg", "New user metric namespace from YAML metric", "metric", metric, "cache_seconds", specs.CacheSeconds) newQueryOverrides[metric] = specs.Query - for i := len(specs.BreakingChanges) - 1; i >= 0; i-- { - if err := specs.BreakingChanges[i].ParseVerTolerant(); err != nil { + columnT := make(map[string]string, 0) + for i := range specs.BreakingChanges { + if err := specs.BreakingChanges[i].parseVerTolerant(); err != nil { return nil, nil, err } if pgVersion.GE(specs.BreakingChanges[i].ver) { - newQueryOverrides[metric] = specs.BreakingChanges[i].FixColumns(specs.Query) - break + for t := range specs.BreakingChanges[i].Columns { + columnT[t] = specs.BreakingChanges[i].Columns[t] + } } } + // nolint: golint + // 2 because old - new + oldnew := make([]string, 0, 2*len(columnT)) + for t := range columnT { + oldnew = append(oldnew, t, columnT[t]) + } + + r := strings.NewReplacer(oldnew...) + + newQueryOverrides[metric] = r.Replace(newQueryOverrides[metric]) + metricMap, ok := metricMaps[metric] if !ok { // Namespace for metric not found - add it. From 1e578c27c6836703b029ee36cdfa632a034a74bb Mon Sep 17 00:00:00 2001 From: vl4deee11 <44677024+vl4deee11@users.noreply.github.com> Date: Wed, 9 Jun 2021 10:38:30 +0300 Subject: [PATCH 09/11] Update pg_breaking_changes.go Signed-off-by: vl4deee11 Signed-off-by: vl4deee11 Signed-off-by: vl4deee11 Signed-off-by: vl4deee11 --- cmd/postgres_exporter/pg_breaking_changes.go | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/cmd/postgres_exporter/pg_breaking_changes.go b/cmd/postgres_exporter/pg_breaking_changes.go index e5e6c73a0..ddd4a76aa 100644 --- a/cmd/postgres_exporter/pg_breaking_changes.go +++ b/cmd/postgres_exporter/pg_breaking_changes.go @@ -14,8 +14,6 @@ package main import ( - "strings" - "github.com/blang/semver" ) @@ -25,7 +23,7 @@ type BreakingChanges struct { Columns map[string]string `yaml:"columns"` } -func (bc *BreakingChanges) ParseVerTolerant() error { +func (bc *BreakingChanges) parseVerTolerant() error { bcVer, err := semver.ParseTolerant(bc.Version) if err != nil { return err @@ -34,14 +32,3 @@ func (bc *BreakingChanges) ParseVerTolerant() error { bc.ver = bcVer return nil } - -func (bc *BreakingChanges) FixColumns(query string) string { - // nolint: golint - // 2 because old - new - oldnew := make([]string, 0, 2*len(bc.Columns)) - for old := range bc.Columns { - oldnew = append(oldnew, old, bc.Columns[old]) - } - r := strings.NewReplacer(oldnew...) - return r.Replace(query) -} From b063e57ad329ac0a17ac796e644297619e3d6948 Mon Sep 17 00:00:00 2001 From: vl4deee11 <44677024+vl4deee11@users.noreply.github.com> Date: Wed, 9 Jun 2021 10:41:09 +0300 Subject: [PATCH 10/11] Update postgres_exporter.go Signed-off-by: vl4deee11 Signed-off-by: vl4deee11 Signed-off-by: vl4deee11 Signed-off-by: vl4deee11 --- cmd/postgres_exporter/postgres_exporter.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/postgres_exporter/postgres_exporter.go b/cmd/postgres_exporter/postgres_exporter.go index 6618071c4..2243c60cf 100644 --- a/cmd/postgres_exporter/postgres_exporter.go +++ b/cmd/postgres_exporter/postgres_exporter.go @@ -533,7 +533,7 @@ func parseUserQueries(content []byte, pgVersion semver.Version) (map[string]inte for metric, specs := range userQueries { level.Debug(logger).Log("msg", "New user metric namespace from YAML metric", "metric", metric, "cache_seconds", specs.CacheSeconds) newQueryOverrides[metric] = specs.Query - columnT := make(map[string]string, 0) + columnT := make(map[string]string) for i := range specs.BreakingChanges { if err := specs.BreakingChanges[i].parseVerTolerant(); err != nil { return nil, nil, err From 0110547187dccfd95812520c2c2804dc3e7e3071 Mon Sep 17 00:00:00 2001 From: vl4deee11 Date: Wed, 9 Jun 2021 20:38:58 +0100 Subject: [PATCH 11/11] delete unused Signed-off-by: vl4deee11 Signed-off-by: vl4deee11 Signed-off-by: vl4deee11 Signed-off-by: vl4deee11 --- cmd/postgres_exporter/postgres_exporter.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/cmd/postgres_exporter/postgres_exporter.go b/cmd/postgres_exporter/postgres_exporter.go index 2243c60cf..132e423a7 100644 --- a/cmd/postgres_exporter/postgres_exporter.go +++ b/cmd/postgres_exporter/postgres_exporter.go @@ -1342,10 +1342,6 @@ func queryNamespaceMapping(server *Server, namespace string, mapping MetricMapNa // an admin tool so you're not injecting SQL right? rows, err = server.db.Query(fmt.Sprintf("SELECT * FROM %s;", namespace)) // nolint: gas } else { - // make for pg13 breaking changes - if server.lastMapVersion.GE(semver.Version{Major: 13}) && namespace == "pg_stat_statements" { - fmt.Println(query) - } rows, err = server.db.Query(query) } if err != nil {