Add support for database schema in PostgreSQL (#8819)

* Add support for database schema

* Require setting search_path for the db user

* Add schema setting to admin/config.tmpl

* Use a schema different from default for psql tests

* Update postgres scripts to use custom schema

* Update to xorm/core 0.7.3 and xorm/xorm c37aff9b3a

* Fix migration test

Co-authored-by: Antoine GIRARD <[email protected]>
Co-authored-by: Lunny Xiao <[email protected]>

Author: 3 people

Makefile

@@ -79,6 +79,7 @@ TEST_PGSQL_HOST ?= pgsql:5432
 TEST_PGSQL_DBNAME ?= testgitea
 TEST_PGSQL_USERNAME ?= postgres
 TEST_PGSQL_PASSWORD ?= postgres
+TEST_PGSQL_SCHEMA ?= gtestschema
 TEST_MSSQL_HOST ?= mssql:1433
 TEST_MSSQL_DBNAME ?= gitea
 TEST_MSSQL_USERNAME ?= sa
@@ -306,6 +307,7 @@ generate-ini-pgsql:
 		-e 's|{{TEST_PGSQL_DBNAME}}|${TEST_PGSQL_DBNAME}|g' \
 		-e 's|{{TEST_PGSQL_USERNAME}}|${TEST_PGSQL_USERNAME}|g' \
 		-e 's|{{TEST_PGSQL_PASSWORD}}|${TEST_PGSQL_PASSWORD}|g' \
+		-e 's|{{TEST_PGSQL_SCHEMA}}|${TEST_PGSQL_SCHEMA}|g' \
 			integrations/pgsql.ini.tmpl > integrations/pgsql.ini
 
 .PHONY: test-pgsql

custom/conf/app.ini.sample

@@ -336,6 +336,10 @@ NAME = gitea
 USER = root
 ; Use PASSWD = `your password` for quoting if you use special characters in the password.
 PASSWD =
+; For Postgres, schema to use if different from "public". The schema must exist beforehand,
+; the user must have creation privileges on it, and the user search path must be set
+; to the look into the schema first. e.g.:ALTER USER user SET SEARCH_PATH = schema_name,"$user",public;
+SCHEMA =
 ; For Postgres, either "disable" (default), "require", or "verify-full"
 ; For MySQL, either "false" (default), "true", or "skip-verify"
 SSL_MODE = disable

docs/content/doc/advanced/config-cheat-sheet.en-us.md

@@ -209,6 +209,9 @@ Values containing `#` or `;` must be quoted using `` ` `` or `"""`.
 - `NAME`: **gitea**: Database name.
 - `USER`: **root**: Database username.
 - `PASSWD`: **\<empty\>**: Database user password. Use \`your password\` for quoting if you use special characters in the password.
+- `SCHEMA`: **\<empty\>**: For PostgreSQL only, schema to use if different from "public". The schema must exist beforehand,
+  the user must have creation privileges on it, and the user search path must be set to the look into the schema first 
+  (e.g. `ALTER USER user SET SEARCH_PATH = schema_name,"$user",public;`).
 - `SSL_MODE`: **disable**: For PostgreSQL and MySQL only.
 - `CHARSET`: **utf8**: For MySQL only, either "utf8" or "utf8mb4", default is "utf8". NOTICE: for "utf8mb4" you must use MySQL InnoDB > 5.6. Gitea is unable to check this.
 - `PATH`: **data/gitea.db**: For SQLite3 only, the database file path.

go.mod

@@ -112,6 +112,6 @@ require (
 	mvdan.cc/xurls/v2 v2.1.0
 	strk.kbt.io/projects/go/libravatar v0.0.0-20191008002943-06d1c002b251
 	xorm.io/builder v0.3.6
-	xorm.io/core v0.7.2
-	xorm.io/xorm v0.8.1
+	xorm.io/core v0.7.3
+	xorm.io/xorm v0.8.2-0.20200120024500-c37aff9b3a4a
 )

go.sum

@@ -760,7 +760,9 @@ xorm.io/builder v0.3.6 h1:ha28mQ2M+TFx96Hxo+iq6tQgnkC9IZkM6D8w9sKHHF8=
 xorm.io/builder v0.3.6/go.mod h1:LEFAPISnRzG+zxaxj2vPicRwz67BdhFreKg8yv8/TgU=
 xorm.io/core v0.7.2 h1:mEO22A2Z7a3fPaZMk6gKL/jMD80iiyNwRrX5HOv3XLw=
 xorm.io/core v0.7.2/go.mod h1:jJfd0UAEzZ4t87nbQYtVjmqpIODugN6PD2D9E+dJvdM=
+xorm.io/core v0.7.3 h1:W8ws1PlrnkS1CZU1YWaYLMQcQilwAmQXU0BJDJon+H0=
+xorm.io/core v0.7.3/go.mod h1:jJfd0UAEzZ4t87nbQYtVjmqpIODugN6PD2D9E+dJvdM=
 xorm.io/xorm v0.8.0 h1:iALxgJrX8O00f8Jk22GbZwPmxJNgssV5Mv4uc2HL9PM=
 xorm.io/xorm v0.8.0/go.mod h1:ZkJLEYLoVyg7amJK/5r779bHyzs2AU8f8VMiP6BM7uY=
-xorm.io/xorm v0.8.1 h1:4f2KXuQxVdaX3RdI3Fw81NzMiSpZeyCZt8m3sEVeIkQ=
-xorm.io/xorm v0.8.1/go.mod h1:ZkJLEYLoVyg7amJK/5r779bHyzs2AU8f8VMiP6BM7uY=
+xorm.io/xorm v0.8.2-0.20200120024500-c37aff9b3a4a h1:hzGd080rlkZ5a7v6Tr3x8PJJnWPfKxGMMl92c8DNcww=
+xorm.io/xorm v0.8.2-0.20200120024500-c37aff9b3a4a/go.mod h1:ZkJLEYLoVyg7amJK/5r779bHyzs2AU8f8VMiP6BM7uY=

integrations/integration_test.go

@@ -153,18 +153,53 @@ func initIntegrationTest() {
 		if err != nil {
 			log.Fatalf("sql.Open: %v", err)
 		}
-		rows, err := db.Query(fmt.Sprintf("SELECT 1 FROM pg_database WHERE datname = '%s'", setting.Database.Name))
+		dbrows, err := db.Query(fmt.Sprintf("SELECT 1 FROM pg_database WHERE datname = '%s'", setting.Database.Name))
 		if err != nil {
 			log.Fatalf("db.Query: %v", err)
 		}
-		defer rows.Close()
+		defer dbrows.Close()
 
-		if rows.Next() {
+		if !dbrows.Next() {
+			if _, err = db.Exec(fmt.Sprintf("CREATE DATABASE %s", setting.Database.Name)); err != nil {
+				log.Fatalf("db.Exec: CREATE DATABASE: %v", err)
+			}
+		}
+		// Check if we need to setup a specific schema
+		if len(setting.Database.Schema) == 0 {
 			break
 		}
-		if _, err = db.Exec(fmt.Sprintf("CREATE DATABASE %s", setting.Database.Name)); err != nil {
-			log.Fatalf("db.Exec: %v", err)
+		db.Close()
+
+		db, err = sql.Open("postgres", fmt.Sprintf("postgres://%s:%s@%s/%s?sslmode=%s",
+			setting.Database.User, setting.Database.Passwd, setting.Database.Host, setting.Database.Name, setting.Database.SSLMode))
+		// This is a different db object; requires a different Close()
+		defer db.Close()
+		if err != nil {
+			log.Fatalf("sql.Open: %v", err)
+		}
+		schrows, err := db.Query(fmt.Sprintf("SELECT 1 FROM information_schema.schemata WHERE schema_name = '%s'", setting.Database.Schema))
+		if err != nil {
+			log.Fatalf("db.Query: %v", err)
+		}
+		defer schrows.Close()
+
+		if !schrows.Next() {
+			// Create and setup a DB schema
+			if _, err = db.Exec(fmt.Sprintf("CREATE SCHEMA %s", setting.Database.Schema)); err != nil {
+				log.Fatalf("db.Exec: CREATE SCHEMA: %v", err)
+			}
+		}
+
+		// Make the user's default search path the created schema; this will affect new connections
+		if _, err = db.Exec(fmt.Sprintf(`ALTER USER "%s" SET search_path = %s`, setting.Database.User, setting.Database.Schema)); err != nil {
+			log.Fatalf("db.Exec: ALTER USER SET search_path: %v", err)
+		}
+
+		// Make the current connection's search the created schema
+		if _, err = db.Exec(fmt.Sprintf(`SET search_path = %s`, setting.Database.Schema)); err != nil {
+			log.Fatalf("db.Exec: ALTER USER SET search_path: %v", err)
 		}
+
 	case setting.Database.UseMSSQL:
 		host, port := setting.ParseMSSQLHostPort(setting.Database.Host)
 		db, err := sql.Open("mssql", fmt.Sprintf("server=%s; port=%s; database=%s; user id=%s; password=%s;",

integrations/migration-test/gitea-v1.5.3.postgres.sql.gz

@@ -168,6 +168,32 @@ func restoreOldDB(t *testing.T, version string) bool {
 		assert.NoError(t, err)
 		db.Close()
 
+		// Check if we need to setup a specific schema
+		if len(setting.Database.Schema) != 0 {
+			db, err = sql.Open("postgres", fmt.Sprintf("postgres://%s:%s@%s/%s?sslmode=%s",
+				setting.Database.User, setting.Database.Passwd, setting.Database.Host, setting.Database.Name, setting.Database.SSLMode))
+			if !assert.NoError(t, err) {
+				return false
+			}
+			schrows, err := db.Query(fmt.Sprintf("SELECT 1 FROM information_schema.schemata WHERE schema_name = '%s'", setting.Database.Schema))
+			if !assert.NoError(t, err) || !assert.NotEmpty(t, schrows) {
+				return false
+			}
+
+			if !schrows.Next() {
+				// Create and setup a DB schema
+				_, err = db.Exec(fmt.Sprintf("CREATE SCHEMA %s", setting.Database.Schema))
+				assert.NoError(t, err)
+			}
+			schrows.Close()
+
+			// Make the user's default search path the created schema; this will affect new connections
+			_, err = db.Exec(fmt.Sprintf(`ALTER USER "%s" SET search_path = %s`, setting.Database.User, setting.Database.Schema))
+			assert.NoError(t, err)
+
+			db.Close()
+		}
+
 		db, err = sql.Open("postgres", fmt.Sprintf("postgres://%s:%s@%s/%s?sslmode=%s",
 			setting.Database.User, setting.Database.Passwd, setting.Database.Host, setting.Database.Name, setting.Database.SSLMode))
 		assert.NoError(t, err)

integrations/migration-test/gitea-v1.6.4.postgres.sql.gz

@@ -7,6 +7,7 @@ HOST     = {{TEST_PGSQL_HOST}}
 NAME     = {{TEST_PGSQL_DBNAME}}
 USER     = {{TEST_PGSQL_USERNAME}}
 PASSWD   = {{TEST_PGSQL_PASSWORD}}
+SCHEMA   = {{TEST_PGSQL_SCHEMA}}
 SSL_MODE = disable
 
 [indexer]

integrations/migration-test/gitea-v1.7.0.postgres.sql.gz

@@ -128,7 +128,12 @@ func getEngine() (*xorm.Engine, error) {
 		return nil, err
 	}
 
-	return xorm.NewEngine(setting.Database.Type, connStr)
+	engine, err := xorm.NewEngine(setting.Database.Type, connStr)
+	if err != nil {
+		return nil, err
+	}
+	engine.SetSchema(setting.Database.Schema)
+	return engine, nil
 }
 
 // NewTestEngine sets a new test xorm.Engine

integrations/migration-test/migration_test.go

@@ -25,6 +25,7 @@ type InstallForm struct {
 	SSLMode  string
 	Charset  string `binding:"Required;In(utf8,utf8mb4)"`
 	DbPath   string
+	DbSchema string
 
 	AppName      string `binding:"Required" locale:"install.app_name"`
 	RepoRootPath string `binding:"Required"`

integrations/pgsql.ini.tmpl

@@ -30,6 +30,7 @@ var (
 		Name              string
 		User              string
 		Passwd            string
+		Schema            string
 		SSLMode           string
 		Path              string
 		LogSQL            bool
@@ -75,6 +76,7 @@ func InitDBConfig() {
 	if len(Database.Passwd) == 0 {
 		Database.Passwd = sec.Key("PASSWD").String()
 	}
+	Database.Schema = sec.Key("SCHEMA").String()
 	Database.SSLMode = sec.Key("SSL_MODE").MustString("disable")
 	Database.Charset = sec.Key("CHARSET").In("utf8", []string{"utf8", "utf8mb4"})
 	Database.Path = sec.Key("PATH").MustString(filepath.Join(AppDataPath, "gitea.db"))

models/models.go

@@ -102,6 +102,8 @@ user = Username
 password = Password
 db_name = Database Name
 db_helper = Note to MySQL users: please use the InnoDB storage engine and if you use "utf8mb4", your InnoDB version must be greater than 5.6 .
+db_schema = Schema
+db_schema_helper = Leave blank for database default ("public").
 ssl_mode = SSL
 charset = Charset
 path = Path
@@ -1953,6 +1955,7 @@ config.db_type = Type
 config.db_host = Host
 config.db_name = Name
 config.db_user = Username
+config.db_schema = Schema
 config.db_ssl_mode = SSL
 config.db_path = Path
 

modules/auth/user_form.go

@@ -54,6 +54,7 @@ func Install(ctx *context.Context) {
 	form.DbPasswd = setting.Database.Passwd
 	form.DbName = setting.Database.Name
 	form.DbPath = setting.Database.Path
+	form.DbSchema = setting.Database.Schema
 	form.Charset = setting.Database.Charset
 
 	ctx.Data["CurDbOption"] = "MySQL"
@@ -147,6 +148,7 @@ func InstallPost(ctx *context.Context, form auth.InstallForm) {
 	setting.Database.User = form.DbUser
 	setting.Database.Passwd = form.DbPasswd
 	setting.Database.Name = form.DbName
+	setting.Database.Schema = form.DbSchema
 	setting.Database.SSLMode = form.SSLMode
 	setting.Database.Charset = form.Charset
 	setting.Database.Path = form.DbPath
@@ -267,6 +269,7 @@ func InstallPost(ctx *context.Context, form auth.InstallForm) {
 	cfg.Section("database").Key("NAME").SetValue(setting.Database.Name)
 	cfg.Section("database").Key("USER").SetValue(setting.Database.User)
 	cfg.Section("database").Key("PASSWD").SetValue(setting.Database.Passwd)
+	cfg.Section("database").Key("SCHEMA").SetValue(setting.Database.Schema)
 	cfg.Section("database").Key("SSL_MODE").SetValue(setting.Database.SSLMode)
 	cfg.Section("database").Key("CHARSET").SetValue(setting.Database.Charset)
 	cfg.Section("database").Key("PATH").SetValue(setting.Database.Path)

modules/setting/database.go

@@ -128,6 +128,8 @@
 					<dd>{{if .DbCfg.User}}{{.DbCfg.User}}{{else}}-{{end}}</dd>
 				{{end}}
 				{{if eq .DbCfg.Type "postgres"}}
+					<dt>{{.i18n.Tr "admin.config.db_schema"}}</dt>
+					<dd>{{if .DbCfg.Schema}}{{.DbCfg.Schema}}{{else}}-{{end}}</dd>
 					<dt>{{.i18n.Tr "admin.config.db_ssl_mode"}}</dt>
 					<dd>{{if .DbCfg.SSLMode}}{{.DbCfg.SSLMode}}{{else}}-{{end}}</dd>
 				{{end}}

options/locale/locale_en-US.ini

@@ -62,6 +62,11 @@
 								</div>
 							</div>
 						</div>
+						<div class="inline field {{if .Err_DbSetting}}error{{end}}">
+							<label for="db_schema">{{.i18n.Tr "install.db_schema"}}</label>
+							<input id="db_schema" name="db_schema" value="{{.db_schema}}">
+							<span class="help">{{.i18n.Tr "install.db_schema_helper"}}</span>
+						</div>
 					</div>
 
 					<div id="mysql_settings" class="{{if not (eq .CurDbOption "MySQL")}}hide{{end}}">

routers/install.go

@@ -613,7 +613,7 @@ mvdan.cc/xurls/v2
 strk.kbt.io/projects/go/libravatar
 # xorm.io/builder v0.3.6
 xorm.io/builder
-# xorm.io/core v0.7.2
+# xorm.io/core v0.7.3
 xorm.io/core
-# xorm.io/xorm v0.8.1
+# xorm.io/xorm v0.8.2-0.20200120024500-c37aff9b3a4a
 xorm.io/xorm