From 11c251bd25d1febbd7b7a35eef4195eef7483de4 Mon Sep 17 00:00:00 2001 From: Adrian Macneil Date: Sun, 19 Jul 2020 15:04:45 -0700 Subject: [PATCH 01/55] Redact passwords in error messages (#145) Fixes #144 --- main.go | 11 ++++++++++- main_test.go | 20 ++++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/main.go b/main.go index 8452277..99909f7 100644 --- a/main.go +++ b/main.go @@ -5,6 +5,7 @@ import ( "log" "net/url" "os" + "regexp" "github.com/joho/godotenv" "github.com/urfave/cli" @@ -19,7 +20,8 @@ func main() { err := app.Run(os.Args) if err != nil { - _, _ = fmt.Fprintf(os.Stderr, "Error: %s\n", err) + errText := redactLogString(fmt.Sprintf("Error: %s\n", err)) + _, _ = fmt.Fprint(os.Stderr, errText) os.Exit(2) } } @@ -219,3 +221,10 @@ func getDatabaseURL(c *cli.Context) (u *url.URL, err error) { return url.Parse(value) } + +// redactLogString attempts to redact passwords from errors +func redactLogString(in string) string { + re := regexp.MustCompile("([a-zA-Z]+://[^:]+:)[^@]+@") + + return re.ReplaceAllString(in, "${1}********@") +} diff --git a/main_test.go b/main_test.go index 1d20d06..d6922db 100644 --- a/main_test.go +++ b/main_test.go @@ -35,3 +35,23 @@ func TestGetDatabaseUrl(t *testing.T) { require.Equal(t, "example.org", u.Host) require.Equal(t, "/db", u.Path) } + +func TestRedactLogString(t *testing.T) { + examples := []struct { + in string + expected string + }{ + {"normal string", + "normal string"}, + // malformed URL example (note forward slash in password) + {"parse \"mysql://username:otS33+tb/e4=@localhost:3306/database\": invalid", + "parse \"mysql://username:********@localhost:3306/database\": invalid"}, + // invalid port, but probably not a password since there is no @ + {"parse \"mysql://localhost:abc/database\": invalid", + "parse \"mysql://localhost:abc/database\": invalid"}, + } + + for _, ex := range examples { + require.Equal(t, ex.expected, redactLogString(ex.in)) + } +} From 8234882546b6ea9be9819f5da9181ffbcb6f42f2 Mon Sep 17 00:00:00 2001 From: Adrian Macneil Date: Sun, 19 Jul 2020 15:23:14 -0700 Subject: [PATCH 02/55] v1.9.1 (#146) --- pkg/dbmate/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/dbmate/version.go b/pkg/dbmate/version.go index 0450345..93532b0 100644 --- a/pkg/dbmate/version.go +++ b/pkg/dbmate/version.go @@ -1,4 +1,4 @@ package dbmate // Version of dbmate -const Version = "1.9.0" +const Version = "1.9.1" From c2dd1bd5af362ca413e7555efa546f490d30cd49 Mon Sep 17 00:00:00 2001 From: Ilia Ablamonov Date: Sat, 8 Aug 2020 00:38:48 +0400 Subject: [PATCH 03/55] Add ClickHouse support (#140) --- Makefile | 1 + README.md | 23 ++- docker-compose.yml | 5 + go.mod | 1 + go.sum | 13 ++ pkg/dbmate/clickhouse.go | 289 ++++++++++++++++++++++++++++++++ pkg/dbmate/clickhouse_test.go | 305 ++++++++++++++++++++++++++++++++++ 7 files changed, 633 insertions(+), 4 deletions(-) create mode 100644 pkg/dbmate/clickhouse.go create mode 100644 pkg/dbmate/clickhouse_test.go diff --git a/Makefile b/Makefile index 9082e15..01c2372 100644 --- a/Makefile +++ b/Makefile @@ -19,6 +19,7 @@ lint: wait: dist/dbmate-linux-amd64 -e MYSQL_URL wait dist/dbmate-linux-amd64 -e POSTGRESQL_URL wait + dist/dbmate-linux-amd64 -e CLICKHOUSE_URL wait .PHONY: clean clean: diff --git a/README.md b/README.md index 0eb4ea1..b536350 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ For a comparison between dbmate and other popular database schema migration tool ## Features -* Supports MySQL, PostgreSQL, and SQLite. +* Supports MySQL, PostgreSQL, SQLite, and ClickHouse. * Uses plain SQL for writing schema migrations. * Migrations are timestamp-versioned, to avoid version number conflicts with multiple developers. * Migrations are run atomically inside a transaction. @@ -117,7 +117,7 @@ DATABASE_URL="postgres://postgres@127.0.0.1:5432/myapp_development?sslmode=disab protocol://username:password@host:port/database_name?options ``` -* `protocol` must be one of `mysql`, `postgres`, `postgresql`, `sqlite`, `sqlite3` +* `protocol` must be one of `mysql`, `postgres`, `postgresql`, `sqlite`, `sqlite3`, `clickhouse` * `host` can be either a hostname or IP address * `options` are driver-specific (refer to the underlying Go SQL drivers if you wish to use these) @@ -161,6 +161,20 @@ To specify an absolute path, add an additional forward slash to the path. The fo DATABASE_URL="sqlite:////tmp/database_name.sqlite3" ``` +**ClickHouse** + +```sh +DATABASE_URL="clickhouse://username:password@127.0.0.1:9000/database_name" +``` + +or + +```sh +DATABASE_URL="clickhouse://127.0.0.1:9000?username=username&password=password&database=database_name" +``` + +[See other supported connection options](https://github.com/ClickHouse/clickhouse-go#dsn). + ### Creating Migrations To create a new migration, run `dbmate new create_users_table`. You can name the migration anything you like. This will create a file `db/migrations/20151127184807_create_users_table.sql` in the current directory: @@ -333,7 +347,7 @@ Why another database schema migration tool? Dbmate was inspired by many other to | | [goose](https://bitbucket.org/liamstask/goose/) | [sql-migrate](https://github.com/rubenv/sql-migrate) | [golang-migrate/migrate](https://github.com/golang-migrate/migrate) | [activerecord](http://guides.rubyonrails.org/active_record_migrations.html) | [sequelize](http://docs.sequelizejs.com/manual/tutorial/migrations.html) | [dbmate](https://github.com/amacneil/dbmate) | | --- |:---:|:---:|:---:|:---:|:---:|:---:| -| **Features** ||||||| +| **Features** | |Plain SQL migration files|:white_check_mark:|:white_check_mark:|:white_check_mark:|||:white_check_mark:| |Support for creating and dropping databases||||:white_check_mark:||:white_check_mark:| |Support for saving schema dump files||||:white_check_mark:||:white_check_mark:| @@ -343,10 +357,11 @@ Why another database schema migration tool? Dbmate was inspired by many other to |Automatically load .env file||||||:white_check_mark:| |No separate configuration file||||:white_check_mark:|:white_check_mark:|:white_check_mark:| |Language/framework independent|:eight_pointed_black_star:|:eight_pointed_black_star:|:eight_pointed_black_star:|||:white_check_mark:| -| **Drivers** ||||||| +| **Drivers** | |PostgreSQL|:white_check_mark:|:white_check_mark:|:white_check_mark:|:white_check_mark:|:white_check_mark:|:white_check_mark:| |MySQL|:white_check_mark:|:white_check_mark:|:white_check_mark:|:white_check_mark:|:white_check_mark:|:white_check_mark:| |SQLite|:white_check_mark:|:white_check_mark:|:white_check_mark:|:white_check_mark:|:white_check_mark:|:white_check_mark:| +|CliсkHouse|||:white_check_mark:|:white_check_mark:|:white_check_mark:|:white_check_mark:| > :eight_pointed_black_star: In theory these tools could be used with other languages, but a Go development environment is required because binary builds are not provided. diff --git a/docker-compose.yml b/docker-compose.yml index 0f81590..57b9996 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,9 +7,11 @@ services: depends_on: - mysql - postgres + - clickhouse environment: MYSQL_URL: mysql://root:root@mysql/dbmate POSTGRESQL_URL: postgres://postgres:postgres@postgres/dbmate?sslmode=disable + CLICKHOUSE_URL: clickhouse://clickhouse:9000?database=dbmate mysql: image: mysql:5.7 @@ -20,3 +22,6 @@ services: image: postgres:10 environment: POSTGRES_PASSWORD: postgres + + clickhouse: + image: yandex/clickhouse-server:19.16 diff --git a/go.mod b/go.mod index d9493c6..a67ca27 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/amacneil/dbmate go 1.14 require ( + github.com/ClickHouse/clickhouse-go v1.4.1 github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/go-sql-driver/mysql v1.5.0 diff --git a/go.sum b/go.sum index 9719e7f..19df7c5 100644 --- a/go.sum +++ b/go.sum @@ -1,12 +1,20 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/ClickHouse/clickhouse-go v1.4.1 h1:D9cihLg76O1ZyILLaXq1eksYzEuV010NdvucgKGGK14= +github.com/ClickHouse/clickhouse-go v1.4.1/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI= +github.com/bkaradzic/go-lz4 v1.0.0 h1:RXc4wYsyz985CkXXeX04y4VnZFGG8Rd43pRaHsOXAKk= +github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4= +github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 h1:F1EaeKL/ta07PY/k9Os/UFtwERei2/XzGemhpGnBKNg= +github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d h1:cVtBfNW5XTHiKQe7jDaDBSh/EVM4XLPutLAGboIXuM0= @@ -16,10 +24,14 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.5.2 h1:yTSXVswvWUOQ3k1sd7vJfDrbSl8lKuscqFJRqjC0ifw= github.com/lib/pq v1.5.2/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v1.13.0 h1:LnJI81JidiW9r7pS/hXe6cFeO5EXNq7KbfvoJLRI69c= github.com/mattn/go-sqlite3 v1.13.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I= +github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= @@ -27,6 +39,7 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/urfave/cli v1.22.4 h1:u7tSpNPPswAFymm8IehJhy4uJMlUuU/GmqSkvJ1InXA= diff --git a/pkg/dbmate/clickhouse.go b/pkg/dbmate/clickhouse.go new file mode 100644 index 0000000..35a0901 --- /dev/null +++ b/pkg/dbmate/clickhouse.go @@ -0,0 +1,289 @@ +package dbmate + +import ( + "bytes" + "database/sql" + "fmt" + "net/url" + "regexp" + "sort" + "strings" + + "github.com/ClickHouse/clickhouse-go" +) + +func init() { + RegisterDriver(ClickHouseDriver{}, "clickhouse") +} + +// ClickHouseDriver provides top level database functions +type ClickHouseDriver struct { +} + +func normalizeClickHouseURL(initialURL *url.URL) *url.URL { + u := *initialURL + + u.Scheme = "tcp" + host := u.Host + if u.Port() == "" { + host = fmt.Sprintf("%s:9000", host) + } + u.Host = host + + query := u.Query() + if query.Get("username") == "" && u.User.Username() != "" { + query.Set("username", u.User.Username()) + } + password, passwordSet := u.User.Password() + if query.Get("password") == "" && passwordSet { + query.Set("password", password) + } + u.User = nil + + if query.Get("database") == "" { + path := strings.Trim(u.Path, "/") + if path != "" { + query.Set("database", path) + u.Path = "" + } + } + u.RawQuery = query.Encode() + + return &u +} + +// Open creates a new database connection +func (drv ClickHouseDriver) Open(u *url.URL) (*sql.DB, error) { + return sql.Open("clickhouse", normalizeClickHouseURL(u).String()) +} + +func (drv ClickHouseDriver) openClickHouseDB(u *url.URL) (*sql.DB, error) { + // connect to clickhouse database + clickhouseURL := normalizeClickHouseURL(u) + values := clickhouseURL.Query() + values.Set("database", "default") + clickhouseURL.RawQuery = values.Encode() + + return drv.Open(clickhouseURL) +} + +func (drv ClickHouseDriver) databaseName(u *url.URL) string { + name := normalizeClickHouseURL(u).Query().Get("database") + if name == "" { + name = "default" + } + return name +} + +var clickhouseValidIdentifier = regexp.MustCompile(`^[a-zA-Z_][0-9a-zA-Z_]*$`) + +func clickhouseQuoteIdentifier(str string) string { + if clickhouseValidIdentifier.MatchString(str) { + return str + } + + str = strings.Replace(str, `"`, `""`, -1) + + return fmt.Sprintf(`"%s"`, str) +} + +// CreateDatabase creates the specified database +func (drv ClickHouseDriver) CreateDatabase(u *url.URL) error { + name := drv.databaseName(u) + fmt.Printf("Creating: %s\n", name) + + db, err := drv.openClickHouseDB(u) + if err != nil { + return err + } + defer mustClose(db) + + _, err = db.Exec("create database " + clickhouseQuoteIdentifier(name)) + + return err +} + +// DropDatabase drops the specified database (if it exists) +func (drv ClickHouseDriver) DropDatabase(u *url.URL) error { + name := drv.databaseName(u) + fmt.Printf("Dropping: %s\n", name) + + db, err := drv.openClickHouseDB(u) + if err != nil { + return err + } + defer mustClose(db) + + _, err = db.Exec("drop database if exists " + clickhouseQuoteIdentifier(name)) + + return err +} + +func clickhouseSchemaDump(db *sql.DB, buf *bytes.Buffer, databaseName string) error { + buf.WriteString("\n--\n-- Database schema\n--\n\n") + + buf.WriteString("CREATE DATABASE " + clickhouseQuoteIdentifier(databaseName) + " IF NOT EXISTS;\n\n") + + tables, err := queryColumn(db, "show tables") + if err != nil { + return err + } + sort.Strings(tables) + + for _, table := range tables { + var clause string + err = db.QueryRow("show create table " + clickhouseQuoteIdentifier(table)).Scan(&clause) + if err != nil { + return err + } + buf.WriteString(clause + ";\n\n") + } + return nil +} + +func clickhouseSchemaMigrationsDump(db *sql.DB, buf *bytes.Buffer) error { + // load applied migrations + migrations, err := queryColumn(db, + "select version from schema_migrations final where applied order by version asc", + ) + if err != nil { + return err + } + + quoter := strings.NewReplacer(`\`, `\\`, `'`, `\'`) + for i := range migrations { + migrations[i] = "'" + quoter.Replace(migrations[i]) + "'" + } + + // build schema_migrations table data + buf.WriteString("\n--\n-- Dbmate schema migrations\n--\n\n") + + if len(migrations) > 0 { + buf.WriteString("INSERT INTO schema_migrations (version) VALUES\n (" + + strings.Join(migrations, "),\n (") + + ");\n") + } + + return nil +} + +// DumpSchema returns the current database schema +func (drv ClickHouseDriver) DumpSchema(u *url.URL, db *sql.DB) ([]byte, error) { + var buf bytes.Buffer + var err error + + err = clickhouseSchemaDump(db, &buf, drv.databaseName(u)) + if err != nil { + return nil, err + } + + err = clickhouseSchemaMigrationsDump(db, &buf) + if err != nil { + return nil, err + } + + return buf.Bytes(), nil +} + +// DatabaseExists determines whether the database exists +func (drv ClickHouseDriver) DatabaseExists(u *url.URL) (bool, error) { + name := drv.databaseName(u) + + db, err := drv.openClickHouseDB(u) + if err != nil { + return false, err + } + defer mustClose(db) + + exists := false + err = db.QueryRow("SELECT 1 FROM system.databases where name = ?", name). + Scan(&exists) + if err == sql.ErrNoRows { + return false, nil + } + + return exists, err +} + +// CreateMigrationsTable creates the schema_migrations table +func (drv ClickHouseDriver) CreateMigrationsTable(db *sql.DB) error { + _, err := db.Exec(` + create table if not exists schema_migrations ( + version String, + ts DateTime default now(), + applied UInt8 default 1 + ) engine = ReplacingMergeTree(ts) + primary key version + order by version + `) + return err +} + +// SelectMigrations returns a list of applied migrations +// with an optional limit (in descending order) +func (drv ClickHouseDriver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, error) { + query := "select version from schema_migrations final where applied order by version desc" + if limit >= 0 { + query = fmt.Sprintf("%s limit %d", query, limit) + } + rows, err := db.Query(query) + if err != nil { + return nil, err + } + + defer mustClose(rows) + + migrations := map[string]bool{} + for rows.Next() { + var version string + if err := rows.Scan(&version); err != nil { + return nil, err + } + + migrations[version] = true + } + + return migrations, nil +} + +// InsertMigration adds a new migration record +func (drv ClickHouseDriver) InsertMigration(db Transaction, version string) error { + _, err := db.Exec("insert into schema_migrations (version) values (?)", version) + return err +} + +// DeleteMigration removes a migration record +func (drv ClickHouseDriver) DeleteMigration(db Transaction, version string) error { + _, err := db.Exec( + "insert into schema_migrations (version, applied) values (?, ?)", + version, false, + ) + + return err +} + +// Ping verifies a connection to the database server. It does not verify whether the +// specified database exists. +func (drv ClickHouseDriver) Ping(u *url.URL) error { + // attempt connection to primary database, not "clickhouse" database + // to support servers with no "clickhouse" database + // (see https://github.com/amacneil/dbmate/issues/78) + db, err := drv.Open(u) + if err != nil { + return err + } + defer mustClose(db) + + err = db.Ping() + if err == nil { + return nil + } + + // ignore 'Database foo doesn't exist' error + chErr, ok := err.(*clickhouse.Exception) + if ok && chErr.Code == 81 { + return nil + } + + return err +} diff --git a/pkg/dbmate/clickhouse_test.go b/pkg/dbmate/clickhouse_test.go new file mode 100644 index 0000000..48285aa --- /dev/null +++ b/pkg/dbmate/clickhouse_test.go @@ -0,0 +1,305 @@ +package dbmate + +import ( + "database/sql" + "net/url" + "testing" + + "github.com/stretchr/testify/require" +) + +func clickhouseTestURL(t *testing.T) *url.URL { + u, err := url.Parse("clickhouse://clickhouse:9000?database=dbmate") + require.NoError(t, err) + + return u +} + +func prepTestClickHouseDB(t *testing.T) *sql.DB { + drv := ClickHouseDriver{} + u := clickhouseTestURL(t) + + // drop any existing database + err := drv.DropDatabase(u) + require.NoError(t, err) + + // create database + err = drv.CreateDatabase(u) + require.NoError(t, err) + + // connect database + db, err := sql.Open("clickhouse", u.String()) + require.NoError(t, err) + + return db +} + +func TestNormalizeClickHouseURLSimplified(t *testing.T) { + u, err := url.Parse("clickhouse://user:pass@host/db") + require.NoError(t, err) + + s := normalizeClickHouseURL(u).String() + require.Equal(t, "tcp://host:9000?database=db&password=pass&username=user", s) +} + +func TestNormalizeClickHouseURLCanonical(t *testing.T) { + u, err := url.Parse("clickhouse://host:9000?database=db&password=pass&username=user") + require.NoError(t, err) + + s := normalizeClickHouseURL(u).String() + require.Equal(t, "tcp://host:9000?database=db&password=pass&username=user", s) +} + +func TestClickHouseCreateDropDatabase(t *testing.T) { + drv := ClickHouseDriver{} + u := clickhouseTestURL(t) + + // drop any existing database + err := drv.DropDatabase(u) + require.NoError(t, err) + + // create database + err = drv.CreateDatabase(u) + require.NoError(t, err) + + // check that database exists and we can connect to it + func() { + db, err := sql.Open("clickhouse", u.String()) + require.NoError(t, err) + defer mustClose(db) + + err = db.Ping() + require.NoError(t, err) + }() + + // drop the database + err = drv.DropDatabase(u) + require.NoError(t, err) + + // check that database no longer exists + func() { + db, err := sql.Open("clickhouse", u.String()) + require.NoError(t, err) + defer mustClose(db) + + err = db.Ping() + require.EqualError(t, err, "code: 81, message: Database dbmate doesn't exist") + }() +} + +func TestClickHouseDumpSchema(t *testing.T) { + drv := ClickHouseDriver{} + u := clickhouseTestURL(t) + + // prepare database + db := prepTestClickHouseDB(t) + defer mustClose(db) + err := drv.CreateMigrationsTable(db) + require.NoError(t, err) + + // insert migration + tx, err := db.Begin() + require.NoError(t, err) + err = drv.InsertMigration(tx, "abc1") + require.NoError(t, err) + err = tx.Commit() + require.NoError(t, err) + tx, err = db.Begin() + require.NoError(t, err) + err = drv.InsertMigration(tx, "abc2") + require.NoError(t, err) + err = tx.Commit() + require.NoError(t, err) + + // DumpSchema should return schema + schema, err := drv.DumpSchema(u, db) + require.NoError(t, err) + require.Contains(t, string(schema), "CREATE TABLE "+drv.databaseName(u)+".schema_migrations") + require.Contains(t, string(schema), "--\n"+ + "-- Dbmate schema migrations\n"+ + "--\n\n"+ + "INSERT INTO schema_migrations (version) VALUES\n"+ + " ('abc1'),\n"+ + " ('abc2');\n") + + // DumpSchema should return error if command fails + values := u.Query() + values.Set("database", "fakedb") + u.RawQuery = values.Encode() + db, err = sql.Open("clickhouse", u.String()) + require.NoError(t, err) + + schema, err = drv.DumpSchema(u, db) + require.Nil(t, schema) + require.EqualError(t, err, "code: 81, message: Database fakedb doesn't exist") +} + +func TestClickHouseDatabaseExists(t *testing.T) { + drv := ClickHouseDriver{} + u := clickhouseTestURL(t) + + // drop any existing database + err := drv.DropDatabase(u) + require.NoError(t, err) + + // DatabaseExists should return false + exists, err := drv.DatabaseExists(u) + require.NoError(t, err) + require.Equal(t, false, exists) + + // create database + err = drv.CreateDatabase(u) + require.NoError(t, err) + + // DatabaseExists should return true + exists, err = drv.DatabaseExists(u) + require.NoError(t, err) + require.Equal(t, true, exists) +} + +func TestClickHouseDatabaseExists_Error(t *testing.T) { + drv := ClickHouseDriver{} + u := clickhouseTestURL(t) + values := u.Query() + values.Set("username", "invalid") + u.RawQuery = values.Encode() + + exists, err := drv.DatabaseExists(u) + require.EqualError(t, err, "code: 192, message: Unknown user invalid") + require.Equal(t, false, exists) +} + +func TestClickHouseCreateMigrationsTable(t *testing.T) { + drv := ClickHouseDriver{} + db := prepTestClickHouseDB(t) + defer mustClose(db) + + // migrations table should not exist + count := 0 + err := db.QueryRow("select count(*) from schema_migrations").Scan(&count) + require.EqualError(t, err, "code: 60, message: Table dbmate.schema_migrations doesn't exist.") + + // create table + err = drv.CreateMigrationsTable(db) + require.NoError(t, err) + + // migrations table should exist + err = db.QueryRow("select count(*) from schema_migrations").Scan(&count) + require.NoError(t, err) + + // create table should be idempotent + err = drv.CreateMigrationsTable(db) + require.NoError(t, err) +} + +func TestClickHouseSelectMigrations(t *testing.T) { + drv := ClickHouseDriver{} + db := prepTestClickHouseDB(t) + defer mustClose(db) + + err := drv.CreateMigrationsTable(db) + require.NoError(t, err) + + tx, err := db.Begin() + require.NoError(t, err) + stmt, err := tx.Prepare("insert into schema_migrations (version) values (?)") + require.NoError(t, err) + _, err = stmt.Exec("abc2") + require.NoError(t, err) + _, err = stmt.Exec("abc1") + require.NoError(t, err) + _, err = stmt.Exec("abc3") + require.NoError(t, err) + err = tx.Commit() + require.NoError(t, err) + + migrations, err := drv.SelectMigrations(db, -1) + require.NoError(t, err) + require.Equal(t, true, migrations["abc1"]) + require.Equal(t, true, migrations["abc2"]) + require.Equal(t, true, migrations["abc2"]) + + // test limit param + migrations, err = drv.SelectMigrations(db, 1) + require.NoError(t, err) + require.Equal(t, true, migrations["abc3"]) + require.Equal(t, false, migrations["abc1"]) + require.Equal(t, false, migrations["abc2"]) +} + +func TestClickHouseInsertMigration(t *testing.T) { + drv := ClickHouseDriver{} + db := prepTestClickHouseDB(t) + defer mustClose(db) + + err := drv.CreateMigrationsTable(db) + require.NoError(t, err) + + count := 0 + err = db.QueryRow("select count(*) from schema_migrations").Scan(&count) + require.NoError(t, err) + require.Equal(t, 0, count) + + // insert migration + tx, err := db.Begin() + require.NoError(t, err) + err = drv.InsertMigration(tx, "abc1") + require.NoError(t, err) + err = tx.Commit() + require.NoError(t, err) + + err = db.QueryRow("select count(*) from schema_migrations where version = 'abc1'").Scan(&count) + require.NoError(t, err) + require.Equal(t, 1, count) +} + +func TestClickHouseDeleteMigration(t *testing.T) { + drv := ClickHouseDriver{} + db := prepTestClickHouseDB(t) + defer mustClose(db) + + err := drv.CreateMigrationsTable(db) + require.NoError(t, err) + + tx, err := db.Begin() + require.NoError(t, err) + stmt, err := tx.Prepare("insert into schema_migrations (version) values (?)") + require.NoError(t, err) + _, err = stmt.Exec("abc2") + require.NoError(t, err) + _, err = stmt.Exec("abc1") + require.NoError(t, err) + err = tx.Commit() + require.NoError(t, err) + + tx, err = db.Begin() + require.NoError(t, err) + err = drv.DeleteMigration(tx, "abc2") + require.NoError(t, err) + err = tx.Commit() + require.NoError(t, err) + + count := 0 + err = db.QueryRow("select count(*) from schema_migrations final where applied").Scan(&count) + require.NoError(t, err) + require.Equal(t, 1, count) +} + +func TestClickHousePing(t *testing.T) { + drv := ClickHouseDriver{} + u := clickhouseTestURL(t) + + // drop any existing database + err := drv.DropDatabase(u) + require.NoError(t, err) + + // ping database + err = drv.Ping(u) + require.NoError(t, err) + + // ping invalid host should return error + u.Host = "clickhouse:404" + err = drv.Ping(u) + require.Error(t, err) + require.Contains(t, err.Error(), "connect: connection refused") +} From 23aa907644e866ceb80961cd4b28f88a8fd983e7 Mon Sep 17 00:00:00 2001 From: Adrian Macneil Date: Fri, 7 Aug 2020 13:45:35 -0700 Subject: [PATCH 04/55] Add Homebrew release action (#147) --- .github/workflows/release.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..f544654 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,16 @@ +name: Release + +on: + push: + tags: 'v*' + +jobs: + homebrew: + name: Bump Homebrew formula + runs-on: ubuntu-latest + steps: + - uses: mislav/bump-homebrew-formula-action@v1 + with: + formula-name: dbmate + env: + COMMITTER_TOKEN: ${{ secrets.RELEASE_TOKEN }} From 0775179987987dd70ae1e952c1360ebf70c1adc0 Mon Sep 17 00:00:00 2001 From: Adrian Macneil Date: Fri, 7 Aug 2020 18:09:03 -0700 Subject: [PATCH 05/55] Update to urfave/cli v2 (#148) --- go.mod | 2 +- go.sum | 4 +-- main.go | 74 ++++++++++++++++++++++++++++------------------------ main_test.go | 4 +-- 4 files changed, 45 insertions(+), 39 deletions(-) diff --git a/go.mod b/go.mod index a67ca27..fb1d76f 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,6 @@ require ( github.com/lib/pq v1.5.2 github.com/mattn/go-sqlite3 v1.13.0 github.com/stretchr/testify v1.4.0 - github.com/urfave/cli v1.22.4 + github.com/urfave/cli/v2 v2.2.0 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect ) diff --git a/go.sum b/go.sum index 19df7c5..d5bcc0c 100644 --- a/go.sum +++ b/go.sum @@ -42,8 +42,8 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/urfave/cli v1.22.4 h1:u7tSpNPPswAFymm8IehJhy4uJMlUuU/GmqSkvJ1InXA= -github.com/urfave/cli v1.22.4/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/urfave/cli/v2 v2.2.0 h1:JTTnM6wKzdA0Jqodd966MVj4vWbbquZykeX1sKbe2C4= +github.com/urfave/cli/v2 v2.2.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/main.go b/main.go index 99909f7..39f4357 100644 --- a/main.go +++ b/main.go @@ -8,7 +8,7 @@ import ( "regexp" "github.com/joho/godotenv" - "github.com/urfave/cli" + "github.com/urfave/cli/v2" "github.com/amacneil/dbmate/pkg/dbmate" ) @@ -34,37 +34,40 @@ func NewApp() *cli.App { app.Version = dbmate.Version app.Flags = []cli.Flag{ - cli.StringFlag{ - Name: "env, e", - Value: "DATABASE_URL", - Usage: "specify an environment variable containing the database URL", + &cli.StringFlag{ + Name: "env", + Aliases: []string{"e"}, + Value: "DATABASE_URL", + Usage: "specify an environment variable containing the database URL", }, - cli.StringFlag{ - Name: "migrations-dir, d", - Value: dbmate.DefaultMigrationsDir, - Usage: "specify the directory containing migration files", + &cli.StringFlag{ + Name: "migrations-dir", + Aliases: []string{"d"}, + Value: dbmate.DefaultMigrationsDir, + Usage: "specify the directory containing migration files", }, - cli.StringFlag{ - Name: "schema-file, s", - Value: dbmate.DefaultSchemaFile, - Usage: "specify the schema file location", + &cli.StringFlag{ + Name: "schema-file", + Aliases: []string{"s"}, + Value: dbmate.DefaultSchemaFile, + Usage: "specify the schema file location", }, - cli.BoolFlag{ + &cli.BoolFlag{ Name: "no-dump-schema", Usage: "don't update the schema file on migrate/rollback", }, - cli.BoolFlag{ + &cli.BoolFlag{ Name: "wait", Usage: "wait for the db to become available before executing the subsequent command", }, - cli.DurationFlag{ + &cli.DurationFlag{ Name: "wait-timeout", Usage: "timeout for --wait flag", Value: dbmate.DefaultWaitTimeout, }, } - app.Commands = []cli.Command{ + app.Commands = []*cli.Command{ { Name: "new", Aliases: []string{"n"}, @@ -78,9 +81,10 @@ func NewApp() *cli.App { Name: "up", Usage: "Create database (if necessary) and migrate to the latest version", Flags: []cli.Flag{ - cli.BoolFlag{ - Name: "verbose, v", - Usage: "print the result of each statement execution", + &cli.BoolFlag{ + Name: "verbose", + Aliases: []string{"v"}, + Usage: "print the result of each statement execution", }, }, Action: action(func(db *dbmate.DB, c *cli.Context) error { @@ -106,9 +110,10 @@ func NewApp() *cli.App { Name: "migrate", Usage: "Migrate to the latest version", Flags: []cli.Flag{ - cli.BoolFlag{ - Name: "verbose, v", - Usage: "print the result of each statement execution", + &cli.BoolFlag{ + Name: "verbose", + Aliases: []string{"v"}, + Usage: "print the result of each statement execution", }, }, Action: action(func(db *dbmate.DB, c *cli.Context) error { @@ -121,9 +126,10 @@ func NewApp() *cli.App { Aliases: []string{"down"}, Usage: "Rollback the most recent migration", Flags: []cli.Flag{ - cli.BoolFlag{ - Name: "verbose, v", - Usage: "print the result of each statement execution", + &cli.BoolFlag{ + Name: "verbose", + Aliases: []string{"v"}, + Usage: "print the result of each statement execution", }, }, Action: action(func(db *dbmate.DB, c *cli.Context) error { @@ -135,11 +141,11 @@ func NewApp() *cli.App { Name: "status", Usage: "List applied and pending migrations", Flags: []cli.Flag{ - cli.BoolFlag{ + &cli.BoolFlag{ Name: "exit-code", Usage: "return 1 if there are pending migrations", }, - cli.BoolFlag{ + &cli.BoolFlag{ Name: "quiet", Usage: "don't output any text (implies --exit-code)", }, @@ -201,11 +207,11 @@ func action(f func(*dbmate.DB, *cli.Context) error) cli.ActionFunc { return err } db := dbmate.New(u) - db.AutoDumpSchema = !c.GlobalBool("no-dump-schema") - db.MigrationsDir = c.GlobalString("migrations-dir") - db.SchemaFile = c.GlobalString("schema-file") - db.WaitBefore = c.GlobalBool("wait") - overrideTimeout := c.GlobalDuration("wait-timeout") + db.AutoDumpSchema = !c.Bool("no-dump-schema") + db.MigrationsDir = c.String("migrations-dir") + db.SchemaFile = c.String("schema-file") + db.WaitBefore = c.Bool("wait") + overrideTimeout := c.Duration("wait-timeout") if overrideTimeout != 0 { db.WaitTimeout = overrideTimeout } @@ -216,7 +222,7 @@ func action(f func(*dbmate.DB, *cli.Context) error) cli.ActionFunc { // getDatabaseURL returns the current environment database url func getDatabaseURL(c *cli.Context) (u *url.URL, err error) { - env := c.GlobalString("env") + env := c.String("env") value := os.Getenv(env) return url.Parse(value) diff --git a/main_test.go b/main_test.go index d6922db..2aa3ab3 100644 --- a/main_test.go +++ b/main_test.go @@ -7,7 +7,7 @@ import ( "testing" "github.com/stretchr/testify/require" - "github.com/urfave/cli" + "github.com/urfave/cli/v2" ) func testContext(t *testing.T, u *url.URL) *cli.Context { @@ -17,7 +17,7 @@ func testContext(t *testing.T, u *url.URL) *cli.Context { app := NewApp() flagset := flag.NewFlagSet(app.Name, flag.ContinueOnError) for _, f := range app.Flags { - f.Apply(flagset) + _ = f.Apply(flagset) } return cli.NewContext(app, flagset, nil) From d1b3334ff75934d0bf703114189a786298055c12 Mon Sep 17 00:00:00 2001 From: Adrian Macneil Date: Fri, 7 Aug 2020 18:45:32 -0700 Subject: [PATCH 06/55] Update dependencies (#149) --- Dockerfile | 2 +- go.mod | 4 ++-- go.sum | 17 +++++++++++++---- pkg/dbmate/sqlite_test.go | 2 +- 4 files changed, 17 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index 227c400..13b6716 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,7 +19,7 @@ RUN apt-get update \ # golangci-lint RUN curl -fsSL -o /tmp/lint-install.sh https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh \ && chmod +x /tmp/lint-install.sh \ - && /tmp/lint-install.sh -b /usr/local/bin v1.27.0 \ + && /tmp/lint-install.sh -b /usr/local/bin v1.30.0 \ && rm -f /tmp/lint-install.sh # download modules diff --git a/go.mod b/go.mod index fb1d76f..5a1b74c 100644 --- a/go.mod +++ b/go.mod @@ -10,8 +10,8 @@ require ( github.com/joho/godotenv v1.3.0 github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d github.com/kr/pretty v0.1.0 // indirect - github.com/lib/pq v1.5.2 - github.com/mattn/go-sqlite3 v1.13.0 + github.com/lib/pq v1.8.0 + github.com/mattn/go-sqlite3 v1.14.0 github.com/stretchr/testify v1.4.0 github.com/urfave/cli/v2 v2.2.0 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect diff --git a/go.sum b/go.sum index d5bcc0c..80b1521 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/ClickHouse/clickhouse-go v1.4.1 h1:D9cihLg76O1ZyILLaXq1eksYzEuV010NdvucgKGGK14= github.com/ClickHouse/clickhouse-go v1.4.1/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI= +github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= +github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= github.com/bkaradzic/go-lz4 v1.0.0 h1:RXc4wYsyz985CkXXeX04y4VnZFGG8Rd43pRaHsOXAKk= github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4= github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 h1:F1EaeKL/ta07PY/k9Os/UFtwERei2/XzGemhpGnBKNg= @@ -25,11 +27,11 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.5.2 h1:yTSXVswvWUOQ3k1sd7vJfDrbSl8lKuscqFJRqjC0ifw= -github.com/lib/pq v1.5.2/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.8.0 h1:9xohqzkUwzR4Ga4ivdTcawVS89YSDVxXMa3xJX3cGzg= +github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= -github.com/mattn/go-sqlite3 v1.13.0 h1:LnJI81JidiW9r7pS/hXe6cFeO5EXNq7KbfvoJLRI69c= -github.com/mattn/go-sqlite3 v1.13.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/go-sqlite3 v1.14.0 h1:mLyGNKR8+Vv9CAU7PphKa2hkEqxxhn8i32J6FPj1/QA= +github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus= github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -44,6 +46,13 @@ github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJy github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/urfave/cli/v2 v2.2.0 h1:JTTnM6wKzdA0Jqodd966MVj4vWbbquZykeX1sKbe2C4= github.com/urfave/cli/v2 v2.2.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/pkg/dbmate/sqlite_test.go b/pkg/dbmate/sqlite_test.go index fa90d11..eb2adcd 100644 --- a/pkg/dbmate/sqlite_test.go +++ b/pkg/dbmate/sqlite_test.go @@ -244,5 +244,5 @@ func TestSQLitePing(t *testing.T) { // ping database should fail err = drv.Ping(u) - require.EqualError(t, err, "unable to open database file") + require.EqualError(t, err, "unable to open database file: is a directory") } From a9aaaad1fba65024010a0b61b1715bd37a472140 Mon Sep 17 00:00:00 2001 From: Adrian Macneil Date: Sat, 8 Aug 2020 11:19:33 -0700 Subject: [PATCH 07/55] Add --url flag (#150) --- README.md | 9 +++++++++ main.go | 16 +++++++++++++--- main_test.go | 35 +++++++++++++++++++---------------- 3 files changed, 41 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index b536350..14889c9 100644 --- a/README.md +++ b/README.md @@ -312,6 +312,7 @@ Please note that the `wait` command does not verify whether your specified datab The following command line options are available with all commands. You must use command line arguments in the order `dbmate [global options] command [command options]`. +* `--url, -u "protocol://host:port/dbname"` - specify the database url directly. * `--env, -e "DATABASE_URL"` - specify an environment variable to read the database connection URL from. * `--migrations-dir, -d "./db/migrations"` - where to keep the migration files. * `--schema-file, -s "./db/schema.sql"` - a path to keep the schema.sql file. @@ -335,6 +336,14 @@ Creating: myapp_test Applying: 20151127184807_create_users_table.sql ``` +Alternatively, you can specify the url directly on the command line: + +```sh +$ dbmate -u "postgres://postgres@127.0.0.1:5432/myapp_test?sslmode=disable" up +``` + +The only advantage of using `dbmate -e TEST_DATABASE_URL` over `dbmate -u $TEST_DATABASE_URL` is that the former takes advantage of dbmate's automatic `.env` file loading. + ## FAQ **How do I use dbmate under Alpine linux?** diff --git a/main.go b/main.go index 39f4357..d05a12a 100644 --- a/main.go +++ b/main.go @@ -34,6 +34,11 @@ func NewApp() *cli.App { app.Version = dbmate.Version app.Flags = []cli.Flag{ + &cli.StringFlag{ + Name: "url", + Aliases: []string{"u"}, + Usage: "specify the database URL", + }, &cli.StringFlag{ Name: "env", Aliases: []string{"e"}, @@ -220,10 +225,15 @@ func action(f func(*dbmate.DB, *cli.Context) error) cli.ActionFunc { } } -// getDatabaseURL returns the current environment database url +// getDatabaseURL returns the current database url from cli flag or environment variable func getDatabaseURL(c *cli.Context) (u *url.URL, err error) { - env := c.String("env") - value := os.Getenv(env) + // check --url flag first + value := c.String("url") + if value == "" { + // if empty, default to --env or DATABASE_URL + env := c.String("env") + value = os.Getenv(env) + } return url.Parse(value) } diff --git a/main_test.go b/main_test.go index 2aa3ab3..df1cab5 100644 --- a/main_test.go +++ b/main_test.go @@ -2,7 +2,6 @@ package main import ( "flag" - "net/url" "os" "testing" @@ -10,30 +9,34 @@ import ( "github.com/urfave/cli/v2" ) -func testContext(t *testing.T, u *url.URL) *cli.Context { - err := os.Setenv("DATABASE_URL", u.String()) - require.NoError(t, err) +func TestGetDatabaseUrl(t *testing.T) { + // set environment variables + require.NoError(t, os.Setenv("DATABASE_URL", "foo://example.org/one")) + require.NoError(t, os.Setenv("CUSTOM_URL", "foo://example.org/two")) app := NewApp() flagset := flag.NewFlagSet(app.Name, flag.ContinueOnError) for _, f := range app.Flags { - _ = f.Apply(flagset) + require.NoError(t, f.Apply(flagset)) } + ctx := cli.NewContext(app, flagset, nil) - return cli.NewContext(app, flagset, nil) -} - -func TestGetDatabaseUrl(t *testing.T) { - envURL, err := url.Parse("foo://example.org/db") - require.NoError(t, err) - ctx := testContext(t, envURL) - + // no flags defaults to DATABASE_URL u, err := getDatabaseURL(ctx) require.NoError(t, err) + require.Equal(t, "foo://example.org/one", u.String()) - require.Equal(t, "foo", u.Scheme) - require.Equal(t, "example.org", u.Host) - require.Equal(t, "/db", u.Path) + // --env overwrites DATABASE_URL + require.NoError(t, ctx.Set("env", "CUSTOM_URL")) + u, err = getDatabaseURL(ctx) + require.NoError(t, err) + require.Equal(t, "foo://example.org/two", u.String()) + + // --url takes precedence over preceding two options + require.NoError(t, ctx.Set("url", "foo://example.org/three")) + u, err = getDatabaseURL(ctx) + require.NoError(t, err) + require.Equal(t, "foo://example.org/three", u.String()) } func TestRedactLogString(t *testing.T) { From 9e2d1b8c3bdcd1de41475e1397f56b7fb28762eb Mon Sep 17 00:00:00 2001 From: Adrian Macneil Date: Sat, 8 Aug 2020 11:50:45 -0700 Subject: [PATCH 08/55] Allow setting flags via environment variables (#152) --- README.md | 15 ++++++++------- main.go | 22 +++++++++++++++------- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 14889c9..178b85b 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ For a comparison between dbmate and other popular database schema migration tool ## Installation -**OSX** +**macOS** Install using Homebrew: @@ -310,14 +310,15 @@ Please note that the `wait` command does not verify whether your specified datab ### Options -The following command line options are available with all commands. You must use command line arguments in the order `dbmate [global options] command [command options]`. +The following command line options are available with all commands. You must use command line arguments in the order `dbmate [global options] command [command options]`. Most options can also be configured via environment variables (and loaded from your `.env` file, which is helpful to share configuration between team members). -* `--url, -u "protocol://host:port/dbname"` - specify the database url directly. +* `--url, -u "protocol://host:port/dbname"` - specify the database url directly. _(env: `$DATABASE_URL`)_ * `--env, -e "DATABASE_URL"` - specify an environment variable to read the database connection URL from. -* `--migrations-dir, -d "./db/migrations"` - where to keep the migration files. -* `--schema-file, -s "./db/schema.sql"` - a path to keep the schema.sql file. -* `--no-dump-schema` - don't auto-update the schema.sql file on migrate/rollback -* `--wait` - wait for the db to become available before executing the subsequent command +* `--migrations-dir, -d "./db/migrations"` - where to keep the migration files. _(env: `$DBMATE_MIGRATIONS_DIR`)_ +* `--schema-file, -s "./db/schema.sql"` - a path to keep the schema.sql file. _(env: `$DBMATE_SCHEMA_FILE`)_ +* `--no-dump-schema` - don't auto-update the schema.sql file on migrate/rollback _(env: `$DBMATE_NO_DUMP_SCHEMA`)_ +* `--wait` - wait for the db to become available before executing the subsequent command _(env: `$DBMATE_WAIT`)_ +* `--wait-timeout 60s` - timeout for --wait flag _(env: `$DBMATE_WAIT_TIMEOUT`)_ For example, before running your test suite, you may wish to drop and recreate the test database. One easy way to do this is to store your test database connection URL in the `TEST_DATABASE_URL` environment variable: diff --git a/main.go b/main.go index d05a12a..749bb69 100644 --- a/main.go +++ b/main.go @@ -48,27 +48,32 @@ func NewApp() *cli.App { &cli.StringFlag{ Name: "migrations-dir", Aliases: []string{"d"}, + EnvVars: []string{"DBMATE_MIGRATIONS_DIR"}, Value: dbmate.DefaultMigrationsDir, Usage: "specify the directory containing migration files", }, &cli.StringFlag{ Name: "schema-file", Aliases: []string{"s"}, + EnvVars: []string{"DBMATE_SCHEMA_FILE"}, Value: dbmate.DefaultSchemaFile, Usage: "specify the schema file location", }, &cli.BoolFlag{ - Name: "no-dump-schema", - Usage: "don't update the schema file on migrate/rollback", + Name: "no-dump-schema", + EnvVars: []string{"DBMATE_NO_DUMP_SCHEMA"}, + Usage: "don't update the schema file on migrate/rollback", }, &cli.BoolFlag{ - Name: "wait", - Usage: "wait for the db to become available before executing the subsequent command", + Name: "wait", + EnvVars: []string{"DBMATE_WAIT"}, + Usage: "wait for the db to become available before executing the subsequent command", }, &cli.DurationFlag{ - Name: "wait-timeout", - Usage: "timeout for --wait flag", - Value: dbmate.DefaultWaitTimeout, + Name: "wait-timeout", + EnvVars: []string{"DBMATE_WAIT_TIMEOUT"}, + Usage: "timeout for --wait flag", + Value: dbmate.DefaultWaitTimeout, }, } @@ -89,6 +94,7 @@ func NewApp() *cli.App { &cli.BoolFlag{ Name: "verbose", Aliases: []string{"v"}, + EnvVars: []string{"DBMATE_VERBOSE"}, Usage: "print the result of each statement execution", }, }, @@ -118,6 +124,7 @@ func NewApp() *cli.App { &cli.BoolFlag{ Name: "verbose", Aliases: []string{"v"}, + EnvVars: []string{"DBMATE_VERBOSE"}, Usage: "print the result of each statement execution", }, }, @@ -134,6 +141,7 @@ func NewApp() *cli.App { &cli.BoolFlag{ Name: "verbose", Aliases: []string{"v"}, + EnvVars: []string{"DBMATE_VERBOSE"}, Usage: "print the result of each statement execution", }, }, From b41a73d70cd6e799f1d607668d7cb17d72a35567 Mon Sep 17 00:00:00 2001 From: Adrian Macneil Date: Sat, 8 Aug 2020 11:58:25 -0700 Subject: [PATCH 09/55] Ignore db directory (#151) --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 74f1fdb..c290c8c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ .DS_Store .env +/db /dbmate /dist /testdata/db/schema.sql From 24b3fccacdfbc73d99f608deb87d026621eae198 Mon Sep 17 00:00:00 2001 From: Adrian Macneil Date: Sat, 8 Aug 2020 11:58:46 -0700 Subject: [PATCH 10/55] Update RELEASING.md (#153) --- RELEASING.md | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/RELEASING.md b/RELEASING.md index d526337..feb13d3 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -2,11 +2,7 @@ The following steps should be followed to publish a new version of dbmate (requires write access to this repository). -1. Update [version.go](/pkg/dbmate/version.go) with new version number ([example PR](https://github.com/amacneil/dbmate/pull/79/files)) +1. Update [version.go](/pkg/dbmate/version.go) with new version number ([example PR](https://github.com/amacneil/dbmate/pull/146/files)) 2. Create new release on GitHub project [releases page](https://github.com/amacneil/dbmate/releases) 3. Travis CI will automatically publish release binaries to GitHub -4. Create PR to update Homebrew package by running the following command: - -``` -$ brew bump-formula-pr --url=https://github.com/amacneil/dbmate/archive/vX.Y.Z.tar.gz dbmate -``` +4. GitHub Actions will automatically create PR to update Homebrew package From 073efc996e2e759f90cae5799f7728aa579dbe1c Mon Sep 17 00:00:00 2001 From: Adrian Macneil Date: Sat, 8 Aug 2020 12:30:22 -0700 Subject: [PATCH 11/55] Update README.md (#154) --- README.md | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 178b85b..2ec04d5 100644 --- a/README.md +++ b/README.md @@ -61,9 +61,9 @@ To use dbmate on Heroku, the easiest method is to store the linux binary in your ```sh $ mkdir -p bin -$ curl -fsSL -o bin/dbmate-heroku https://github.com/amacneil/dbmate/releases/latest/download/dbmate-linux-amd64 -$ chmod +x bin/dbmate-heroku -$ git add bin/dbmate-heroku +$ curl -fsSL -o bin/dbmate https://github.com/amacneil/dbmate/releases/latest/download/dbmate-linux-amd64 +$ chmod +x bin/dbmate +$ git add bin/dbmate $ git commit -m "Add dbmate binary" $ git push heroku master ``` @@ -71,15 +71,7 @@ $ git push heroku master You can then run dbmate on heroku: ```sh -$ heroku run bin/dbmate-heroku up -``` - -**Other** - -Dbmate can be installed directly using `go get`: - -```sh -$ GO111MODULE=on go get -u github.com/amacneil/dbmate +$ heroku run bin/dbmate up ``` ## Commands From df461ff6c7ba3e3f482cf56b87ee7d80e0599ef8 Mon Sep 17 00:00:00 2001 From: Adrian Macneil Date: Sat, 8 Aug 2020 16:25:13 -0700 Subject: [PATCH 12/55] Add linux/arm64 build (#155) --- .travis.yml | 4 ++-- Dockerfile | 2 +- Makefile | 24 ++++++++++++------------ 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.travis.yml b/.travis.yml index 13ffa0c..0876b02 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,10 +9,10 @@ install: - docker-compose build - docker-compose up -d script: - - docker-compose run --rm --volume "$PWD/dist:/src/dist" dbmate make build + - docker-compose run --rm --volume "$PWD/dist:/src/dist" dbmate make build-all + - docker-compose run --rm dbmate make lint - docker-compose run --rm dbmate make wait - docker-compose run --rm dbmate make test - - docker-compose run --rm dbmate make lint - docker build -t dbmate . - docker run --rm dbmate --help deploy: diff --git a/Dockerfile b/Dockerfile index 13b6716..302accb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,7 +28,7 @@ RUN go mod download # build COPY . ./ -RUN make build-linux +RUN make build # runtime image FROM gcr.io/distroless/base diff --git a/Makefile b/Makefile index 01c2372..91cc426 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ LDFLAGS := -ldflags '-s' .PHONY: all -all: test lint build +all: build lint test .PHONY: test test: @@ -26,28 +26,28 @@ clean: rm -rf dist/* .PHONY: build -build: clean build-linux build-macos build-windows +build: clean build-linux-amd64 ls -lh dist -.PHONY: build-linux -build-linux: +.PHONY: build-linux-amd64 +build-linux-amd64: GOOS=linux GOARCH=amd64 CGO_ENABLED=1 \ go build $(LDFLAGS) -o dist/dbmate-linux-amd64 . + +.PHONY: build-all +build-all: clean build-linux-amd64 GOOS=linux GOARCH=amd64 CGO_ENABLED=0 \ go build $(LDFLAGS) -o dist/dbmate-linux-musl-amd64 . - -.PHONY: build-macos -build-macos: + GOOS=linux GOARCH=arm64 CGO_ENABLED=1 CC=aarch64-linux-gnu-gcc-5 CXX=aarch64-linux-gnu-g++-5 \ + go build $(LDFLAGS) -o dist/dbmate-linux-arm64 . GOOS=darwin GOARCH=amd64 CGO_ENABLED=1 CC=o64-clang CXX=o64-clang++ \ go build $(LDFLAGS) -o dist/dbmate-macos-amd64 . - -.PHONY: build-windows -build-windows: GOOS=windows GOARCH=amd64 CGO_ENABLED=1 CC=x86_64-w64-mingw32-gcc-posix CXX=x86_64-w64-mingw32-g++-posix \ go build $(LDFLAGS) -o dist/dbmate-windows-amd64.exe . + ls -lh dist -.PHONY: docker-all -docker-all: +.PHONY: docker-make +docker-make: docker-compose build docker-compose run --rm dbmate make From 4581acafada56e7555103f51ca407ad08e83dfb9 Mon Sep 17 00:00:00 2001 From: Adrian Macneil Date: Tue, 11 Aug 2020 13:05:55 -0700 Subject: [PATCH 13/55] Build statically linked binaries (#156) Prior to this commit, we released two linux binaries: * `dbmate-linux-amd64` (built with cgo, dynamically linked) * `dbmate-linux-musl-amd64` (built without cgo, statically linked, no sqlite support) The statically linked binary is desirable for alpine linux users (or anyone else using musl libc or minimal docker images). The original reason for having two separate binaries was that the easiest method to create a static binary for go is to set `CGO_ENABLED=0`, but unfortunately this also prevented us from building sqlite (which requires cgo). With this commit, all linux and windows binaries are explicitly statically linked while leaving cgo enabled. Hat tip to https://www.arp242.net/static-go.html which explained the necessary flags to enable this. As an added bonus, the `dbmate` docker image now now uses a `scratch` base rather than `gcr.io/distroless/base`, reducing the image size from 26.7 MB to 9.8 MB. --- Dockerfile | 7 ++++--- Makefile | 25 ++++++++++++++----------- README.md | 6 ------ 3 files changed, 18 insertions(+), 20 deletions(-) diff --git a/Dockerfile b/Dockerfile index 302accb..35bac16 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,6 @@ # build image FROM techknowlogick/xgo:go-1.14.x as build WORKDIR /src -ENTRYPOINT [] -CMD ["/bin/bash"] # enable cgo to build sqlite ENV CGO_ENABLED 1 @@ -30,7 +28,10 @@ RUN go mod download COPY . ./ RUN make build +ENTRYPOINT [] +CMD ["/bin/bash"] + # runtime image -FROM gcr.io/distroless/base +FROM scratch COPY --from=build /src/dist/dbmate-linux-amd64 /dbmate ENTRYPOINT ["/dbmate"] diff --git a/Makefile b/Makefile index 91cc426..07a6005 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,16 @@ +# no static linking for macos LDFLAGS := -ldflags '-s' +# statically link binaries (to support alpine + scratch containers) +STATICLDFLAGS := -ldflags '-s -extldflags "-static"' +# avoid building code that is incompatible with static linking +TAGS := -tags netgo,osusergo,sqlite_omit_load_extension .PHONY: all all: build lint test .PHONY: test test: - go test -v ./... + go test -v $(TAGS) $(STATICLDFLAGS) ./... .PHONY: fix fix: @@ -31,19 +36,17 @@ build: clean build-linux-amd64 .PHONY: build-linux-amd64 build-linux-amd64: - GOOS=linux GOARCH=amd64 CGO_ENABLED=1 \ - go build $(LDFLAGS) -o dist/dbmate-linux-amd64 . + GOOS=linux GOARCH=amd64 \ + go build $(TAGS) $(STATICLDFLAGS) -o dist/dbmate-linux-amd64 . .PHONY: build-all build-all: clean build-linux-amd64 - GOOS=linux GOARCH=amd64 CGO_ENABLED=0 \ - go build $(LDFLAGS) -o dist/dbmate-linux-musl-amd64 . - GOOS=linux GOARCH=arm64 CGO_ENABLED=1 CC=aarch64-linux-gnu-gcc-5 CXX=aarch64-linux-gnu-g++-5 \ - go build $(LDFLAGS) -o dist/dbmate-linux-arm64 . - GOOS=darwin GOARCH=amd64 CGO_ENABLED=1 CC=o64-clang CXX=o64-clang++ \ - go build $(LDFLAGS) -o dist/dbmate-macos-amd64 . - GOOS=windows GOARCH=amd64 CGO_ENABLED=1 CC=x86_64-w64-mingw32-gcc-posix CXX=x86_64-w64-mingw32-g++-posix \ - go build $(LDFLAGS) -o dist/dbmate-windows-amd64.exe . + GOOS=linux GOARCH=arm64 CC=aarch64-linux-gnu-gcc-5 CXX=aarch64-linux-gnu-g++-5 \ + go build $(TAGS) $(STATICLDFLAGS) -o dist/dbmate-linux-arm64 . + GOOS=darwin GOARCH=amd64 CC=o64-clang CXX=o64-clang++ \ + go build $(TAGS) $(LDFLAGS) -o dist/dbmate-macos-amd64 . + GOOS=windows GOARCH=amd64 CC=x86_64-w64-mingw32-gcc-posix CXX=x86_64-w64-mingw32-g++-posix \ + go build $(TAGS) $(STATICLDFLAGS) -o dist/dbmate-windows-amd64.exe . ls -lh dist .PHONY: docker-make diff --git a/README.md b/README.md index 2ec04d5..92a7396 100644 --- a/README.md +++ b/README.md @@ -337,12 +337,6 @@ $ dbmate -u "postgres://postgres@127.0.0.1:5432/myapp_test?sslmode=disable" up The only advantage of using `dbmate -e TEST_DATABASE_URL` over `dbmate -u $TEST_DATABASE_URL` is that the former takes advantage of dbmate's automatic `.env` file loading. -## FAQ - -**How do I use dbmate under Alpine linux?** - -Alpine linux uses [musl libc](https://www.musl-libc.org/), which is incompatible with how we build SQLite support (using [cgo](https://golang.org/cmd/cgo/)). If you want Alpine linux support, and don't mind sacrificing SQLite support, please use the `dbmate-linux-musl-amd64` build found on the [releases page](https://github.com/amacneil/dbmate/releases). - ## Alternatives Why another database schema migration tool? Dbmate was inspired by many other tools, primarily [Active Record Migrations](http://guides.rubyonrails.org/active_record_migrations.html), with the goals of being trivial to configure, and language & framework independent. Here is a comparison between dbmate and other popular migration tools. From 4387633e1fd1473fb29acd0f8ffe4faa750da69d Mon Sep 17 00:00:00 2001 From: Adrian Macneil Date: Tue, 11 Aug 2020 15:36:07 -0700 Subject: [PATCH 14/55] Add sql clients to docker image (#157) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Instead of a nice super-minimal production docker image, create one based on alpine with sql clients installed (`mariadb-client`, `postgresql-client`, and `sqlite`). * Increases docker image size from 10 MB to 56 MB 👎 * Allows people to run `dbmate dump` command with our docker image (fixes #114) 👍 * I'm not sure what compatibility is like between `mysqldump` from `mariadb-client` versus `mysql-client`, but starting here since mariadb is included with alpine, and the version I built using mysql and ubuntu weighed in at 165 MB. 🤔 --- Dockerfile | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 35bac16..20b2a39 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,6 +32,10 @@ ENTRYPOINT [] CMD ["/bin/bash"] # runtime image -FROM scratch -COPY --from=build /src/dist/dbmate-linux-amd64 /dbmate -ENTRYPOINT ["/dbmate"] +FROM alpine +RUN apk add --no-cache \ + mariadb-client \ + postgresql-client \ + sqlite +COPY --from=build /src/dist/dbmate-linux-amd64 /usr/local/bin/dbmate +ENTRYPOINT ["dbmate"] From 616c4fe442012ab05805340c7022d65d5155cf46 Mon Sep 17 00:00:00 2001 From: Adrian Macneil Date: Thu, 13 Aug 2020 08:30:58 -0700 Subject: [PATCH 15/55] v1.10.0 (#158) --- pkg/dbmate/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/dbmate/version.go b/pkg/dbmate/version.go index 93532b0..ffb2f9c 100644 --- a/pkg/dbmate/version.go +++ b/pkg/dbmate/version.go @@ -1,4 +1,4 @@ package dbmate // Version of dbmate -const Version = "1.9.1" +const Version = "1.10.0" From 26f17d5141fbf0bf6dc85d7d9e6a9bca6841bea7 Mon Sep 17 00:00:00 2001 From: Adrian Macneil Date: Thu, 24 Sep 2020 10:14:31 -0700 Subject: [PATCH 16/55] Document migration order (#160) --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 92a7396..34d133d 100644 --- a/README.md +++ b/README.md @@ -205,6 +205,8 @@ Writing: ./db/schema.sql > Note: `dbmate up` will create the database if it does not already exist (assuming the current user has permission to create databases). If you want to run migrations without creating the database, run `dbmate migrate`. +Pending migrations are always applied in numerical order. However, dbmate does not prevent migrations from being applied out of order if they are committed independently (for example: if a developer has been working on a branch for a long time, and commits a migration which has a lower version number than other already-applied migrations, dbmate will simply apply the pending migration). See [#159](https://github.com/amacneil/dbmate/issues/159) for a more detailed explanation. + ### Rolling Back Migrations By default, dbmate doesn't know how to roll back a migration. In development, it's often useful to be able to revert your database to a previous state. To accomplish this, implement the `migrate:down` section: From 2b3f59fbe317cf29bf35bb898d39ca7387586134 Mon Sep 17 00:00:00 2001 From: Adrian Macneil Date: Sun, 27 Sep 2020 14:50:30 -0700 Subject: [PATCH 17/55] Go v1.15 (#161) --- Dockerfile | 4 ++-- go.mod | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 20b2a39..cb10d37 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # build image -FROM techknowlogick/xgo:go-1.14.x as build +FROM techknowlogick/xgo:go-1.15.x as build WORKDIR /src # enable cgo to build sqlite @@ -17,7 +17,7 @@ RUN apt-get update \ # golangci-lint RUN curl -fsSL -o /tmp/lint-install.sh https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh \ && chmod +x /tmp/lint-install.sh \ - && /tmp/lint-install.sh -b /usr/local/bin v1.30.0 \ + && /tmp/lint-install.sh -b /usr/local/bin v1.31.0 \ && rm -f /tmp/lint-install.sh # download modules diff --git a/go.mod b/go.mod index 5a1b74c..35f4cb5 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/amacneil/dbmate -go 1.14 +go 1.15 require ( github.com/ClickHouse/clickhouse-go v1.4.1 From af41fbfb4e5566fb2c85819a8f26371eb1c6ead9 Mon Sep 17 00:00:00 2001 From: Adrian Macneil Date: Fri, 30 Oct 2020 19:04:02 +1300 Subject: [PATCH 18/55] Make golint happy (#165) --- .golangci.yml | 1 + Dockerfile | 2 +- pkg/dbmate/db_test.go | 4 ++-- pkg/dbmate/utils.go | 4 ++-- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 34cc2bf..20b9a33 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -5,6 +5,7 @@ linters: - depguard - errcheck - goimports + - golint - gosimple - govet - ineffassign diff --git a/Dockerfile b/Dockerfile index cb10d37..4109fd2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,7 +17,7 @@ RUN apt-get update \ # golangci-lint RUN curl -fsSL -o /tmp/lint-install.sh https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh \ && chmod +x /tmp/lint-install.sh \ - && /tmp/lint-install.sh -b /usr/local/bin v1.31.0 \ + && /tmp/lint-install.sh -b /usr/local/bin v1.32.0 \ && rm -f /tmp/lint-install.sh # download modules diff --git a/pkg/dbmate/db_test.go b/pkg/dbmate/db_test.go index 5eeec24..4b6a890 100644 --- a/pkg/dbmate/db_test.go +++ b/pkg/dbmate/db_test.go @@ -341,7 +341,7 @@ func TestRollback(t *testing.T) { } } -func testStatusUrl(t *testing.T, u *url.URL) { +func testStatusURL(t *testing.T, u *url.URL) { db := newTestDB(t, u) // drop, recreate, and migrate database @@ -387,6 +387,6 @@ func testStatusUrl(t *testing.T, u *url.URL) { func TestStatus(t *testing.T) { for _, u := range testURLs(t) { - testStatusUrl(t, u) + testStatusURL(t, u) } } diff --git a/pkg/dbmate/utils.go b/pkg/dbmate/utils.go index aa2a6e6..1eddb1f 100644 --- a/pkg/dbmate/utils.go +++ b/pkg/dbmate/utils.go @@ -129,9 +129,9 @@ func queryColumn(db *sql.DB, query string) ([]string, error) { } func printVerbose(result sql.Result) { - lastInsertId, err := result.LastInsertId() + lastInsertID, err := result.LastInsertId() if err == nil { - fmt.Printf("Last insert ID: %d\n", lastInsertId) + fmt.Printf("Last insert ID: %d\n", lastInsertID) } rowsAffected, err := result.RowsAffected() if err == nil { From d4ecd0b2592431a5545f2dd7c594e6f5b0c7a7af Mon Sep 17 00:00:00 2001 From: Adrian Macneil Date: Sat, 31 Oct 2020 17:13:49 +1300 Subject: [PATCH 19/55] postgres: Limit pg_dump to schemas in search_path (#166) --- pkg/dbmate/postgres.go | 32 +++++++++++--- pkg/dbmate/postgres_test.go | 84 +++++++++++++++++++++---------------- 2 files changed, 74 insertions(+), 42 deletions(-) diff --git a/pkg/dbmate/postgres.go b/pkg/dbmate/postgres.go index 084cff5..5d751cd 100644 --- a/pkg/dbmate/postgres.go +++ b/pkg/dbmate/postgres.go @@ -19,7 +19,7 @@ func init() { type PostgresDriver struct { } -func normalizePostgresURL(u *url.URL) string { +func normalizePostgresURL(u *url.URL) *url.URL { hostname := u.Hostname() port := u.Port() query := u.Query() @@ -54,12 +54,33 @@ func normalizePostgresURL(u *url.URL) string { out.Host = fmt.Sprintf("%s:%s", hostname, port) out.RawQuery = query.Encode() - return out.String() + return out +} + +func normalizePostgresURLForDump(u *url.URL) []string { + u = normalizePostgresURL(u) + + // find schemas from search_path + query := u.Query() + schemas := strings.Split(query.Get("search_path"), ",") + query.Del("search_path") + u.RawQuery = query.Encode() + + out := []string{} + for _, schema := range schemas { + schema = strings.TrimSpace(schema) + if schema != "" { + out = append(out, "--schema", schema) + } + } + out = append(out, u.String()) + + return out } // Open creates a new database connection func (drv PostgresDriver) Open(u *url.URL) (*sql.DB, error) { - return sql.Open("postgres", normalizePostgresURL(u)) + return sql.Open("postgres", normalizePostgresURL(u).String()) } func (drv PostgresDriver) openPostgresDB(u *url.URL) (*sql.DB, error) { @@ -128,8 +149,9 @@ func postgresSchemaMigrationsDump(db *sql.DB) ([]byte, error) { // DumpSchema returns the current database schema func (drv PostgresDriver) DumpSchema(u *url.URL, db *sql.DB) ([]byte, error) { // load schema - schema, err := runCommand("pg_dump", "--format=plain", "--encoding=UTF8", - "--schema-only", "--no-privileges", "--no-owner", normalizePostgresURL(u)) + args := append([]string{"--format=plain", "--encoding=UTF8", "--schema-only", + "--no-privileges", "--no-owner"}, normalizePostgresURLForDump(u)...) + schema, err := runCommand("pg_dump", args...) if err != nil { return nil, err } diff --git a/pkg/dbmate/postgres_test.go b/pkg/dbmate/postgres_test.go index 1294a3e..4e5739a 100644 --- a/pkg/dbmate/postgres_test.go +++ b/pkg/dbmate/postgres_test.go @@ -34,47 +34,57 @@ func prepTestPostgresDB(t *testing.T) *sql.DB { return db } -func TestNormalizePostgresURLDefaults(t *testing.T) { - u, err := url.Parse("postgres:///foo") - require.NoError(t, err) - s := normalizePostgresURL(u) - require.Equal(t, "postgres://localhost:5432/foo", s) +func TestNormalizePostgresURL(t *testing.T) { + cases := []struct { + input string + expected string + }{ + // defaults + {"postgres:///foo", "postgres://localhost:5432/foo"}, + // support custom url params + {"postgres://bob:secret@myhost:1234/foo?bar=baz", "postgres://bob:secret@myhost:1234/foo?bar=baz"}, + // support `host` and `port` via url params + {"postgres://bob:secret@myhost:1234/foo?host=new&port=9999", "postgres://bob:secret@:9999/foo?host=new"}, + {"postgres://bob:secret@myhost:1234/foo?port=9999&bar=baz", "postgres://bob:secret@myhost:9999/foo?bar=baz"}, + // support unix sockets via `host` or `socket` param + {"postgres://bob:secret@myhost:1234/foo?host=/var/run/postgresql", "postgres://bob:secret@:1234/foo?host=%2Fvar%2Frun%2Fpostgresql"}, + {"postgres://bob:secret@localhost/foo?socket=/var/run/postgresql", "postgres://bob:secret@:5432/foo?host=%2Fvar%2Frun%2Fpostgresql"}, + {"postgres:///foo?socket=/var/run/postgresql", "postgres://:5432/foo?host=%2Fvar%2Frun%2Fpostgresql"}, + } + + for _, c := range cases { + t.Run(c.input, func(t *testing.T) { + u, err := url.Parse(c.input) + require.NoError(t, err) + + actual := normalizePostgresURL(u).String() + require.Equal(t, c.expected, actual) + }) + } } -func TestNormalizePostgresURLCustom(t *testing.T) { - u, err := url.Parse("postgres://bob:secret@myhost:1234/foo?bar=baz") - require.NoError(t, err) - s := normalizePostgresURL(u) - require.Equal(t, "postgres://bob:secret@myhost:1234/foo?bar=baz", s) -} +func TestNormalizePostgresURLForDump(t *testing.T) { + cases := []struct { + input string + expected []string + }{ + // defaults + {"postgres:///foo", []string{"postgres://localhost:5432/foo"}}, + // support single schema + {"postgres:///foo?search_path=foo", []string{"--schema", "foo", "postgres://localhost:5432/foo"}}, + // support multiple schemas + {"postgres:///foo?search_path=foo,public", []string{"--schema", "foo", "--schema", "public", "postgres://localhost:5432/foo"}}, + } -func TestNormalizePostgresURLHostPortParams(t *testing.T) { - u, err := url.Parse("postgres://bob:secret@myhost:1234/foo?port=9999&bar=baz") - require.NoError(t, err) - s := normalizePostgresURL(u) - require.Equal(t, "postgres://bob:secret@myhost:9999/foo?bar=baz", s) + for _, c := range cases { + t.Run(c.input, func(t *testing.T) { + u, err := url.Parse(c.input) + require.NoError(t, err) - u, err = url.Parse("postgres://bob:secret@myhost:1234/foo?host=new&port=9999") - require.NoError(t, err) - s = normalizePostgresURL(u) - require.Equal(t, "postgres://bob:secret@:9999/foo?host=new", s) - - u, err = url.Parse("postgres://bob:secret@myhost:1234/foo?host=/var/run/postgresql") - require.NoError(t, err) - s = normalizePostgresURL(u) - require.Equal(t, "postgres://bob:secret@:1234/foo?host=%2Fvar%2Frun%2Fpostgresql", s) -} - -func TestNormalizePostgresURLSocketParam(t *testing.T) { - u, err := url.Parse("postgres://bob:secret@localhost/foo?socket=/var/run/postgresql") - require.NoError(t, err) - s := normalizePostgresURL(u) - require.Equal(t, "postgres://bob:secret@:5432/foo?host=%2Fvar%2Frun%2Fpostgresql", s) - - u, err = url.Parse("postgres:///foo?socket=/var/run/postgresql") - require.NoError(t, err) - s = normalizePostgresURL(u) - require.Equal(t, "postgres://:5432/foo?host=%2Fvar%2Frun%2Fpostgresql", s) + actual := normalizePostgresURLForDump(u) + require.Equal(t, c.expected, actual) + }) + } } func TestPostgresCreateDropDatabase(t *testing.T) { From 55a8065efe03f0e0d68cd13712a0eacca58cd2fb Mon Sep 17 00:00:00 2001 From: Adrian Macneil Date: Sun, 1 Nov 2020 13:30:35 +1300 Subject: [PATCH 20/55] postgres: Support custom schema for schema_migrations table (#167) Instead of hardcoding `schema_migrations` table to the `public` schema, add support for specifying a schema via the `search_path` URL parameter. **Backwards compatibility note**: If anyone was using the previously undocumented `search_path` behavior (affecting migrations themselves, but always storing the `schema_migrations` table in `public`), you will need to either prepend `public` to your `search_path`, or migrate your `schema_migrations` table to your primary schema: ```sql ALTER TABLE public.schema_migrations SET SCHEMA myschema; ``` Closes #110 --- README.md | 11 ++++++ pkg/dbmate/driver.go | 2 ++ pkg/dbmate/postgres.go | 64 +++++++++++++++++++++++++++------- pkg/dbmate/postgres_test.go | 69 +++++++++++++++++++++++++++++++++---- pkg/dbmate/utils.go | 15 +++++++- 5 files changed, 141 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 34d133d..cbf07ed 100644 --- a/README.md +++ b/README.md @@ -139,6 +139,17 @@ A `socket` or `host` parameter can be specified to connect through a unix socket DATABASE_URL="postgres://username:password@/database_name?socket=/var/run/postgresql" ``` +A `search_path` parameter can be used to specify the [current schema](https://www.postgresql.org/docs/13/ddl-schemas.html#DDL-SCHEMAS-PATH) while applying migrations, as well as for dbmate's `schema_migrations` table. +If multiple comma-separated schemas are passed, the first will be used for the `schema_migrations` table. + +```sh +DATABASE_URL="postgres://username:password@127.0.0.1:5432/database_name?search_path=myschema" +``` + +```sh +DATABASE_URL="postgres://username:password@127.0.0.1:5432/database_name?search_path=myschema,public" +``` + **SQLite** SQLite databases are stored on the filesystem, so you do not need to specify a host. By default, files are relative to the current directory. For example, the following will create a database at `./db/database_name.sqlite3`: diff --git a/pkg/dbmate/driver.go b/pkg/dbmate/driver.go index ddfff1b..f640e21 100644 --- a/pkg/dbmate/driver.go +++ b/pkg/dbmate/driver.go @@ -30,6 +30,8 @@ func RegisterDriver(drv Driver, scheme string) { // Transaction can represent a database or open transaction type Transaction interface { Exec(query string, args ...interface{}) (sql.Result, error) + Query(query string, args ...interface{}) (*sql.Rows, error) + QueryRow(query string, args ...interface{}) *sql.Row } // GetDriver loads a database driver by name diff --git a/pkg/dbmate/postgres.go b/pkg/dbmate/postgres.go index 5d751cd..3ac56ba 100644 --- a/pkg/dbmate/postgres.go +++ b/pkg/dbmate/postgres.go @@ -125,20 +125,25 @@ func (drv PostgresDriver) DropDatabase(u *url.URL) error { return err } -func postgresSchemaMigrationsDump(db *sql.DB) ([]byte, error) { - // load applied migrations - migrations, err := queryColumn(db, - "select quote_literal(version) from public.schema_migrations order by version asc") +func (drv PostgresDriver) postgresSchemaMigrationsDump(db *sql.DB) ([]byte, error) { + migrationsTable, err := drv.migrationsTableName(db) if err != nil { return nil, err } - // build schema_migrations table data + // load applied migrations + migrations, err := queryColumn(db, + "select quote_literal(version) from "+migrationsTable+" order by version asc") + if err != nil { + return nil, err + } + + // build migrations table data var buf bytes.Buffer buf.WriteString("\n--\n-- Dbmate schema migrations\n--\n\n") if len(migrations) > 0 { - buf.WriteString("INSERT INTO public.schema_migrations (version) VALUES\n (" + + buf.WriteString("INSERT INTO " + migrationsTable + " (version) VALUES\n (" + strings.Join(migrations, "),\n (") + ");\n") } @@ -156,7 +161,7 @@ func (drv PostgresDriver) DumpSchema(u *url.URL, db *sql.DB) ([]byte, error) { return nil, err } - migrations, err := postgresSchemaMigrationsDump(db) + migrations, err := drv.postgresSchemaMigrationsDump(db) if err != nil { return nil, err } @@ -187,8 +192,13 @@ func (drv PostgresDriver) DatabaseExists(u *url.URL) (bool, error) { // CreateMigrationsTable creates the schema_migrations table func (drv PostgresDriver) CreateMigrationsTable(db *sql.DB) error { - _, err := db.Exec("create table if not exists public.schema_migrations " + - "(version varchar(255) primary key)") + migrationsTable, err := drv.migrationsTableName(db) + if err != nil { + return err + } + + _, err = db.Exec("create table if not exists " + migrationsTable + + " (version varchar(255) primary key)") return err } @@ -196,7 +206,12 @@ func (drv PostgresDriver) CreateMigrationsTable(db *sql.DB) error { // SelectMigrations returns a list of applied migrations // with an optional limit (in descending order) func (drv PostgresDriver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, error) { - query := "select version from public.schema_migrations order by version desc" + migrationsTable, err := drv.migrationsTableName(db) + if err != nil { + return nil, err + } + + query := "select version from " + migrationsTable + " order by version desc" if limit >= 0 { query = fmt.Sprintf("%s limit %d", query, limit) } @@ -222,14 +237,24 @@ func (drv PostgresDriver) SelectMigrations(db *sql.DB, limit int) (map[string]bo // InsertMigration adds a new migration record func (drv PostgresDriver) InsertMigration(db Transaction, version string) error { - _, err := db.Exec("insert into public.schema_migrations (version) values ($1)", version) + migrationsTable, err := drv.migrationsTableName(db) + if err != nil { + return err + } + + _, err = db.Exec("insert into "+migrationsTable+" (version) values ($1)", version) return err } // DeleteMigration removes a migration record func (drv PostgresDriver) DeleteMigration(db Transaction, version string) error { - _, err := db.Exec("delete from public.schema_migrations where version = $1", version) + migrationsTable, err := drv.migrationsTableName(db) + if err != nil { + return err + } + + _, err = db.Exec("delete from "+migrationsTable+" where version = $1", version) return err } @@ -259,3 +284,18 @@ func (drv PostgresDriver) Ping(u *url.URL) error { return err } + +func (drv PostgresDriver) migrationsTableName(db Transaction) (string, error) { + // get current schema + schema, err := queryRow(db, "select quote_ident(current_schema())") + if err != nil { + return "", err + } + + // if the search path is empty, or does not contain a valid schema, default to public + if schema == "" { + schema = "public" + } + + return schema + ".schema_migrations", nil +} diff --git a/pkg/dbmate/postgres_test.go b/pkg/dbmate/postgres_test.go index 4e5739a..b283698 100644 --- a/pkg/dbmate/postgres_test.go +++ b/pkg/dbmate/postgres_test.go @@ -15,9 +15,8 @@ func postgresTestURL(t *testing.T) *url.URL { return u } -func prepTestPostgresDB(t *testing.T) *sql.DB { +func prepTestPostgresDB(t *testing.T, u *url.URL) *sql.DB { drv := PostgresDriver{} - u := postgresTestURL(t) // drop any existing database err := drv.DropDatabase(u) @@ -130,7 +129,7 @@ func TestPostgresDumpSchema(t *testing.T) { u := postgresTestURL(t) // prepare database - db := prepTestPostgresDB(t) + db := prepTestPostgresDB(t, u) defer mustClose(db) err := drv.CreateMigrationsTable(db) require.NoError(t, err) @@ -198,7 +197,8 @@ func TestPostgresDatabaseExists_Error(t *testing.T) { func TestPostgresCreateMigrationsTable(t *testing.T) { drv := PostgresDriver{} - db := prepTestPostgresDB(t) + u := postgresTestURL(t) + db := prepTestPostgresDB(t, u) defer mustClose(db) // migrations table should not exist @@ -221,7 +221,8 @@ func TestPostgresCreateMigrationsTable(t *testing.T) { func TestPostgresSelectMigrations(t *testing.T) { drv := PostgresDriver{} - db := prepTestPostgresDB(t) + u := postgresTestURL(t) + db := prepTestPostgresDB(t, u) defer mustClose(db) err := drv.CreateMigrationsTable(db) @@ -247,7 +248,8 @@ func TestPostgresSelectMigrations(t *testing.T) { func TestPostgresInsertMigration(t *testing.T) { drv := PostgresDriver{} - db := prepTestPostgresDB(t) + u := postgresTestURL(t) + db := prepTestPostgresDB(t, u) defer mustClose(db) err := drv.CreateMigrationsTable(db) @@ -270,7 +272,8 @@ func TestPostgresInsertMigration(t *testing.T) { func TestPostgresDeleteMigration(t *testing.T) { drv := PostgresDriver{} - db := prepTestPostgresDB(t) + u := postgresTestURL(t) + db := prepTestPostgresDB(t, u) defer mustClose(db) err := drv.CreateMigrationsTable(db) @@ -307,3 +310,55 @@ func TestPostgresPing(t *testing.T) { require.Error(t, err) require.Contains(t, err.Error(), "connect: connection refused") } + +func TestMigrationsTableName(t *testing.T) { + drv := PostgresDriver{} + + t.Run("default schema", func(t *testing.T) { + u := postgresTestURL(t) + db := prepTestPostgresDB(t, u) + defer mustClose(db) + + name, err := drv.migrationsTableName(db) + require.NoError(t, err) + require.Equal(t, "public.schema_migrations", name) + }) + + t.Run("custom schema", func(t *testing.T) { + u, err := url.Parse(postgresTestURL(t).String() + "&search_path=foo,bar,public") + require.NoError(t, err) + db := prepTestPostgresDB(t, u) + defer mustClose(db) + + // if "foo" schema does not exist, current schema should be "public" + _, err = db.Exec("drop schema if exists foo") + require.NoError(t, err) + _, err = db.Exec("drop schema if exists bar") + require.NoError(t, err) + name, err := drv.migrationsTableName(db) + require.NoError(t, err) + require.Equal(t, "public.schema_migrations", name) + + // if "foo" schema exists, it should be used + _, err = db.Exec("create schema foo") + require.NoError(t, err) + name, err = drv.migrationsTableName(db) + require.NoError(t, err) + require.Equal(t, "foo.schema_migrations", name) + }) + + t.Run("no schema", func(t *testing.T) { + u := postgresTestURL(t) + db := prepTestPostgresDB(t, u) + defer mustClose(db) + + // this is an unlikely edge case, but if for some reason there is + // no current schema then we should default to "public" + _, err := db.Exec("select pg_catalog.set_config('search_path', '', false)") + require.NoError(t, err) + + name, err := drv.migrationsTableName(db) + require.NoError(t, err) + require.Equal(t, "public.schema_migrations", name) + }) +} diff --git a/pkg/dbmate/utils.go b/pkg/dbmate/utils.go index 1eddb1f..cee8476 100644 --- a/pkg/dbmate/utils.go +++ b/pkg/dbmate/utils.go @@ -104,7 +104,7 @@ func trimLeadingSQLComments(data []byte) ([]byte, error) { // queryColumn runs a SQL statement and returns a slice of strings // it is assumed that the statement returns only one column // e.g. schema_migrations table -func queryColumn(db *sql.DB, query string) ([]string, error) { +func queryColumn(db Transaction, query string) ([]string, error) { rows, err := db.Query(query) if err != nil { return nil, err @@ -128,6 +128,19 @@ func queryColumn(db *sql.DB, query string) ([]string, error) { return result, nil } +// queryRow runs a SQL statement and returns a single string +// it is assumed that the statement returns only one row and one column +// sql NULL is returned as empty string +func queryRow(db Transaction, query string) (string, error) { + var result sql.NullString + err := db.QueryRow(query).Scan(&result) + if err != nil || !result.Valid { + return "", err + } + + return result.String, nil +} + func printVerbose(result sql.Result) { lastInsertID, err := result.LastInsertId() if err == nil { From ac718a23dc398a97412ae52c5ee09c67f41d7193 Mon Sep 17 00:00:00 2001 From: Adrian Macneil Date: Sun, 1 Nov 2020 15:09:13 +1300 Subject: [PATCH 21/55] postgres: Automatically create search_path schema when needed (#169) In #167 we added support for specifying a postgres `search_path`, which is used to store the `schema_migrations` table. However, if the schema does not already exist it will cause an error. In this PR we automatically create the first schema in the `search_path` if it does not exist. --- README.md | 2 +- pkg/dbmate/clickhouse.go | 2 +- pkg/dbmate/clickhouse_test.go | 29 ++++++------ pkg/dbmate/db.go | 2 +- pkg/dbmate/driver.go | 2 +- pkg/dbmate/mysql.go | 2 +- pkg/dbmate/mysql_test.go | 29 ++++++------ pkg/dbmate/postgres.go | 28 +++++++++++- pkg/dbmate/postgres_test.go | 86 ++++++++++++++++++++++++++--------- pkg/dbmate/sqlite.go | 2 +- pkg/dbmate/sqlite_test.go | 29 ++++++------ 11 files changed, 146 insertions(+), 67 deletions(-) diff --git a/README.md b/README.md index cbf07ed..dae36c7 100644 --- a/README.md +++ b/README.md @@ -140,7 +140,7 @@ DATABASE_URL="postgres://username:password@/database_name?socket=/var/run/postgr ``` A `search_path` parameter can be used to specify the [current schema](https://www.postgresql.org/docs/13/ddl-schemas.html#DDL-SCHEMAS-PATH) while applying migrations, as well as for dbmate's `schema_migrations` table. -If multiple comma-separated schemas are passed, the first will be used for the `schema_migrations` table. +If the schema does not exist, it will be created automatically. If multiple comma-separated schemas are passed, the first will be used for the `schema_migrations` table. ```sh DATABASE_URL="postgres://username:password@127.0.0.1:5432/database_name?search_path=myschema" diff --git a/pkg/dbmate/clickhouse.go b/pkg/dbmate/clickhouse.go index 35a0901..930a69b 100644 --- a/pkg/dbmate/clickhouse.go +++ b/pkg/dbmate/clickhouse.go @@ -206,7 +206,7 @@ func (drv ClickHouseDriver) DatabaseExists(u *url.URL) (bool, error) { } // CreateMigrationsTable creates the schema_migrations table -func (drv ClickHouseDriver) CreateMigrationsTable(db *sql.DB) error { +func (drv ClickHouseDriver) CreateMigrationsTable(u *url.URL, db *sql.DB) error { _, err := db.Exec(` create table if not exists schema_migrations ( version String, diff --git a/pkg/dbmate/clickhouse_test.go b/pkg/dbmate/clickhouse_test.go index 48285aa..4503535 100644 --- a/pkg/dbmate/clickhouse_test.go +++ b/pkg/dbmate/clickhouse_test.go @@ -15,9 +15,8 @@ func clickhouseTestURL(t *testing.T) *url.URL { return u } -func prepTestClickHouseDB(t *testing.T) *sql.DB { +func prepTestClickHouseDB(t *testing.T, u *url.URL) *sql.DB { drv := ClickHouseDriver{} - u := clickhouseTestURL(t) // drop any existing database err := drv.DropDatabase(u) @@ -92,9 +91,9 @@ func TestClickHouseDumpSchema(t *testing.T) { u := clickhouseTestURL(t) // prepare database - db := prepTestClickHouseDB(t) + db := prepTestClickHouseDB(t, u) defer mustClose(db) - err := drv.CreateMigrationsTable(db) + err := drv.CreateMigrationsTable(u, db) require.NoError(t, err) // insert migration @@ -171,7 +170,8 @@ func TestClickHouseDatabaseExists_Error(t *testing.T) { func TestClickHouseCreateMigrationsTable(t *testing.T) { drv := ClickHouseDriver{} - db := prepTestClickHouseDB(t) + u := clickhouseTestURL(t) + db := prepTestClickHouseDB(t, u) defer mustClose(db) // migrations table should not exist @@ -180,7 +180,7 @@ func TestClickHouseCreateMigrationsTable(t *testing.T) { require.EqualError(t, err, "code: 60, message: Table dbmate.schema_migrations doesn't exist.") // create table - err = drv.CreateMigrationsTable(db) + err = drv.CreateMigrationsTable(u, db) require.NoError(t, err) // migrations table should exist @@ -188,16 +188,17 @@ func TestClickHouseCreateMigrationsTable(t *testing.T) { require.NoError(t, err) // create table should be idempotent - err = drv.CreateMigrationsTable(db) + err = drv.CreateMigrationsTable(u, db) require.NoError(t, err) } func TestClickHouseSelectMigrations(t *testing.T) { drv := ClickHouseDriver{} - db := prepTestClickHouseDB(t) + u := clickhouseTestURL(t) + db := prepTestClickHouseDB(t, u) defer mustClose(db) - err := drv.CreateMigrationsTable(db) + err := drv.CreateMigrationsTable(u, db) require.NoError(t, err) tx, err := db.Begin() @@ -229,10 +230,11 @@ func TestClickHouseSelectMigrations(t *testing.T) { func TestClickHouseInsertMigration(t *testing.T) { drv := ClickHouseDriver{} - db := prepTestClickHouseDB(t) + u := clickhouseTestURL(t) + db := prepTestClickHouseDB(t, u) defer mustClose(db) - err := drv.CreateMigrationsTable(db) + err := drv.CreateMigrationsTable(u, db) require.NoError(t, err) count := 0 @@ -255,10 +257,11 @@ func TestClickHouseInsertMigration(t *testing.T) { func TestClickHouseDeleteMigration(t *testing.T) { drv := ClickHouseDriver{} - db := prepTestClickHouseDB(t) + u := clickhouseTestURL(t) + db := prepTestClickHouseDB(t, u) defer mustClose(db) - err := drv.CreateMigrationsTable(db) + err := drv.CreateMigrationsTable(u, db) require.NoError(t, err) tx, err := db.Begin() diff --git a/pkg/dbmate/db.go b/pkg/dbmate/db.go index 8c9342b..bd9ec6b 100644 --- a/pkg/dbmate/db.go +++ b/pkg/dbmate/db.go @@ -252,7 +252,7 @@ func (db *DB) openDatabaseForMigration() (Driver, *sql.DB, error) { return nil, nil, err } - if err := drv.CreateMigrationsTable(sqlDB); err != nil { + if err := drv.CreateMigrationsTable(db.DatabaseURL, sqlDB); err != nil { mustClose(sqlDB) return nil, nil, err } diff --git a/pkg/dbmate/driver.go b/pkg/dbmate/driver.go index f640e21..8a007b4 100644 --- a/pkg/dbmate/driver.go +++ b/pkg/dbmate/driver.go @@ -13,7 +13,7 @@ type Driver interface { CreateDatabase(*url.URL) error DropDatabase(*url.URL) error DumpSchema(*url.URL, *sql.DB) ([]byte, error) - CreateMigrationsTable(*sql.DB) error + CreateMigrationsTable(*url.URL, *sql.DB) error SelectMigrations(*sql.DB, int) (map[string]bool, error) InsertMigration(Transaction, string) error DeleteMigration(Transaction, string) error diff --git a/pkg/dbmate/mysql.go b/pkg/dbmate/mysql.go index 636b63a..9d23f79 100644 --- a/pkg/dbmate/mysql.go +++ b/pkg/dbmate/mysql.go @@ -192,7 +192,7 @@ func (drv MySQLDriver) DatabaseExists(u *url.URL) (bool, error) { } // CreateMigrationsTable creates the schema_migrations table -func (drv MySQLDriver) CreateMigrationsTable(db *sql.DB) error { +func (drv MySQLDriver) CreateMigrationsTable(u *url.URL, db *sql.DB) error { _, err := db.Exec("create table if not exists schema_migrations " + "(version varchar(255) primary key)") diff --git a/pkg/dbmate/mysql_test.go b/pkg/dbmate/mysql_test.go index a962324..8e34d6e 100644 --- a/pkg/dbmate/mysql_test.go +++ b/pkg/dbmate/mysql_test.go @@ -15,9 +15,8 @@ func mySQLTestURL(t *testing.T) *url.URL { return u } -func prepTestMySQLDB(t *testing.T) *sql.DB { +func prepTestMySQLDB(t *testing.T, u *url.URL) *sql.DB { drv := MySQLDriver{} - u := mySQLTestURL(t) // drop any existing database err := drv.DropDatabase(u) @@ -121,9 +120,9 @@ func TestMySQLDumpSchema(t *testing.T) { u := mySQLTestURL(t) // prepare database - db := prepTestMySQLDB(t) + db := prepTestMySQLDB(t, u) defer mustClose(db) - err := drv.CreateMigrationsTable(db) + err := drv.CreateMigrationsTable(u, db) require.NoError(t, err) // insert migration @@ -191,7 +190,8 @@ func TestMySQLDatabaseExists_Error(t *testing.T) { func TestMySQLCreateMigrationsTable(t *testing.T) { drv := MySQLDriver{} - db := prepTestMySQLDB(t) + u := mySQLTestURL(t) + db := prepTestMySQLDB(t, u) defer mustClose(db) // migrations table should not exist @@ -200,7 +200,7 @@ func TestMySQLCreateMigrationsTable(t *testing.T) { require.Regexp(t, "Table 'dbmate.schema_migrations' doesn't exist", err.Error()) // create table - err = drv.CreateMigrationsTable(db) + err = drv.CreateMigrationsTable(u, db) require.NoError(t, err) // migrations table should exist @@ -208,16 +208,17 @@ func TestMySQLCreateMigrationsTable(t *testing.T) { require.NoError(t, err) // create table should be idempotent - err = drv.CreateMigrationsTable(db) + err = drv.CreateMigrationsTable(u, db) require.NoError(t, err) } func TestMySQLSelectMigrations(t *testing.T) { drv := MySQLDriver{} - db := prepTestMySQLDB(t) + u := mySQLTestURL(t) + db := prepTestMySQLDB(t, u) defer mustClose(db) - err := drv.CreateMigrationsTable(db) + err := drv.CreateMigrationsTable(u, db) require.NoError(t, err) _, err = db.Exec(`insert into schema_migrations (version) @@ -240,10 +241,11 @@ func TestMySQLSelectMigrations(t *testing.T) { func TestMySQLInsertMigration(t *testing.T) { drv := MySQLDriver{} - db := prepTestMySQLDB(t) + u := mySQLTestURL(t) + db := prepTestMySQLDB(t, u) defer mustClose(db) - err := drv.CreateMigrationsTable(db) + err := drv.CreateMigrationsTable(u, db) require.NoError(t, err) count := 0 @@ -263,10 +265,11 @@ func TestMySQLInsertMigration(t *testing.T) { func TestMySQLDeleteMigration(t *testing.T) { drv := MySQLDriver{} - db := prepTestMySQLDB(t) + u := mySQLTestURL(t) + db := prepTestMySQLDB(t, u) defer mustClose(db) - err := drv.CreateMigrationsTable(db) + err := drv.CreateMigrationsTable(u, db) require.NoError(t, err) _, err = db.Exec(`insert into schema_migrations (version) diff --git a/pkg/dbmate/postgres.go b/pkg/dbmate/postgres.go index 3ac56ba..b77d752 100644 --- a/pkg/dbmate/postgres.go +++ b/pkg/dbmate/postgres.go @@ -191,7 +191,33 @@ func (drv PostgresDriver) DatabaseExists(u *url.URL) (bool, error) { } // CreateMigrationsTable creates the schema_migrations table -func (drv PostgresDriver) CreateMigrationsTable(db *sql.DB) error { +func (drv PostgresDriver) CreateMigrationsTable(u *url.URL, db *sql.DB) error { + // get schema from URL search_path param + searchPath := strings.Split(u.Query().Get("search_path"), ",") + urlSchema := strings.TrimSpace(searchPath[0]) + if urlSchema == "" { + urlSchema = "public" + } + + // get *unquoted* current schema from database + dbSchema, err := queryRow(db, "select current_schema()") + if err != nil { + return err + } + + // if urlSchema and dbSchema are not equal, the most likely explanation is that the schema + // has not yet been created + if urlSchema != dbSchema { + // in theory we could just execute this statement every time, but we do the comparison + // above in case the user doesn't have permissions to create schemas and the schema + // already exists + fmt.Printf("Creating schema: %s\n", urlSchema) + _, err = db.Exec("create schema if not exists " + pq.QuoteIdentifier(urlSchema)) + if err != nil { + return err + } + } + migrationsTable, err := drv.migrationsTableName(db) if err != nil { return err diff --git a/pkg/dbmate/postgres_test.go b/pkg/dbmate/postgres_test.go index b283698..aafb177 100644 --- a/pkg/dbmate/postgres_test.go +++ b/pkg/dbmate/postgres_test.go @@ -119,7 +119,7 @@ func TestPostgresCreateDropDatabase(t *testing.T) { defer mustClose(db) err = db.Ping() - require.NotNil(t, err) + require.Error(t, err) require.Equal(t, "pq: database \"dbmate\" does not exist", err.Error()) }() } @@ -131,7 +131,7 @@ func TestPostgresDumpSchema(t *testing.T) { // prepare database db := prepTestPostgresDB(t, u) defer mustClose(db) - err := drv.CreateMigrationsTable(db) + err := drv.CreateMigrationsTable(u, db) require.NoError(t, err) // insert migration @@ -191,32 +191,76 @@ func TestPostgresDatabaseExists_Error(t *testing.T) { u.User = url.User("invalid") exists, err := drv.DatabaseExists(u) + require.Error(t, err) require.Equal(t, "pq: password authentication failed for user \"invalid\"", err.Error()) require.Equal(t, false, exists) } func TestPostgresCreateMigrationsTable(t *testing.T) { drv := PostgresDriver{} - u := postgresTestURL(t) - db := prepTestPostgresDB(t, u) - defer mustClose(db) - // migrations table should not exist - count := 0 - err := db.QueryRow("select count(*) from public.schema_migrations").Scan(&count) - require.Equal(t, "pq: relation \"public.schema_migrations\" does not exist", err.Error()) + t.Run("default schema", func(t *testing.T) { + u := postgresTestURL(t) + db := prepTestPostgresDB(t, u) + defer mustClose(db) - // create table - err = drv.CreateMigrationsTable(db) - require.NoError(t, err) + // migrations table should not exist + count := 0 + err := db.QueryRow("select count(*) from public.schema_migrations").Scan(&count) + require.Error(t, err) + require.Equal(t, "pq: relation \"public.schema_migrations\" does not exist", err.Error()) - // migrations table should exist - err = db.QueryRow("select count(*) from public.schema_migrations").Scan(&count) - require.NoError(t, err) + // create table + err = drv.CreateMigrationsTable(u, db) + require.NoError(t, err) - // create table should be idempotent - err = drv.CreateMigrationsTable(db) - require.NoError(t, err) + // migrations table should exist + err = db.QueryRow("select count(*) from public.schema_migrations").Scan(&count) + require.NoError(t, err) + + // create table should be idempotent + err = drv.CreateMigrationsTable(u, db) + require.NoError(t, err) + }) + + t.Run("custom schema", func(t *testing.T) { + u, err := url.Parse(postgresTestURL(t).String() + "&search_path=foo") + require.NoError(t, err) + db := prepTestPostgresDB(t, u) + defer mustClose(db) + + // delete schema + _, err = db.Exec("drop schema if exists foo") + require.NoError(t, err) + + // drop any schema_migrations table in public schema + _, err = db.Exec("drop table if exists public.schema_migrations") + require.NoError(t, err) + + // migrations table should not exist in either schema + count := 0 + err = db.QueryRow("select count(*) from foo.schema_migrations").Scan(&count) + require.Error(t, err) + require.Equal(t, "pq: relation \"foo.schema_migrations\" does not exist", err.Error()) + err = db.QueryRow("select count(*) from public.schema_migrations").Scan(&count) + require.Error(t, err) + require.Equal(t, "pq: relation \"public.schema_migrations\" does not exist", err.Error()) + + // create table + err = drv.CreateMigrationsTable(u, db) + require.NoError(t, err) + + // foo schema should be created, and migrations table should exist only in foo schema + err = db.QueryRow("select count(*) from foo.schema_migrations").Scan(&count) + require.NoError(t, err) + err = db.QueryRow("select count(*) from public.schema_migrations").Scan(&count) + require.Error(t, err) + require.Equal(t, "pq: relation \"public.schema_migrations\" does not exist", err.Error()) + + // create table should be idempotent + err = drv.CreateMigrationsTable(u, db) + require.NoError(t, err) + }) } func TestPostgresSelectMigrations(t *testing.T) { @@ -225,7 +269,7 @@ func TestPostgresSelectMigrations(t *testing.T) { db := prepTestPostgresDB(t, u) defer mustClose(db) - err := drv.CreateMigrationsTable(db) + err := drv.CreateMigrationsTable(u, db) require.NoError(t, err) _, err = db.Exec(`insert into public.schema_migrations (version) @@ -252,7 +296,7 @@ func TestPostgresInsertMigration(t *testing.T) { db := prepTestPostgresDB(t, u) defer mustClose(db) - err := drv.CreateMigrationsTable(db) + err := drv.CreateMigrationsTable(u, db) require.NoError(t, err) count := 0 @@ -276,7 +320,7 @@ func TestPostgresDeleteMigration(t *testing.T) { db := prepTestPostgresDB(t, u) defer mustClose(db) - err := drv.CreateMigrationsTable(db) + err := drv.CreateMigrationsTable(u, db) require.NoError(t, err) _, err = db.Exec(`insert into public.schema_migrations (version) diff --git a/pkg/dbmate/sqlite.go b/pkg/dbmate/sqlite.go index 3f9298e..5eded31 100644 --- a/pkg/dbmate/sqlite.go +++ b/pkg/dbmate/sqlite.go @@ -117,7 +117,7 @@ func (drv SQLiteDriver) DatabaseExists(u *url.URL) (bool, error) { } // CreateMigrationsTable creates the schema_migrations table -func (drv SQLiteDriver) CreateMigrationsTable(db *sql.DB) error { +func (drv SQLiteDriver) CreateMigrationsTable(u *url.URL, db *sql.DB) error { _, err := db.Exec("create table if not exists schema_migrations " + "(version varchar(255) primary key)") diff --git a/pkg/dbmate/sqlite_test.go b/pkg/dbmate/sqlite_test.go index eb2adcd..3d32889 100644 --- a/pkg/dbmate/sqlite_test.go +++ b/pkg/dbmate/sqlite_test.go @@ -18,9 +18,8 @@ func sqliteTestURL(t *testing.T) *url.URL { return u } -func prepTestSQLiteDB(t *testing.T) *sql.DB { +func prepTestSQLiteDB(t *testing.T, u *url.URL) *sql.DB { drv := SQLiteDriver{} - u := sqliteTestURL(t) // drop any existing database err := drv.DropDatabase(u) @@ -69,9 +68,9 @@ func TestSQLiteDumpSchema(t *testing.T) { u := sqliteTestURL(t) // prepare database - db := prepTestSQLiteDB(t) + db := prepTestSQLiteDB(t, u) defer mustClose(db) - err := drv.CreateMigrationsTable(db) + err := drv.CreateMigrationsTable(u, db) require.NoError(t, err) // insert migration @@ -122,7 +121,8 @@ func TestSQLiteDatabaseExists(t *testing.T) { func TestSQLiteCreateMigrationsTable(t *testing.T) { drv := SQLiteDriver{} - db := prepTestSQLiteDB(t) + u := sqliteTestURL(t) + db := prepTestSQLiteDB(t, u) defer mustClose(db) // migrations table should not exist @@ -131,7 +131,7 @@ func TestSQLiteCreateMigrationsTable(t *testing.T) { require.Regexp(t, "no such table: schema_migrations", err.Error()) // create table - err = drv.CreateMigrationsTable(db) + err = drv.CreateMigrationsTable(u, db) require.NoError(t, err) // migrations table should exist @@ -139,16 +139,17 @@ func TestSQLiteCreateMigrationsTable(t *testing.T) { require.NoError(t, err) // create table should be idempotent - err = drv.CreateMigrationsTable(db) + err = drv.CreateMigrationsTable(u, db) require.NoError(t, err) } func TestSQLiteSelectMigrations(t *testing.T) { drv := SQLiteDriver{} - db := prepTestSQLiteDB(t) + u := sqliteTestURL(t) + db := prepTestSQLiteDB(t, u) defer mustClose(db) - err := drv.CreateMigrationsTable(db) + err := drv.CreateMigrationsTable(u, db) require.NoError(t, err) _, err = db.Exec(`insert into schema_migrations (version) @@ -171,10 +172,11 @@ func TestSQLiteSelectMigrations(t *testing.T) { func TestSQLiteInsertMigration(t *testing.T) { drv := SQLiteDriver{} - db := prepTestSQLiteDB(t) + u := sqliteTestURL(t) + db := prepTestSQLiteDB(t, u) defer mustClose(db) - err := drv.CreateMigrationsTable(db) + err := drv.CreateMigrationsTable(u, db) require.NoError(t, err) count := 0 @@ -194,10 +196,11 @@ func TestSQLiteInsertMigration(t *testing.T) { func TestSQLiteDeleteMigration(t *testing.T) { drv := SQLiteDriver{} - db := prepTestSQLiteDB(t) + u := sqliteTestURL(t) + db := prepTestSQLiteDB(t, u) defer mustClose(db) - err := drv.CreateMigrationsTable(db) + err := drv.CreateMigrationsTable(u, db) require.NoError(t, err) _, err = db.Exec(`insert into schema_migrations (version) From 203065661cce6a1757b10c6d912e17bafd9d39da Mon Sep 17 00:00:00 2001 From: Adrian Macneil Date: Sun, 1 Nov 2020 15:30:20 +1300 Subject: [PATCH 22/55] Properly check errors from sql.Rows (#170) --- .golangci.yml | 1 + pkg/dbmate/clickhouse.go | 4 ++++ pkg/dbmate/mysql.go | 4 ++++ pkg/dbmate/postgres.go | 4 ++++ pkg/dbmate/sqlite.go | 4 ++++ 5 files changed, 17 insertions(+) diff --git a/.golangci.yml b/.golangci.yml index 20b9a33..fcad107 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -11,6 +11,7 @@ linters: - ineffassign - misspell - nakedret + - rowserrcheck - staticcheck - structcheck - typecheck diff --git a/pkg/dbmate/clickhouse.go b/pkg/dbmate/clickhouse.go index 930a69b..63070e3 100644 --- a/pkg/dbmate/clickhouse.go +++ b/pkg/dbmate/clickhouse.go @@ -243,6 +243,10 @@ func (drv ClickHouseDriver) SelectMigrations(db *sql.DB, limit int) (map[string] migrations[version] = true } + if err = rows.Err(); err != nil { + return nil, err + } + return migrations, nil } diff --git a/pkg/dbmate/mysql.go b/pkg/dbmate/mysql.go index 9d23f79..f809587 100644 --- a/pkg/dbmate/mysql.go +++ b/pkg/dbmate/mysql.go @@ -223,6 +223,10 @@ func (drv MySQLDriver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, migrations[version] = true } + if err = rows.Err(); err != nil { + return nil, err + } + return migrations, nil } diff --git a/pkg/dbmate/postgres.go b/pkg/dbmate/postgres.go index b77d752..7cbc8a0 100644 --- a/pkg/dbmate/postgres.go +++ b/pkg/dbmate/postgres.go @@ -258,6 +258,10 @@ func (drv PostgresDriver) SelectMigrations(db *sql.DB, limit int) (map[string]bo migrations[version] = true } + if err = rows.Err(); err != nil { + return nil, err + } + return migrations, nil } diff --git a/pkg/dbmate/sqlite.go b/pkg/dbmate/sqlite.go index 5eded31..3ef10dd 100644 --- a/pkg/dbmate/sqlite.go +++ b/pkg/dbmate/sqlite.go @@ -148,6 +148,10 @@ func (drv SQLiteDriver) SelectMigrations(db *sql.DB, limit int) (map[string]bool migrations[version] = true } + if err = rows.Err(); err != nil { + return nil, err + } + return migrations, nil } From 5b7e6cef18196707865b19f73df23d3e0dc92ef7 Mon Sep 17 00:00:00 2001 From: Adrian Macneil Date: Sun, 1 Nov 2020 18:30:30 +1300 Subject: [PATCH 23/55] Add codeql-analysis.yml (#163) --- .github/workflows/codeql-analysis.yml | 62 +++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 .github/workflows/codeql-analysis.yml diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 0000000..6c1953e --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,62 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. + +name: "CodeQL" + +on: + push: + branches: [ master ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ master ] + schedule: + - cron: '0 0 * * 4' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + language: [ 'go' ] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] + # Learn more... + # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + # queries: ./path/to/local/query, your-org/your-repo/queries@main + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v1 + + # ℹ️ Command-line programs to run using the OS shell. + # 📚 https://git.io/JvXDl + + # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines + # and modify them (or add more) to build your code if your project + # uses a compiled language + + #- run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 From 2950db713111a2e892db8a3efc8b8141b5f82ff2 Mon Sep 17 00:00:00 2001 From: Enrico Date: Mon, 2 Nov 2020 00:02:28 +0100 Subject: [PATCH 24/55] mysql: Create the schema_migrations table using latin1 charset (#172) Force the `schema_migrations` table to be created with `latin1` character set, to fix the behavior of dbmate in MariaDB 10.1 and older (or MySQL 5.6 and older) when the system charset has more than 1 byte per character (e.g. utf8mb4) Closes #85 --- pkg/dbmate/mysql.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/dbmate/mysql.go b/pkg/dbmate/mysql.go index f809587..dd80e60 100644 --- a/pkg/dbmate/mysql.go +++ b/pkg/dbmate/mysql.go @@ -194,7 +194,7 @@ func (drv MySQLDriver) DatabaseExists(u *url.URL) (bool, error) { // CreateMigrationsTable creates the schema_migrations table func (drv MySQLDriver) CreateMigrationsTable(u *url.URL, db *sql.DB) error { _, err := db.Exec("create table if not exists schema_migrations " + - "(version varchar(255) primary key)") + "(version varchar(255) primary key) character set latin1 collate latin1_bin") return err } From e44e09eb678bf10388cd5d3f08963d642ffbf5ee Mon Sep 17 00:00:00 2001 From: Adrian Macneil Date: Mon, 2 Nov 2020 20:22:59 +1300 Subject: [PATCH 25/55] Migrate CI to GitHub Actions (#171) --- .dockerignore | 1 + .github/workflows/build.yml | 63 +++++++++++++++++++++++++++++++++++++ .gitignore | 1 + .travis.yml | 27 ---------------- Dockerfile | 24 +++++++------- Makefile | 8 ++--- docker-compose.override.yml | 7 ----- docker-compose.yml | 11 +++++-- 8 files changed, 91 insertions(+), 51 deletions(-) create mode 100644 .dockerignore create mode 100644 .github/workflows/build.yml delete mode 100644 .travis.yml delete mode 100644 docker-compose.override.yml diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..6b8710a --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +.git diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..34eb6da --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,63 @@ +name: CI + +on: + push: + branches: [ master ] + tags: 'v*' + pull_request: + branches: [ master ] + +jobs: + build: + name: Build & Test + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Environment + run: | + set -x + docker version + docker-compose version + + - name: Cache + uses: actions/cache@v2 + with: + key: cache + path: .cache + + - name: Build docker image + run: | + set -x + docker-compose build + docker-compose run --rm --no-deps dbmate --version + + - name: Build binaries + run: | + set -x + docker-compose run --rm --no-deps dev make build-all + dist/dbmate-linux-amd64 --version + + - name: Lint + run: docker-compose run --rm --no-deps dev make lint + + - name: Start test dependencies + run: | + set -x + docker-compose pull --quiet + docker-compose up --detach + + - name: Run tests + run: | + set -x + docker-compose run --rm dev make wait + docker-compose run --rm dev make test + + - name: Release + uses: softprops/action-gh-release@v1 + if: ${{ startsWith(github.ref, 'refs/tags/v') }} + with: + files: dist/* + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index c290c8c..44a09fe 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ .DS_Store .env +/.cache /db /dbmate /dist diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 0876b02..0000000 --- a/.travis.yml +++ /dev/null @@ -1,27 +0,0 @@ -sudo: required -services: - - docker -install: - - docker version - - docker-compose version - - rm docker-compose.override.yml - - docker-compose pull - - docker-compose build - - docker-compose up -d -script: - - docker-compose run --rm --volume "$PWD/dist:/src/dist" dbmate make build-all - - docker-compose run --rm dbmate make lint - - docker-compose run --rm dbmate make wait - - docker-compose run --rm dbmate make test - - docker build -t dbmate . - - docker run --rm dbmate --help -deploy: - provider: releases - api_key: - secure: LuDKEwGYaJWqYe0Ym6qoxHAbZx1kDiTigYcycribnmugGVDvRpZp5MJuQivTD1eZ4sl58UO3NX6jyf8pfx814m6G+3gjWaQ56OtJIKF2OwtxnwvMZNaVz63hSi8n1jCdbGTHlOqDAUQbjGFGrmsI5wAGsUM16yRktCAEn5APHxNHMnQcGTIe3Wcp+G4Fp+iRQ80Ro6BLPo2ys+WWDxz6Wahv3U6CJWtkQMAuZyZTSXL1Pl6kqlZyGKhUbPHvq1KU0wWccvwT5P6KVo314aF5Skw0LJ3qciwUTnc7dsHCkvJKF5/Nev3/KWWVKR3DBh98gS2hDNjpSozYAO/e9QiIjaidqYYifoEFIY7Jx0DArJwaw3PLnRMKGKMyww2CaFopxr5HT1s18EGMytRbduASUieeF+7pFs29Bouc8xC0OnKZdlXRewAYFjWzWdCiXQVU18q3DggFK6fb1HWLmy6NX2RmxDODSv3B8P3DzmsdwR0vc64IxmnS+zTdjUwE0+FuxOEmWl/iqYi+nXKXOj0domFudfaBxGT2f5ThBw5Ns9FXKBGxyRSD8wf8+sDbUIUxUdZw1kCttNM/JSbbz9ErLV/Ik23BWBPkjDxo4DpLgqVMg8LHPbmhCuKHvckhoCBpORuvX3PTzzdCsJfiYJCr6nMt/deAp/B/O2O/3/2nFYI= - file_glob: true - file: dist/* - skip_cleanup: true - on: - tags: true - repo: amacneil/dbmate diff --git a/Dockerfile b/Dockerfile index 4109fd2..0ee8269 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,7 @@ -# build image -FROM techknowlogick/xgo:go-1.15.x as build +# development image +FROM techknowlogick/xgo:go-1.15.x as dev WORKDIR /src +ENV GOCACHE /src/.cache/go-build # enable cgo to build sqlite ENV CGO_ENABLED 1 @@ -24,18 +25,19 @@ RUN curl -fsSL -o /tmp/lint-install.sh https://raw.githubusercontent.com/golangc COPY go.* ./ RUN go mod download -# build -COPY . ./ -RUN make build - ENTRYPOINT [] CMD ["/bin/bash"] -# runtime image -FROM alpine +# build stage +FROM dev as build +COPY . ./ +RUN make build + +# release stage +FROM alpine as release RUN apk add --no-cache \ - mariadb-client \ - postgresql-client \ - sqlite + mariadb-client \ + postgresql-client \ + sqlite COPY --from=build /src/dist/dbmate-linux-amd64 /usr/local/bin/dbmate ENTRYPOINT ["dbmate"] diff --git a/Makefile b/Makefile index 07a6005..ef9dfc0 100644 --- a/Makefile +++ b/Makefile @@ -52,8 +52,8 @@ build-all: clean build-linux-amd64 .PHONY: docker-make docker-make: docker-compose build - docker-compose run --rm dbmate make + docker-compose run --rm dev make -.PHONY: docker-bash -docker-bash: - -docker-compose run --rm dbmate bash +.PHONY: docker-sh +docker-sh: + -docker-compose run --rm dev diff --git a/docker-compose.override.yml b/docker-compose.override.yml deleted file mode 100644 index 82661d4..0000000 --- a/docker-compose.override.yml +++ /dev/null @@ -1,7 +0,0 @@ -# this file is used to mount the current directory as a volume -# we remove it in CI to use files inside the container only -version: '2.3' -services: - dbmate: - volumes: - - .:/src diff --git a/docker-compose.yml b/docker-compose.yml index 57b9996..09d8331 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,9 +1,11 @@ version: '2.3' services: - dbmate: + dev: build: context: . - target: build + target: dev + volumes: + - .:/src depends_on: - mysql - postgres @@ -13,6 +15,11 @@ services: POSTGRESQL_URL: postgres://postgres:postgres@postgres/dbmate?sslmode=disable CLICKHOUSE_URL: clickhouse://clickhouse:9000?database=dbmate + dbmate: + build: + context: . + target: release + mysql: image: mysql:5.7 environment: From c2b14bb7e047225f5dd717694368051bd4fd9281 Mon Sep 17 00:00:00 2001 From: Adrian Macneil Date: Tue, 10 Nov 2020 12:38:51 +1300 Subject: [PATCH 26/55] v1.11.0 (#174) --- Dockerfile | 2 +- go.mod | 7 ++++--- go.sum | 25 ++++++++++--------------- main.go | 2 +- pkg/dbmate/version.go | 2 +- 5 files changed, 17 insertions(+), 21 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0ee8269..ebdbdf9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,7 +18,7 @@ RUN apt-get update \ # golangci-lint RUN curl -fsSL -o /tmp/lint-install.sh https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh \ && chmod +x /tmp/lint-install.sh \ - && /tmp/lint-install.sh -b /usr/local/bin v1.32.0 \ + && /tmp/lint-install.sh -b /usr/local/bin v1.32.2 \ && rm -f /tmp/lint-install.sh # download modules diff --git a/go.mod b/go.mod index 35f4cb5..420929a 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/amacneil/dbmate go 1.15 require ( - github.com/ClickHouse/clickhouse-go v1.4.1 + github.com/ClickHouse/clickhouse-go v1.4.3 github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/go-sql-driver/mysql v1.5.0 @@ -11,8 +11,9 @@ require ( github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d github.com/kr/pretty v0.1.0 // indirect github.com/lib/pq v1.8.0 - github.com/mattn/go-sqlite3 v1.14.0 + github.com/mattn/go-sqlite3 v1.14.4 + github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/stretchr/testify v1.4.0 - github.com/urfave/cli/v2 v2.2.0 + github.com/urfave/cli/v2 v2.3.0 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect ) diff --git a/go.sum b/go.sum index 80b1521..142d1fc 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,6 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/ClickHouse/clickhouse-go v1.4.1 h1:D9cihLg76O1ZyILLaXq1eksYzEuV010NdvucgKGGK14= -github.com/ClickHouse/clickhouse-go v1.4.1/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI= -github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= -github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= +github.com/ClickHouse/clickhouse-go v1.4.3 h1:iAFMa2UrQdR5bHJ2/yaSLffZkxpcOYQMCUuKeNXGdqc= +github.com/ClickHouse/clickhouse-go v1.4.3/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI= github.com/bkaradzic/go-lz4 v1.0.0 h1:RXc4wYsyz985CkXXeX04y4VnZFGG8Rd43pRaHsOXAKk= github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4= github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 h1:F1EaeKL/ta07PY/k9Os/UFtwERei2/XzGemhpGnBKNg= @@ -30,31 +28,28 @@ github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.8.0 h1:9xohqzkUwzR4Ga4ivdTcawVS89YSDVxXMa3xJX3cGzg= github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= -github.com/mattn/go-sqlite3 v1.14.0 h1:mLyGNKR8+Vv9CAU7PphKa2hkEqxxhn8i32J6FPj1/QA= -github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus= +github.com/mattn/go-sqlite3 v1.14.4 h1:4rQjbDxdu9fSgI/r3KN72G3c2goxknAqHHgPWWs8UlI= +github.com/mattn/go-sqlite3 v1.14.4/go.mod h1:WVKg1VTActs4Qso6iwGbiFih2UIHo0ENGwNd0Lj+XmI= github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/urfave/cli/v2 v2.2.0 h1:JTTnM6wKzdA0Jqodd966MVj4vWbbquZykeX1sKbe2C4= -github.com/urfave/cli/v2 v2.2.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M= +github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3 h1:fvjTMHxHEw/mxHbtzPi3JCcKXQRAnQTBRo6YCJSVHKI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/main.go b/main.go index 749bb69..7d70497 100644 --- a/main.go +++ b/main.go @@ -176,7 +176,7 @@ func NewApp() *cli.App { } if pending > 0 && setExitCode { - return cli.NewExitError("", 1) + return cli.Exit("", 1) } return nil diff --git a/pkg/dbmate/version.go b/pkg/dbmate/version.go index ffb2f9c..a851cca 100644 --- a/pkg/dbmate/version.go +++ b/pkg/dbmate/version.go @@ -1,4 +1,4 @@ package dbmate // Version of dbmate -const Version = "1.10.0" +const Version = "1.11.0" From 656dc0253a7f78c2eb7633a18895ee414f2507f1 Mon Sep 17 00:00:00 2001 From: Adrian Macneil Date: Tue, 10 Nov 2020 14:09:53 +1300 Subject: [PATCH 27/55] Update docs (#175) --- README.md | 8 ++++---- RELEASING.md | 5 ++--- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index dae36c7..4abbba4 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Dbmate -[![Build Status](https://travis-ci.org/amacneil/dbmate.svg?branch=master)](https://travis-ci.org/amacneil/dbmate) +[![GitHub Build](https://img.shields.io/github/workflow/status/amacneil/dbmate/CI/master)](https://github.com/amacneil/dbmate/actions?query=branch%3Amaster+event%3Apush+workflow%3ACI) [![Go Report Card](https://goreportcard.com/badge/github.com/amacneil/dbmate)](https://goreportcard.com/report/github.com/amacneil/dbmate) [![GitHub Release](https://img.shields.io/github/release/amacneil/dbmate.svg)](https://github.com/amacneil/dbmate/releases) @@ -46,13 +46,13 @@ $ sudo chmod +x /usr/local/bin/dbmate You can run dbmate using the official docker image (remember to set `--network=host` or see [this comment](https://github.com/amacneil/dbmate/issues/128#issuecomment-615924611) for more tips on using dbmate with docker networking): ```sh -$ docker run --rm --network=host -it amacneil/dbmate --help +$ docker run --rm -it --network=host amacneil/dbmate --help ``` -If you wish to create or apply migrations, you will need to use Docker's [bind mount](https://docs.docker.com/storage/bind-mounts/) feature to make your local working directory available inside the dbmate container: +If you wish to create or apply migrations, you will need to use Docker's [bind mount](https://docs.docker.com/storage/bind-mounts/) feature to make your local working directory (`pwd`) available inside the dbmate container: ```sh -$ docker run --rm -it -v "$(pwd)"/db:/db amacneil/dbmate new create_users_table +$ docker run --rm -it --network=host -v "$(pwd)/db:/db" amacneil/dbmate new create_users_table ``` **Heroku** diff --git a/RELEASING.md b/RELEASING.md index feb13d3..ab49b26 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -3,6 +3,5 @@ The following steps should be followed to publish a new version of dbmate (requires write access to this repository). 1. Update [version.go](/pkg/dbmate/version.go) with new version number ([example PR](https://github.com/amacneil/dbmate/pull/146/files)) -2. Create new release on GitHub project [releases page](https://github.com/amacneil/dbmate/releases) -3. Travis CI will automatically publish release binaries to GitHub -4. GitHub Actions will automatically create PR to update Homebrew package +2. Create new release on [releases page](https://github.com/amacneil/dbmate/releases) and write release notes +3. GitHub Actions will automatically publish release binaries and submit Homebrew PR From c907c3f5c60f85165835a27c0c833610297870c6 Mon Sep 17 00:00:00 2001 From: Adrian Macneil Date: Tue, 17 Nov 2020 18:11:24 +1300 Subject: [PATCH 28/55] Ability to specify custom migrations table name (#178) Supported via `--migrations-table` CLI flag or `DBMATE_MIGRATIONS_TABLE` environment variable. Specified table name is quoted when necessary. For PostgreSQL specifically, it's also possible to specify a custom schema (for example: `--migrations-table=foo.migrations`). Closes #168 --- main.go | 7 + pkg/dbmate/clickhouse.go | 89 +++++++---- pkg/dbmate/clickhouse_test.go | 133 ++++++++++++---- pkg/dbmate/db.go | 44 ++++-- pkg/dbmate/db_test.go | 22 ++- pkg/dbmate/driver.go | 17 +- pkg/dbmate/driver_test.go | 10 +- pkg/dbmate/mysql.go | 76 +++++---- pkg/dbmate/mysql_test.go | 76 ++++++--- pkg/dbmate/postgres.go | 138 ++++++++++------ pkg/dbmate/postgres_test.go | 287 ++++++++++++++++++++++++++-------- pkg/dbmate/sqlite.go | 77 ++++++--- pkg/dbmate/sqlite_test.go | 123 +++++++++++---- pkg/dbmate/utils.go | 10 +- pkg/dbmate/utils_test.go | 23 +++ 15 files changed, 807 insertions(+), 325 deletions(-) diff --git a/main.go b/main.go index 7d70497..04a777f 100644 --- a/main.go +++ b/main.go @@ -52,6 +52,12 @@ func NewApp() *cli.App { Value: dbmate.DefaultMigrationsDir, Usage: "specify the directory containing migration files", }, + &cli.StringFlag{ + Name: "migrations-table", + EnvVars: []string{"DBMATE_MIGRATIONS_TABLE"}, + Value: dbmate.DefaultMigrationsTableName, + Usage: "specify the database table to record migrations in", + }, &cli.StringFlag{ Name: "schema-file", Aliases: []string{"s"}, @@ -222,6 +228,7 @@ func action(f func(*dbmate.DB, *cli.Context) error) cli.ActionFunc { db := dbmate.New(u) db.AutoDumpSchema = !c.Bool("no-dump-schema") db.MigrationsDir = c.String("migrations-dir") + db.MigrationsTableName = c.String("migrations-table") db.SchemaFile = c.String("schema-file") db.WaitBefore = c.Bool("wait") overrideTimeout := c.Duration("wait-timeout") diff --git a/pkg/dbmate/clickhouse.go b/pkg/dbmate/clickhouse.go index 63070e3..f0acd3b 100644 --- a/pkg/dbmate/clickhouse.go +++ b/pkg/dbmate/clickhouse.go @@ -13,11 +13,12 @@ import ( ) func init() { - RegisterDriver(ClickHouseDriver{}, "clickhouse") + RegisterDriver(&ClickHouseDriver{}, "clickhouse") } // ClickHouseDriver provides top level database functions type ClickHouseDriver struct { + migrationsTableName string } func normalizeClickHouseURL(initialURL *url.URL) *url.URL { @@ -52,12 +53,17 @@ func normalizeClickHouseURL(initialURL *url.URL) *url.URL { return &u } +// SetMigrationsTableName sets the schema migrations table name +func (drv *ClickHouseDriver) SetMigrationsTableName(name string) { + drv.migrationsTableName = name +} + // Open creates a new database connection -func (drv ClickHouseDriver) Open(u *url.URL) (*sql.DB, error) { +func (drv *ClickHouseDriver) Open(u *url.URL) (*sql.DB, error) { return sql.Open("clickhouse", normalizeClickHouseURL(u).String()) } -func (drv ClickHouseDriver) openClickHouseDB(u *url.URL) (*sql.DB, error) { +func (drv *ClickHouseDriver) openClickHouseDB(u *url.URL) (*sql.DB, error) { // connect to clickhouse database clickhouseURL := normalizeClickHouseURL(u) values := clickhouseURL.Query() @@ -67,7 +73,7 @@ func (drv ClickHouseDriver) openClickHouseDB(u *url.URL) (*sql.DB, error) { return drv.Open(clickhouseURL) } -func (drv ClickHouseDriver) databaseName(u *url.URL) string { +func (drv *ClickHouseDriver) databaseName(u *url.URL) string { name := normalizeClickHouseURL(u).Query().Get("database") if name == "" { name = "default" @@ -77,7 +83,7 @@ func (drv ClickHouseDriver) databaseName(u *url.URL) string { var clickhouseValidIdentifier = regexp.MustCompile(`^[a-zA-Z_][0-9a-zA-Z_]*$`) -func clickhouseQuoteIdentifier(str string) string { +func (drv *ClickHouseDriver) quoteIdentifier(str string) string { if clickhouseValidIdentifier.MatchString(str) { return str } @@ -88,7 +94,7 @@ func clickhouseQuoteIdentifier(str string) string { } // CreateDatabase creates the specified database -func (drv ClickHouseDriver) CreateDatabase(u *url.URL) error { +func (drv *ClickHouseDriver) CreateDatabase(u *url.URL) error { name := drv.databaseName(u) fmt.Printf("Creating: %s\n", name) @@ -98,13 +104,13 @@ func (drv ClickHouseDriver) CreateDatabase(u *url.URL) error { } defer mustClose(db) - _, err = db.Exec("create database " + clickhouseQuoteIdentifier(name)) + _, err = db.Exec("create database " + drv.quoteIdentifier(name)) return err } // DropDatabase drops the specified database (if it exists) -func (drv ClickHouseDriver) DropDatabase(u *url.URL) error { +func (drv *ClickHouseDriver) DropDatabase(u *url.URL) error { name := drv.databaseName(u) fmt.Printf("Dropping: %s\n", name) @@ -114,15 +120,15 @@ func (drv ClickHouseDriver) DropDatabase(u *url.URL) error { } defer mustClose(db) - _, err = db.Exec("drop database if exists " + clickhouseQuoteIdentifier(name)) + _, err = db.Exec("drop database if exists " + drv.quoteIdentifier(name)) return err } -func clickhouseSchemaDump(db *sql.DB, buf *bytes.Buffer, databaseName string) error { +func (drv *ClickHouseDriver) schemaDump(db *sql.DB, buf *bytes.Buffer, databaseName string) error { buf.WriteString("\n--\n-- Database schema\n--\n\n") - buf.WriteString("CREATE DATABASE " + clickhouseQuoteIdentifier(databaseName) + " IF NOT EXISTS;\n\n") + buf.WriteString("CREATE DATABASE " + drv.quoteIdentifier(databaseName) + " IF NOT EXISTS;\n\n") tables, err := queryColumn(db, "show tables") if err != nil { @@ -132,7 +138,7 @@ func clickhouseSchemaDump(db *sql.DB, buf *bytes.Buffer, databaseName string) er for _, table := range tables { var clause string - err = db.QueryRow("show create table " + clickhouseQuoteIdentifier(table)).Scan(&clause) + err = db.QueryRow("show create table " + drv.quoteIdentifier(table)).Scan(&clause) if err != nil { return err } @@ -141,10 +147,13 @@ func clickhouseSchemaDump(db *sql.DB, buf *bytes.Buffer, databaseName string) er return nil } -func clickhouseSchemaMigrationsDump(db *sql.DB, buf *bytes.Buffer) error { +func (drv *ClickHouseDriver) schemaMigrationsDump(db *sql.DB, buf *bytes.Buffer) error { + migrationsTable := drv.quotedMigrationsTableName() + // load applied migrations migrations, err := queryColumn(db, - "select version from schema_migrations final where applied order by version asc", + fmt.Sprintf("select version from %s final ", migrationsTable)+ + "where applied order by version asc", ) if err != nil { return err @@ -155,29 +164,30 @@ func clickhouseSchemaMigrationsDump(db *sql.DB, buf *bytes.Buffer) error { migrations[i] = "'" + quoter.Replace(migrations[i]) + "'" } - // build schema_migrations table data + // build schema migrations table data buf.WriteString("\n--\n-- Dbmate schema migrations\n--\n\n") if len(migrations) > 0 { - buf.WriteString("INSERT INTO schema_migrations (version) VALUES\n (" + - strings.Join(migrations, "),\n (") + - ");\n") + buf.WriteString( + fmt.Sprintf("INSERT INTO %s (version) VALUES\n (", migrationsTable) + + strings.Join(migrations, "),\n (") + + ");\n") } return nil } // DumpSchema returns the current database schema -func (drv ClickHouseDriver) DumpSchema(u *url.URL, db *sql.DB) ([]byte, error) { +func (drv *ClickHouseDriver) DumpSchema(u *url.URL, db *sql.DB) ([]byte, error) { var buf bytes.Buffer var err error - err = clickhouseSchemaDump(db, &buf, drv.databaseName(u)) + err = drv.schemaDump(db, &buf, drv.databaseName(u)) if err != nil { return nil, err } - err = clickhouseSchemaMigrationsDump(db, &buf) + err = drv.schemaMigrationsDump(db, &buf) if err != nil { return nil, err } @@ -186,7 +196,7 @@ func (drv ClickHouseDriver) DumpSchema(u *url.URL, db *sql.DB) ([]byte, error) { } // DatabaseExists determines whether the database exists -func (drv ClickHouseDriver) DatabaseExists(u *url.URL) (bool, error) { +func (drv *ClickHouseDriver) DatabaseExists(u *url.URL) (bool, error) { name := drv.databaseName(u) db, err := drv.openClickHouseDB(u) @@ -205,24 +215,27 @@ func (drv ClickHouseDriver) DatabaseExists(u *url.URL) (bool, error) { return exists, err } -// CreateMigrationsTable creates the schema_migrations table -func (drv ClickHouseDriver) CreateMigrationsTable(u *url.URL, db *sql.DB) error { - _, err := db.Exec(` - create table if not exists schema_migrations ( +// CreateMigrationsTable creates the schema migrations table +func (drv *ClickHouseDriver) CreateMigrationsTable(u *url.URL, db *sql.DB) error { + _, err := db.Exec(fmt.Sprintf(` + create table if not exists %s ( version String, ts DateTime default now(), applied UInt8 default 1 ) engine = ReplacingMergeTree(ts) primary key version order by version - `) + `, drv.quotedMigrationsTableName())) + return err } // SelectMigrations returns a list of applied migrations // with an optional limit (in descending order) -func (drv ClickHouseDriver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, error) { - query := "select version from schema_migrations final where applied order by version desc" +func (drv *ClickHouseDriver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, error) { + query := fmt.Sprintf("select version from %s final where applied order by version desc", + drv.quotedMigrationsTableName()) + if limit >= 0 { query = fmt.Sprintf("%s limit %d", query, limit) } @@ -251,15 +264,19 @@ func (drv ClickHouseDriver) SelectMigrations(db *sql.DB, limit int) (map[string] } // InsertMigration adds a new migration record -func (drv ClickHouseDriver) InsertMigration(db Transaction, version string) error { - _, err := db.Exec("insert into schema_migrations (version) values (?)", version) +func (drv *ClickHouseDriver) InsertMigration(db Transaction, version string) error { + _, err := db.Exec( + fmt.Sprintf("insert into %s (version) values (?)", drv.quotedMigrationsTableName()), + version) + return err } // DeleteMigration removes a migration record -func (drv ClickHouseDriver) DeleteMigration(db Transaction, version string) error { +func (drv *ClickHouseDriver) DeleteMigration(db Transaction, version string) error { _, err := db.Exec( - "insert into schema_migrations (version, applied) values (?, ?)", + fmt.Sprintf("insert into %s (version, applied) values (?, ?)", + drv.quotedMigrationsTableName()), version, false, ) @@ -268,7 +285,7 @@ func (drv ClickHouseDriver) DeleteMigration(db Transaction, version string) erro // Ping verifies a connection to the database server. It does not verify whether the // specified database exists. -func (drv ClickHouseDriver) Ping(u *url.URL) error { +func (drv *ClickHouseDriver) Ping(u *url.URL) error { // attempt connection to primary database, not "clickhouse" database // to support servers with no "clickhouse" database // (see https://github.com/amacneil/dbmate/issues/78) @@ -291,3 +308,7 @@ func (drv ClickHouseDriver) Ping(u *url.URL) error { return err } + +func (drv *ClickHouseDriver) quotedMigrationsTableName() string { + return drv.quoteIdentifier(drv.migrationsTableName) +} diff --git a/pkg/dbmate/clickhouse_test.go b/pkg/dbmate/clickhouse_test.go index 4503535..c1aeefd 100644 --- a/pkg/dbmate/clickhouse_test.go +++ b/pkg/dbmate/clickhouse_test.go @@ -15,8 +15,15 @@ func clickhouseTestURL(t *testing.T) *url.URL { return u } +func testClickHouseDriver() *ClickHouseDriver { + drv := &ClickHouseDriver{} + drv.SetMigrationsTableName(DefaultMigrationsTableName) + + return drv +} + func prepTestClickHouseDB(t *testing.T, u *url.URL) *sql.DB { - drv := ClickHouseDriver{} + drv := testClickHouseDriver() // drop any existing database err := drv.DropDatabase(u) @@ -50,7 +57,7 @@ func TestNormalizeClickHouseURLCanonical(t *testing.T) { } func TestClickHouseCreateDropDatabase(t *testing.T) { - drv := ClickHouseDriver{} + drv := testClickHouseDriver() u := clickhouseTestURL(t) // drop any existing database @@ -87,7 +94,9 @@ func TestClickHouseCreateDropDatabase(t *testing.T) { } func TestClickHouseDumpSchema(t *testing.T) { - drv := ClickHouseDriver{} + drv := testClickHouseDriver() + drv.SetMigrationsTableName("test_migrations") + u := clickhouseTestURL(t) // prepare database @@ -113,11 +122,11 @@ func TestClickHouseDumpSchema(t *testing.T) { // DumpSchema should return schema schema, err := drv.DumpSchema(u, db) require.NoError(t, err) - require.Contains(t, string(schema), "CREATE TABLE "+drv.databaseName(u)+".schema_migrations") + require.Contains(t, string(schema), "CREATE TABLE "+drv.databaseName(u)+".test_migrations") require.Contains(t, string(schema), "--\n"+ "-- Dbmate schema migrations\n"+ "--\n\n"+ - "INSERT INTO schema_migrations (version) VALUES\n"+ + "INSERT INTO test_migrations (version) VALUES\n"+ " ('abc1'),\n"+ " ('abc2');\n") @@ -134,7 +143,7 @@ func TestClickHouseDumpSchema(t *testing.T) { } func TestClickHouseDatabaseExists(t *testing.T) { - drv := ClickHouseDriver{} + drv := testClickHouseDriver() u := clickhouseTestURL(t) // drop any existing database @@ -157,7 +166,7 @@ func TestClickHouseDatabaseExists(t *testing.T) { } func TestClickHouseDatabaseExists_Error(t *testing.T) { - drv := ClickHouseDriver{} + drv := testClickHouseDriver() u := clickhouseTestURL(t) values := u.Query() values.Set("username", "invalid") @@ -169,31 +178,61 @@ func TestClickHouseDatabaseExists_Error(t *testing.T) { } func TestClickHouseCreateMigrationsTable(t *testing.T) { - drv := ClickHouseDriver{} - u := clickhouseTestURL(t) - db := prepTestClickHouseDB(t, u) - defer mustClose(db) + t.Run("default table", func(t *testing.T) { + drv := testClickHouseDriver() + u := clickhouseTestURL(t) + db := prepTestClickHouseDB(t, u) + defer mustClose(db) - // migrations table should not exist - count := 0 - err := db.QueryRow("select count(*) from schema_migrations").Scan(&count) - require.EqualError(t, err, "code: 60, message: Table dbmate.schema_migrations doesn't exist.") + // migrations table should not exist + count := 0 + err := db.QueryRow("select count(*) from schema_migrations").Scan(&count) + require.EqualError(t, err, "code: 60, message: Table dbmate.schema_migrations doesn't exist.") - // create table - err = drv.CreateMigrationsTable(u, db) - require.NoError(t, err) + // create table + err = drv.CreateMigrationsTable(u, db) + require.NoError(t, err) - // migrations table should exist - err = db.QueryRow("select count(*) from schema_migrations").Scan(&count) - require.NoError(t, err) + // migrations table should exist + err = db.QueryRow("select count(*) from schema_migrations").Scan(&count) + require.NoError(t, err) - // create table should be idempotent - err = drv.CreateMigrationsTable(u, db) - require.NoError(t, err) + // create table should be idempotent + err = drv.CreateMigrationsTable(u, db) + require.NoError(t, err) + }) + + t.Run("custom table", func(t *testing.T) { + drv := testClickHouseDriver() + drv.SetMigrationsTableName("testMigrations") + + u := clickhouseTestURL(t) + db := prepTestClickHouseDB(t, u) + defer mustClose(db) + + // migrations table should not exist + count := 0 + err := db.QueryRow("select count(*) from \"testMigrations\"").Scan(&count) + require.EqualError(t, err, "code: 60, message: Table dbmate.testMigrations doesn't exist.") + + // create table + err = drv.CreateMigrationsTable(u, db) + require.NoError(t, err) + + // migrations table should exist + err = db.QueryRow("select count(*) from \"testMigrations\"").Scan(&count) + require.NoError(t, err) + + // create table should be idempotent + err = drv.CreateMigrationsTable(u, db) + require.NoError(t, err) + }) } func TestClickHouseSelectMigrations(t *testing.T) { - drv := ClickHouseDriver{} + drv := testClickHouseDriver() + drv.SetMigrationsTableName("test_migrations") + u := clickhouseTestURL(t) db := prepTestClickHouseDB(t, u) defer mustClose(db) @@ -203,7 +242,7 @@ func TestClickHouseSelectMigrations(t *testing.T) { tx, err := db.Begin() require.NoError(t, err) - stmt, err := tx.Prepare("insert into schema_migrations (version) values (?)") + stmt, err := tx.Prepare("insert into test_migrations (version) values (?)") require.NoError(t, err) _, err = stmt.Exec("abc2") require.NoError(t, err) @@ -229,7 +268,9 @@ func TestClickHouseSelectMigrations(t *testing.T) { } func TestClickHouseInsertMigration(t *testing.T) { - drv := ClickHouseDriver{} + drv := testClickHouseDriver() + drv.SetMigrationsTableName("test_migrations") + u := clickhouseTestURL(t) db := prepTestClickHouseDB(t, u) defer mustClose(db) @@ -238,7 +279,7 @@ func TestClickHouseInsertMigration(t *testing.T) { require.NoError(t, err) count := 0 - err = db.QueryRow("select count(*) from schema_migrations").Scan(&count) + err = db.QueryRow("select count(*) from test_migrations").Scan(&count) require.NoError(t, err) require.Equal(t, 0, count) @@ -250,13 +291,15 @@ func TestClickHouseInsertMigration(t *testing.T) { err = tx.Commit() require.NoError(t, err) - err = db.QueryRow("select count(*) from schema_migrations where version = 'abc1'").Scan(&count) + err = db.QueryRow("select count(*) from test_migrations where version = 'abc1'").Scan(&count) require.NoError(t, err) require.Equal(t, 1, count) } func TestClickHouseDeleteMigration(t *testing.T) { - drv := ClickHouseDriver{} + drv := testClickHouseDriver() + drv.SetMigrationsTableName("test_migrations") + u := clickhouseTestURL(t) db := prepTestClickHouseDB(t, u) defer mustClose(db) @@ -266,7 +309,7 @@ func TestClickHouseDeleteMigration(t *testing.T) { tx, err := db.Begin() require.NoError(t, err) - stmt, err := tx.Prepare("insert into schema_migrations (version) values (?)") + stmt, err := tx.Prepare("insert into test_migrations (version) values (?)") require.NoError(t, err) _, err = stmt.Exec("abc2") require.NoError(t, err) @@ -283,13 +326,13 @@ func TestClickHouseDeleteMigration(t *testing.T) { require.NoError(t, err) count := 0 - err = db.QueryRow("select count(*) from schema_migrations final where applied").Scan(&count) + err = db.QueryRow("select count(*) from test_migrations final where applied").Scan(&count) require.NoError(t, err) require.Equal(t, 1, count) } func TestClickHousePing(t *testing.T) { - drv := ClickHouseDriver{} + drv := testClickHouseDriver() u := clickhouseTestURL(t) // drop any existing database @@ -306,3 +349,27 @@ func TestClickHousePing(t *testing.T) { require.Error(t, err) require.Contains(t, err.Error(), "connect: connection refused") } + +func TestClickHouseQuotedMigrationsTableName(t *testing.T) { + t.Run("default name", func(t *testing.T) { + drv := testClickHouseDriver() + name := drv.quotedMigrationsTableName() + require.Equal(t, "schema_migrations", name) + }) + + t.Run("custom name", func(t *testing.T) { + drv := testClickHouseDriver() + drv.SetMigrationsTableName("fooMigrations") + + name := drv.quotedMigrationsTableName() + require.Equal(t, "fooMigrations", name) + }) + + t.Run("quoted name", func(t *testing.T) { + drv := testClickHouseDriver() + drv.SetMigrationsTableName("bizarre\"$name") + + name := drv.quotedMigrationsTableName() + require.Equal(t, `"bizarre""$name"`, name) + }) +} diff --git a/pkg/dbmate/db.go b/pkg/dbmate/db.go index bd9ec6b..2ddc0be 100644 --- a/pkg/dbmate/db.go +++ b/pkg/dbmate/db.go @@ -15,6 +15,9 @@ import ( // DefaultMigrationsDir specifies default directory to find migration files const DefaultMigrationsDir = "./db/migrations" +// DefaultMigrationsTableName specifies default database tables to record migraitons in +const DefaultMigrationsTableName = "schema_migrations" + // DefaultSchemaFile specifies default location for schema.sql const DefaultSchemaFile = "./db/schema.sql" @@ -26,14 +29,15 @@ const DefaultWaitTimeout = 60 * time.Second // DB allows dbmate actions to be performed on a specified database type DB struct { - AutoDumpSchema bool - DatabaseURL *url.URL - MigrationsDir string - SchemaFile string - Verbose bool - WaitBefore bool - WaitInterval time.Duration - WaitTimeout time.Duration + AutoDumpSchema bool + DatabaseURL *url.URL + MigrationsDir string + MigrationsTableName string + SchemaFile string + Verbose bool + WaitBefore bool + WaitInterval time.Duration + WaitTimeout time.Duration } // migrationFileRegexp pattern for valid migration files @@ -47,19 +51,27 @@ type statusResult struct { // New initializes a new dbmate database func New(databaseURL *url.URL) *DB { return &DB{ - AutoDumpSchema: true, - DatabaseURL: databaseURL, - MigrationsDir: DefaultMigrationsDir, - SchemaFile: DefaultSchemaFile, - WaitBefore: false, - WaitInterval: DefaultWaitInterval, - WaitTimeout: DefaultWaitTimeout, + AutoDumpSchema: true, + DatabaseURL: databaseURL, + MigrationsDir: DefaultMigrationsDir, + MigrationsTableName: DefaultMigrationsTableName, + SchemaFile: DefaultSchemaFile, + WaitBefore: false, + WaitInterval: DefaultWaitInterval, + WaitTimeout: DefaultWaitTimeout, } } // GetDriver loads the required database driver func (db *DB) GetDriver() (Driver, error) { - return GetDriver(db.DatabaseURL.Scheme) + drv, err := getDriver(db.DatabaseURL.Scheme) + if err != nil { + return nil, err + } + + drv.SetMigrationsTableName(db.MigrationsTableName) + + return drv, err } // Wait blocks until the database server is available. It does not verify that diff --git a/pkg/dbmate/db_test.go b/pkg/dbmate/db_test.go index 4b6a890..f3c8c5e 100644 --- a/pkg/dbmate/db_test.go +++ b/pkg/dbmate/db_test.go @@ -38,12 +38,26 @@ func TestNew(t *testing.T) { require.True(t, db.AutoDumpSchema) require.Equal(t, u.String(), db.DatabaseURL.String()) require.Equal(t, "./db/migrations", db.MigrationsDir) + require.Equal(t, "schema_migrations", db.MigrationsTableName) require.Equal(t, "./db/schema.sql", db.SchemaFile) require.False(t, db.WaitBefore) require.Equal(t, time.Second, db.WaitInterval) require.Equal(t, 60*time.Second, db.WaitTimeout) } +func TestGetDriver(t *testing.T) { + u := postgresTestURL(t) + db := New(u) + + drv, err := db.GetDriver() + require.NoError(t, err) + + // driver should have default migrations table set + pgDrv, ok := drv.(*PostgresDriver) + require.True(t, ok) + require.Equal(t, "schema_migrations", pgDrv.migrationsTableName) +} + func TestWait(t *testing.T) { u := postgresTestURL(t) db := newTestDB(t, u) @@ -242,7 +256,7 @@ func testMigrateURL(t *testing.T, u *url.URL) { require.NoError(t, err) // verify results - sqlDB, err := GetDriverOpen(u) + sqlDB, err := getDriverOpen(u) require.NoError(t, err) defer mustClose(sqlDB) @@ -275,7 +289,7 @@ func testUpURL(t *testing.T, u *url.URL) { require.NoError(t, err) // verify results - sqlDB, err := GetDriverOpen(u) + sqlDB, err := getDriverOpen(u) require.NoError(t, err) defer mustClose(sqlDB) @@ -308,7 +322,7 @@ func testRollbackURL(t *testing.T, u *url.URL) { require.NoError(t, err) // verify migration - sqlDB, err := GetDriverOpen(u) + sqlDB, err := getDriverOpen(u) require.NoError(t, err) defer mustClose(sqlDB) @@ -351,7 +365,7 @@ func testStatusURL(t *testing.T, u *url.URL) { require.NoError(t, err) // verify migration - sqlDB, err := GetDriverOpen(u) + sqlDB, err := getDriverOpen(u) require.NoError(t, err) defer mustClose(sqlDB) diff --git a/pkg/dbmate/driver.go b/pkg/dbmate/driver.go index 8a007b4..f5674ad 100644 --- a/pkg/dbmate/driver.go +++ b/pkg/dbmate/driver.go @@ -13,6 +13,7 @@ type Driver interface { CreateDatabase(*url.URL) error DropDatabase(*url.URL) error DumpSchema(*url.URL, *sql.DB) ([]byte, error) + SetMigrationsTableName(string) CreateMigrationsTable(*url.URL, *sql.DB) error SelectMigrations(*sql.DB, int) (map[string]bool, error) InsertMigration(Transaction, string) error @@ -34,18 +35,20 @@ type Transaction interface { QueryRow(query string, args ...interface{}) *sql.Row } -// GetDriver loads a database driver by name -func GetDriver(name string) (Driver, error) { - if val, ok := drivers[name]; ok { - return val, nil +// getDriver loads a database driver by name +func getDriver(name string) (Driver, error) { + if drv, ok := drivers[name]; ok { + drv.SetMigrationsTableName(DefaultMigrationsTableName) + + return drv, nil } return nil, fmt.Errorf("unsupported driver: %s", name) } -// GetDriverOpen is a shortcut for GetDriver(u.Scheme).Open(u) -func GetDriverOpen(u *url.URL) (*sql.DB, error) { - drv, err := GetDriver(u.Scheme) +// getDriverOpen is a shortcut for GetDriver(u.Scheme).Open(u) +func getDriverOpen(u *url.URL) (*sql.DB, error) { + drv, err := getDriver(u.Scheme) if err != nil { return nil, err } diff --git a/pkg/dbmate/driver_test.go b/pkg/dbmate/driver_test.go index d8f5f3e..0a23eee 100644 --- a/pkg/dbmate/driver_test.go +++ b/pkg/dbmate/driver_test.go @@ -7,21 +7,21 @@ import ( ) func TestGetDriver_Postgres(t *testing.T) { - drv, err := GetDriver("postgres") + drv, err := getDriver("postgres") require.NoError(t, err) - _, ok := drv.(PostgresDriver) + _, ok := drv.(*PostgresDriver) require.Equal(t, true, ok) } func TestGetDriver_MySQL(t *testing.T) { - drv, err := GetDriver("mysql") + drv, err := getDriver("mysql") require.NoError(t, err) - _, ok := drv.(MySQLDriver) + _, ok := drv.(*MySQLDriver) require.Equal(t, true, ok) } func TestGetDriver_Error(t *testing.T) { - drv, err := GetDriver("foo") + drv, err := getDriver("foo") require.EqualError(t, err, "unsupported driver: foo") require.Nil(t, drv) } diff --git a/pkg/dbmate/mysql.go b/pkg/dbmate/mysql.go index dd80e60..6443351 100644 --- a/pkg/dbmate/mysql.go +++ b/pkg/dbmate/mysql.go @@ -11,11 +11,12 @@ import ( ) func init() { - RegisterDriver(MySQLDriver{}, "mysql") + RegisterDriver(&MySQLDriver{}, "mysql") } // MySQLDriver provides top level database functions type MySQLDriver struct { + migrationsTableName string } func normalizeMySQLURL(u *url.URL) string { @@ -52,12 +53,17 @@ func normalizeMySQLURL(u *url.URL) string { return normalizedString } +// SetMigrationsTableName sets the schema migrations table name +func (drv *MySQLDriver) SetMigrationsTableName(name string) { + drv.migrationsTableName = name +} + // Open creates a new database connection -func (drv MySQLDriver) Open(u *url.URL) (*sql.DB, error) { +func (drv *MySQLDriver) Open(u *url.URL) (*sql.DB, error) { return sql.Open("mysql", normalizeMySQLURL(u)) } -func (drv MySQLDriver) openRootDB(u *url.URL) (*sql.DB, error) { +func (drv *MySQLDriver) openRootDB(u *url.URL) (*sql.DB, error) { // connect to no particular database rootURL := *u rootURL.Path = "/" @@ -65,14 +71,14 @@ func (drv MySQLDriver) openRootDB(u *url.URL) (*sql.DB, error) { return drv.Open(&rootURL) } -func mysqlQuoteIdentifier(str string) string { +func (drv *MySQLDriver) quoteIdentifier(str string) string { str = strings.Replace(str, "`", "\\`", -1) return fmt.Sprintf("`%s`", str) } // CreateDatabase creates the specified database -func (drv MySQLDriver) CreateDatabase(u *url.URL) error { +func (drv *MySQLDriver) CreateDatabase(u *url.URL) error { name := databaseName(u) fmt.Printf("Creating: %s\n", name) @@ -83,13 +89,13 @@ func (drv MySQLDriver) CreateDatabase(u *url.URL) error { defer mustClose(db) _, err = db.Exec(fmt.Sprintf("create database %s", - mysqlQuoteIdentifier(name))) + drv.quoteIdentifier(name))) return err } // DropDatabase drops the specified database (if it exists) -func (drv MySQLDriver) DropDatabase(u *url.URL) error { +func (drv *MySQLDriver) DropDatabase(u *url.URL) error { name := databaseName(u) fmt.Printf("Dropping: %s\n", name) @@ -100,12 +106,12 @@ func (drv MySQLDriver) DropDatabase(u *url.URL) error { defer mustClose(db) _, err = db.Exec(fmt.Sprintf("drop database if exists %s", - mysqlQuoteIdentifier(name))) + drv.quoteIdentifier(name))) return err } -func mysqldumpArgs(u *url.URL) []string { +func (drv *MySQLDriver) mysqldumpArgs(u *url.URL) []string { // generate CLI arguments args := []string{"--opt", "--routines", "--no-data", "--skip-dump-date", "--skip-add-drop-table"} @@ -131,10 +137,12 @@ func mysqldumpArgs(u *url.URL) []string { return args } -func mysqlSchemaMigrationsDump(db *sql.DB) ([]byte, error) { +func (drv *MySQLDriver) schemaMigrationsDump(db *sql.DB) ([]byte, error) { + migrationsTable := drv.quotedMigrationsTableName() + // load applied migrations migrations, err := queryColumn(db, - "select quote(version) from schema_migrations order by version asc") + fmt.Sprintf("select quote(version) from %s order by version asc", migrationsTable)) if err != nil { return nil, err } @@ -142,12 +150,13 @@ func mysqlSchemaMigrationsDump(db *sql.DB) ([]byte, error) { // build schema_migrations table data var buf bytes.Buffer buf.WriteString("\n--\n-- Dbmate schema migrations\n--\n\n" + - "LOCK TABLES `schema_migrations` WRITE;\n") + fmt.Sprintf("LOCK TABLES %s WRITE;\n", migrationsTable)) if len(migrations) > 0 { - buf.WriteString("INSERT INTO `schema_migrations` (version) VALUES\n (" + - strings.Join(migrations, "),\n (") + - ");\n") + buf.WriteString( + fmt.Sprintf("INSERT INTO %s (version) VALUES\n (", migrationsTable) + + strings.Join(migrations, "),\n (") + + ");\n") } buf.WriteString("UNLOCK TABLES;\n") @@ -156,13 +165,13 @@ func mysqlSchemaMigrationsDump(db *sql.DB) ([]byte, error) { } // DumpSchema returns the current database schema -func (drv MySQLDriver) DumpSchema(u *url.URL, db *sql.DB) ([]byte, error) { - schema, err := runCommand("mysqldump", mysqldumpArgs(u)...) +func (drv *MySQLDriver) DumpSchema(u *url.URL, db *sql.DB) ([]byte, error) { + schema, err := runCommand("mysqldump", drv.mysqldumpArgs(u)...) if err != nil { return nil, err } - migrations, err := mysqlSchemaMigrationsDump(db) + migrations, err := drv.schemaMigrationsDump(db) if err != nil { return nil, err } @@ -172,7 +181,7 @@ func (drv MySQLDriver) DumpSchema(u *url.URL, db *sql.DB) ([]byte, error) { } // DatabaseExists determines whether the database exists -func (drv MySQLDriver) DatabaseExists(u *url.URL) (bool, error) { +func (drv *MySQLDriver) DatabaseExists(u *url.URL) (bool, error) { name := databaseName(u) db, err := drv.openRootDB(u) @@ -192,17 +201,18 @@ func (drv MySQLDriver) DatabaseExists(u *url.URL) (bool, error) { } // CreateMigrationsTable creates the schema_migrations table -func (drv MySQLDriver) CreateMigrationsTable(u *url.URL, db *sql.DB) error { - _, err := db.Exec("create table if not exists schema_migrations " + - "(version varchar(255) primary key) character set latin1 collate latin1_bin") +func (drv *MySQLDriver) CreateMigrationsTable(u *url.URL, db *sql.DB) error { + _, err := db.Exec(fmt.Sprintf("create table if not exists %s "+ + "(version varchar(255) primary key) character set latin1 collate latin1_bin", + drv.quotedMigrationsTableName())) return err } // SelectMigrations returns a list of applied migrations // with an optional limit (in descending order) -func (drv MySQLDriver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, error) { - query := "select version from schema_migrations order by version desc" +func (drv *MySQLDriver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, error) { + query := fmt.Sprintf("select version from %s order by version desc", drv.quotedMigrationsTableName()) if limit >= 0 { query = fmt.Sprintf("%s limit %d", query, limit) } @@ -231,22 +241,26 @@ func (drv MySQLDriver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, } // InsertMigration adds a new migration record -func (drv MySQLDriver) InsertMigration(db Transaction, version string) error { - _, err := db.Exec("insert into schema_migrations (version) values (?)", version) +func (drv *MySQLDriver) InsertMigration(db Transaction, version string) error { + _, err := db.Exec( + fmt.Sprintf("insert into %s (version) values (?)", drv.quotedMigrationsTableName()), + version) return err } // DeleteMigration removes a migration record -func (drv MySQLDriver) DeleteMigration(db Transaction, version string) error { - _, err := db.Exec("delete from schema_migrations where version = ?", version) +func (drv *MySQLDriver) DeleteMigration(db Transaction, version string) error { + _, err := db.Exec( + fmt.Sprintf("delete from %s where version = ?", drv.quotedMigrationsTableName()), + version) return err } // Ping verifies a connection to the database server. It does not verify whether the // specified database exists. -func (drv MySQLDriver) Ping(u *url.URL) error { +func (drv *MySQLDriver) Ping(u *url.URL) error { db, err := drv.openRootDB(u) if err != nil { return err @@ -255,3 +269,7 @@ func (drv MySQLDriver) Ping(u *url.URL) error { return db.Ping() } + +func (drv *MySQLDriver) quotedMigrationsTableName() string { + return drv.quoteIdentifier(drv.migrationsTableName) +} diff --git a/pkg/dbmate/mysql_test.go b/pkg/dbmate/mysql_test.go index 8e34d6e..c5483cc 100644 --- a/pkg/dbmate/mysql_test.go +++ b/pkg/dbmate/mysql_test.go @@ -15,8 +15,15 @@ func mySQLTestURL(t *testing.T) *url.URL { return u } +func testMySQLDriver() *MySQLDriver { + drv := &MySQLDriver{} + drv.SetMigrationsTableName(DefaultMigrationsTableName) + + return drv +} + func prepTestMySQLDB(t *testing.T, u *url.URL) *sql.DB { - drv := MySQLDriver{} + drv := testMySQLDriver() // drop any existing database err := drv.DropDatabase(u) @@ -78,7 +85,7 @@ func TestNormalizeMySQLURLSocket(t *testing.T) { } func TestMySQLCreateDropDatabase(t *testing.T) { - drv := MySQLDriver{} + drv := testMySQLDriver() u := mySQLTestURL(t) // drop any existing database @@ -116,7 +123,9 @@ func TestMySQLCreateDropDatabase(t *testing.T) { } func TestMySQLDumpSchema(t *testing.T) { - drv := MySQLDriver{} + drv := testMySQLDriver() + drv.SetMigrationsTableName("test_migrations") + u := mySQLTestURL(t) // prepare database @@ -134,13 +143,13 @@ func TestMySQLDumpSchema(t *testing.T) { // DumpSchema should return schema schema, err := drv.DumpSchema(u, db) require.NoError(t, err) - require.Contains(t, string(schema), "CREATE TABLE `schema_migrations`") + require.Contains(t, string(schema), "CREATE TABLE `test_migrations`") require.Contains(t, string(schema), "\n-- Dump completed\n\n"+ "--\n"+ "-- Dbmate schema migrations\n"+ "--\n\n"+ - "LOCK TABLES `schema_migrations` WRITE;\n"+ - "INSERT INTO `schema_migrations` (version) VALUES\n"+ + "LOCK TABLES `test_migrations` WRITE;\n"+ + "INSERT INTO `test_migrations` (version) VALUES\n"+ " ('abc1'),\n"+ " ('abc2');\n"+ "UNLOCK TABLES;\n") @@ -156,7 +165,7 @@ func TestMySQLDumpSchema(t *testing.T) { } func TestMySQLDatabaseExists(t *testing.T) { - drv := MySQLDriver{} + drv := testMySQLDriver() u := mySQLTestURL(t) // drop any existing database @@ -179,7 +188,7 @@ func TestMySQLDatabaseExists(t *testing.T) { } func TestMySQLDatabaseExists_Error(t *testing.T) { - drv := MySQLDriver{} + drv := testMySQLDriver() u := mySQLTestURL(t) u.User = url.User("invalid") @@ -189,22 +198,25 @@ func TestMySQLDatabaseExists_Error(t *testing.T) { } func TestMySQLCreateMigrationsTable(t *testing.T) { - drv := MySQLDriver{} + drv := testMySQLDriver() + drv.SetMigrationsTableName("test_migrations") + u := mySQLTestURL(t) db := prepTestMySQLDB(t, u) defer mustClose(db) // migrations table should not exist count := 0 - err := db.QueryRow("select count(*) from schema_migrations").Scan(&count) - require.Regexp(t, "Table 'dbmate.schema_migrations' doesn't exist", err.Error()) + err := db.QueryRow("select count(*) from test_migrations").Scan(&count) + require.Error(t, err) + require.Regexp(t, "Table 'dbmate.test_migrations' doesn't exist", err.Error()) // create table err = drv.CreateMigrationsTable(u, db) require.NoError(t, err) // migrations table should exist - err = db.QueryRow("select count(*) from schema_migrations").Scan(&count) + err = db.QueryRow("select count(*) from test_migrations").Scan(&count) require.NoError(t, err) // create table should be idempotent @@ -213,7 +225,9 @@ func TestMySQLCreateMigrationsTable(t *testing.T) { } func TestMySQLSelectMigrations(t *testing.T) { - drv := MySQLDriver{} + drv := testMySQLDriver() + drv.SetMigrationsTableName("test_migrations") + u := mySQLTestURL(t) db := prepTestMySQLDB(t, u) defer mustClose(db) @@ -221,7 +235,7 @@ func TestMySQLSelectMigrations(t *testing.T) { err := drv.CreateMigrationsTable(u, db) require.NoError(t, err) - _, err = db.Exec(`insert into schema_migrations (version) + _, err = db.Exec(`insert into test_migrations (version) values ('abc2'), ('abc1'), ('abc3')`) require.NoError(t, err) @@ -240,7 +254,9 @@ func TestMySQLSelectMigrations(t *testing.T) { } func TestMySQLInsertMigration(t *testing.T) { - drv := MySQLDriver{} + drv := testMySQLDriver() + drv.SetMigrationsTableName("test_migrations") + u := mySQLTestURL(t) db := prepTestMySQLDB(t, u) defer mustClose(db) @@ -249,7 +265,7 @@ func TestMySQLInsertMigration(t *testing.T) { require.NoError(t, err) count := 0 - err = db.QueryRow("select count(*) from schema_migrations").Scan(&count) + err = db.QueryRow("select count(*) from test_migrations").Scan(&count) require.NoError(t, err) require.Equal(t, 0, count) @@ -257,14 +273,16 @@ func TestMySQLInsertMigration(t *testing.T) { err = drv.InsertMigration(db, "abc1") require.NoError(t, err) - err = db.QueryRow("select count(*) from schema_migrations where version = 'abc1'"). + err = db.QueryRow("select count(*) from test_migrations where version = 'abc1'"). Scan(&count) require.NoError(t, err) require.Equal(t, 1, count) } func TestMySQLDeleteMigration(t *testing.T) { - drv := MySQLDriver{} + drv := testMySQLDriver() + drv.SetMigrationsTableName("test_migrations") + u := mySQLTestURL(t) db := prepTestMySQLDB(t, u) defer mustClose(db) @@ -272,7 +290,7 @@ func TestMySQLDeleteMigration(t *testing.T) { err := drv.CreateMigrationsTable(u, db) require.NoError(t, err) - _, err = db.Exec(`insert into schema_migrations (version) + _, err = db.Exec(`insert into test_migrations (version) values ('abc1'), ('abc2')`) require.NoError(t, err) @@ -280,13 +298,13 @@ func TestMySQLDeleteMigration(t *testing.T) { require.NoError(t, err) count := 0 - err = db.QueryRow("select count(*) from schema_migrations").Scan(&count) + err = db.QueryRow("select count(*) from test_migrations").Scan(&count) require.NoError(t, err) require.Equal(t, 1, count) } func TestMySQLPing(t *testing.T) { - drv := MySQLDriver{} + drv := testMySQLDriver() u := mySQLTestURL(t) // drop any existing database @@ -303,3 +321,19 @@ func TestMySQLPing(t *testing.T) { require.Error(t, err) require.Contains(t, err.Error(), "connect: connection refused") } + +func TestMySQLQuotedMigrationsTableName(t *testing.T) { + t.Run("default name", func(t *testing.T) { + drv := testMySQLDriver() + name := drv.quotedMigrationsTableName() + require.Equal(t, "`schema_migrations`", name) + }) + + t.Run("custom name", func(t *testing.T) { + drv := testMySQLDriver() + drv.SetMigrationsTableName("fooMigrations") + + name := drv.quotedMigrationsTableName() + require.Equal(t, "`fooMigrations`", name) + }) +} diff --git a/pkg/dbmate/postgres.go b/pkg/dbmate/postgres.go index 7cbc8a0..172ccc3 100644 --- a/pkg/dbmate/postgres.go +++ b/pkg/dbmate/postgres.go @@ -11,12 +11,14 @@ import ( ) func init() { - RegisterDriver(PostgresDriver{}, "postgres") - RegisterDriver(PostgresDriver{}, "postgresql") + drv := &PostgresDriver{} + RegisterDriver(drv, "postgres") + RegisterDriver(drv, "postgresql") } // PostgresDriver provides top level database functions type PostgresDriver struct { + migrationsTableName string } func normalizePostgresURL(u *url.URL) *url.URL { @@ -78,12 +80,17 @@ func normalizePostgresURLForDump(u *url.URL) []string { return out } +// SetMigrationsTableName sets the schema migrations table name +func (drv *PostgresDriver) SetMigrationsTableName(name string) { + drv.migrationsTableName = name +} + // Open creates a new database connection -func (drv PostgresDriver) Open(u *url.URL) (*sql.DB, error) { +func (drv *PostgresDriver) Open(u *url.URL) (*sql.DB, error) { return sql.Open("postgres", normalizePostgresURL(u).String()) } -func (drv PostgresDriver) openPostgresDB(u *url.URL) (*sql.DB, error) { +func (drv *PostgresDriver) openPostgresDB(u *url.URL) (*sql.DB, error) { // connect to postgres database postgresURL := *u postgresURL.Path = "postgres" @@ -92,7 +99,7 @@ func (drv PostgresDriver) openPostgresDB(u *url.URL) (*sql.DB, error) { } // CreateDatabase creates the specified database -func (drv PostgresDriver) CreateDatabase(u *url.URL) error { +func (drv *PostgresDriver) CreateDatabase(u *url.URL) error { name := databaseName(u) fmt.Printf("Creating: %s\n", name) @@ -109,7 +116,7 @@ func (drv PostgresDriver) CreateDatabase(u *url.URL) error { } // DropDatabase drops the specified database (if it exists) -func (drv PostgresDriver) DropDatabase(u *url.URL) error { +func (drv *PostgresDriver) DropDatabase(u *url.URL) error { name := databaseName(u) fmt.Printf("Dropping: %s\n", name) @@ -125,8 +132,8 @@ func (drv PostgresDriver) DropDatabase(u *url.URL) error { return err } -func (drv PostgresDriver) postgresSchemaMigrationsDump(db *sql.DB) ([]byte, error) { - migrationsTable, err := drv.migrationsTableName(db) +func (drv *PostgresDriver) schemaMigrationsDump(db *sql.DB) ([]byte, error) { + migrationsTable, err := drv.quotedMigrationsTableName(db) if err != nil { return nil, err } @@ -152,7 +159,7 @@ func (drv PostgresDriver) postgresSchemaMigrationsDump(db *sql.DB) ([]byte, erro } // DumpSchema returns the current database schema -func (drv PostgresDriver) DumpSchema(u *url.URL, db *sql.DB) ([]byte, error) { +func (drv *PostgresDriver) DumpSchema(u *url.URL, db *sql.DB) ([]byte, error) { // load schema args := append([]string{"--format=plain", "--encoding=UTF8", "--schema-only", "--no-privileges", "--no-owner"}, normalizePostgresURLForDump(u)...) @@ -161,7 +168,7 @@ func (drv PostgresDriver) DumpSchema(u *url.URL, db *sql.DB) ([]byte, error) { return nil, err } - migrations, err := drv.postgresSchemaMigrationsDump(db) + migrations, err := drv.schemaMigrationsDump(db) if err != nil { return nil, err } @@ -171,7 +178,7 @@ func (drv PostgresDriver) DumpSchema(u *url.URL, db *sql.DB) ([]byte, error) { } // DatabaseExists determines whether the database exists -func (drv PostgresDriver) DatabaseExists(u *url.URL) (bool, error) { +func (drv *PostgresDriver) DatabaseExists(u *url.URL) (bool, error) { name := databaseName(u) db, err := drv.openPostgresDB(u) @@ -191,48 +198,45 @@ func (drv PostgresDriver) DatabaseExists(u *url.URL) (bool, error) { } // CreateMigrationsTable creates the schema_migrations table -func (drv PostgresDriver) CreateMigrationsTable(u *url.URL, db *sql.DB) error { - // get schema from URL search_path param - searchPath := strings.Split(u.Query().Get("search_path"), ",") - urlSchema := strings.TrimSpace(searchPath[0]) - if urlSchema == "" { - urlSchema = "public" - } - - // get *unquoted* current schema from database - dbSchema, err := queryRow(db, "select current_schema()") +func (drv *PostgresDriver) CreateMigrationsTable(u *url.URL, db *sql.DB) error { + schema, migrationsTable, err := drv.quotedMigrationsTableNameParts(db, u) if err != nil { return err } - // if urlSchema and dbSchema are not equal, the most likely explanation is that the schema - // has not yet been created - if urlSchema != dbSchema { - // in theory we could just execute this statement every time, but we do the comparison - // above in case the user doesn't have permissions to create schemas and the schema - // already exists - fmt.Printf("Creating schema: %s\n", urlSchema) - _, err = db.Exec("create schema if not exists " + pq.QuoteIdentifier(urlSchema)) - if err != nil { - return err - } + // first attempt at creating migrations table + createTableStmt := fmt.Sprintf("create table if not exists %s.%s", schema, migrationsTable) + + " (version varchar(255) primary key)" + _, err = db.Exec(createTableStmt) + if err == nil { + // table exists or created successfully + return nil } - migrationsTable, err := drv.migrationsTableName(db) + // catch 'schema does not exist' error + pqErr, ok := err.(*pq.Error) + if !ok || pqErr.Code != "3F000" { + // unknown error + return err + } + + // in theory we could attempt to create the schema every time, but we avoid that + // in case the user doesn't have permissions to create schemas + fmt.Printf("Creating schema: %s\n", schema) + _, err = db.Exec(fmt.Sprintf("create schema if not exists %s", schema)) if err != nil { return err } - _, err = db.Exec("create table if not exists " + migrationsTable + - " (version varchar(255) primary key)") - + // second and final attempt at creating migrations table + _, err = db.Exec(createTableStmt) return err } // SelectMigrations returns a list of applied migrations // with an optional limit (in descending order) -func (drv PostgresDriver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, error) { - migrationsTable, err := drv.migrationsTableName(db) +func (drv *PostgresDriver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, error) { + migrationsTable, err := drv.quotedMigrationsTableName(db) if err != nil { return nil, err } @@ -266,8 +270,8 @@ func (drv PostgresDriver) SelectMigrations(db *sql.DB, limit int) (map[string]bo } // InsertMigration adds a new migration record -func (drv PostgresDriver) InsertMigration(db Transaction, version string) error { - migrationsTable, err := drv.migrationsTableName(db) +func (drv *PostgresDriver) InsertMigration(db Transaction, version string) error { + migrationsTable, err := drv.quotedMigrationsTableName(db) if err != nil { return err } @@ -278,8 +282,8 @@ func (drv PostgresDriver) InsertMigration(db Transaction, version string) error } // DeleteMigration removes a migration record -func (drv PostgresDriver) DeleteMigration(db Transaction, version string) error { - migrationsTable, err := drv.migrationsTableName(db) +func (drv *PostgresDriver) DeleteMigration(db Transaction, version string) error { + migrationsTable, err := drv.quotedMigrationsTableName(db) if err != nil { return err } @@ -291,7 +295,7 @@ func (drv PostgresDriver) DeleteMigration(db Transaction, version string) error // Ping verifies a connection to the database server. It does not verify whether the // specified database exists. -func (drv PostgresDriver) Ping(u *url.URL) error { +func (drv *PostgresDriver) Ping(u *url.URL) error { // attempt connection to primary database, not "postgres" database // to support servers with no "postgres" database // (see https://github.com/amacneil/dbmate/issues/78) @@ -306,7 +310,7 @@ func (drv PostgresDriver) Ping(u *url.URL) error { return nil } - // ignore 'database "foo" does not exist' error + // ignore 'database does not exist' error pqErr, ok := err.(*pq.Error) if ok && pqErr.Code == "3D000" { return nil @@ -315,17 +319,53 @@ func (drv PostgresDriver) Ping(u *url.URL) error { return err } -func (drv PostgresDriver) migrationsTableName(db Transaction) (string, error) { - // get current schema - schema, err := queryRow(db, "select quote_ident(current_schema())") +func (drv *PostgresDriver) quotedMigrationsTableName(db Transaction) (string, error) { + schema, name, err := drv.quotedMigrationsTableNameParts(db, nil) if err != nil { return "", err } - // if the search path is empty, or does not contain a valid schema, default to public + return schema + "." + name, nil +} + +func (drv *PostgresDriver) quotedMigrationsTableNameParts(db Transaction, u *url.URL) (string, string, error) { + schema := "" + tableNameParts := strings.Split(drv.migrationsTableName, ".") + if len(tableNameParts) > 1 { + // schema specified as part of table name + schema, tableNameParts = tableNameParts[0], tableNameParts[1:] + } + + if schema == "" && u != nil { + // no schema specified with table name, try URL search path if available + searchPath := strings.Split(u.Query().Get("search_path"), ",") + schema = strings.TrimSpace(searchPath[0]) + } + + var err error + if schema == "" { + // if no URL available, use current schema + // this is a hack because we don't always have the URL context available + schema, err = queryValue(db, "select current_schema()") + if err != nil { + return "", "", err + } + } + + // fall back to public schema as last resort if schema == "" { schema = "public" } - return schema + ".schema_migrations", nil + // quote all parts + // use server rather than client to do this to avoid unnecessary quotes + // (which would change schema.sql diff) + tableNameParts = append([]string{schema}, tableNameParts...) + quotedNameParts, err := queryColumn(db, "select quote_ident(unnest($1::text[]))", pq.Array(tableNameParts)) + if err != nil { + return "", "", err + } + + // if more than one part, we already have a schema + return quotedNameParts[0], strings.Join(quotedNameParts[1:], "."), nil } diff --git a/pkg/dbmate/postgres_test.go b/pkg/dbmate/postgres_test.go index aafb177..a2e63d0 100644 --- a/pkg/dbmate/postgres_test.go +++ b/pkg/dbmate/postgres_test.go @@ -15,8 +15,15 @@ func postgresTestURL(t *testing.T) *url.URL { return u } +func testPostgresDriver() *PostgresDriver { + drv := &PostgresDriver{} + drv.SetMigrationsTableName(DefaultMigrationsTableName) + + return drv +} + func prepTestPostgresDB(t *testing.T, u *url.URL) *sql.DB { - drv := PostgresDriver{} + drv := testPostgresDriver() // drop any existing database err := drv.DropDatabase(u) @@ -87,7 +94,7 @@ func TestNormalizePostgresURLForDump(t *testing.T) { } func TestPostgresCreateDropDatabase(t *testing.T) { - drv := PostgresDriver{} + drv := testPostgresDriver() u := postgresTestURL(t) // drop any existing database @@ -125,45 +132,80 @@ func TestPostgresCreateDropDatabase(t *testing.T) { } func TestPostgresDumpSchema(t *testing.T) { - drv := PostgresDriver{} - u := postgresTestURL(t) + t.Run("default migrations table", func(t *testing.T) { + drv := testPostgresDriver() + u := postgresTestURL(t) - // prepare database - db := prepTestPostgresDB(t, u) - defer mustClose(db) - err := drv.CreateMigrationsTable(u, db) - require.NoError(t, err) + // prepare database + db := prepTestPostgresDB(t, u) + defer mustClose(db) + err := drv.CreateMigrationsTable(u, db) + require.NoError(t, err) - // insert migration - err = drv.InsertMigration(db, "abc1") - require.NoError(t, err) - err = drv.InsertMigration(db, "abc2") - require.NoError(t, err) + // insert migration + err = drv.InsertMigration(db, "abc1") + require.NoError(t, err) + err = drv.InsertMigration(db, "abc2") + require.NoError(t, err) - // DumpSchema should return schema - schema, err := drv.DumpSchema(u, db) - require.NoError(t, err) - require.Contains(t, string(schema), "CREATE TABLE public.schema_migrations") - require.Contains(t, string(schema), "\n--\n"+ - "-- PostgreSQL database dump complete\n"+ - "--\n\n\n"+ - "--\n"+ - "-- Dbmate schema migrations\n"+ - "--\n\n"+ - "INSERT INTO public.schema_migrations (version) VALUES\n"+ - " ('abc1'),\n"+ - " ('abc2');\n") + // DumpSchema should return schema + schema, err := drv.DumpSchema(u, db) + require.NoError(t, err) + require.Contains(t, string(schema), "CREATE TABLE public.schema_migrations") + require.Contains(t, string(schema), "\n--\n"+ + "-- PostgreSQL database dump complete\n"+ + "--\n\n\n"+ + "--\n"+ + "-- Dbmate schema migrations\n"+ + "--\n\n"+ + "INSERT INTO public.schema_migrations (version) VALUES\n"+ + " ('abc1'),\n"+ + " ('abc2');\n") - // DumpSchema should return error if command fails - u.Path = "/fakedb" - schema, err = drv.DumpSchema(u, db) - require.Nil(t, schema) - require.EqualError(t, err, "pg_dump: [archiver (db)] connection to database "+ - "\"fakedb\" failed: FATAL: database \"fakedb\" does not exist") + // DumpSchema should return error if command fails + u.Path = "/fakedb" + schema, err = drv.DumpSchema(u, db) + require.Nil(t, schema) + require.EqualError(t, err, "pg_dump: [archiver (db)] connection to database "+ + "\"fakedb\" failed: FATAL: database \"fakedb\" does not exist") + }) + + t.Run("custom migrations table with schema", func(t *testing.T) { + drv := testPostgresDriver() + drv.SetMigrationsTableName("camelSchema.testMigrations") + + u := postgresTestURL(t) + + // prepare database + db := prepTestPostgresDB(t, u) + defer mustClose(db) + err := drv.CreateMigrationsTable(u, db) + require.NoError(t, err) + + // insert migration + err = drv.InsertMigration(db, "abc1") + require.NoError(t, err) + err = drv.InsertMigration(db, "abc2") + require.NoError(t, err) + + // DumpSchema should return schema + schema, err := drv.DumpSchema(u, db) + require.NoError(t, err) + require.Contains(t, string(schema), "CREATE TABLE \"camelSchema\".\"testMigrations\"") + require.Contains(t, string(schema), "\n--\n"+ + "-- PostgreSQL database dump complete\n"+ + "--\n\n\n"+ + "--\n"+ + "-- Dbmate schema migrations\n"+ + "--\n\n"+ + "INSERT INTO \"camelSchema\".\"testMigrations\" (version) VALUES\n"+ + " ('abc1'),\n"+ + " ('abc2');\n") + }) } func TestPostgresDatabaseExists(t *testing.T) { - drv := PostgresDriver{} + drv := testPostgresDriver() u := postgresTestURL(t) // drop any existing database @@ -186,7 +228,7 @@ func TestPostgresDatabaseExists(t *testing.T) { } func TestPostgresDatabaseExists_Error(t *testing.T) { - drv := PostgresDriver{} + drv := testPostgresDriver() u := postgresTestURL(t) u.User = url.User("invalid") @@ -197,9 +239,8 @@ func TestPostgresDatabaseExists_Error(t *testing.T) { } func TestPostgresCreateMigrationsTable(t *testing.T) { - drv := PostgresDriver{} - t.Run("default schema", func(t *testing.T) { + drv := testPostgresDriver() u := postgresTestURL(t) db := prepTestPostgresDB(t, u) defer mustClose(db) @@ -223,39 +264,81 @@ func TestPostgresCreateMigrationsTable(t *testing.T) { require.NoError(t, err) }) - t.Run("custom schema", func(t *testing.T) { - u, err := url.Parse(postgresTestURL(t).String() + "&search_path=foo") + t.Run("custom search path", func(t *testing.T) { + drv := testPostgresDriver() + drv.SetMigrationsTableName("testMigrations") + + u, err := url.Parse(postgresTestURL(t).String() + "&search_path=camelFoo") require.NoError(t, err) db := prepTestPostgresDB(t, u) defer mustClose(db) // delete schema - _, err = db.Exec("drop schema if exists foo") + _, err = db.Exec("drop schema if exists \"camelFoo\"") require.NoError(t, err) - // drop any schema_migrations table in public schema - _, err = db.Exec("drop table if exists public.schema_migrations") + // drop any testMigrations table in public schema + _, err = db.Exec("drop table if exists public.\"testMigrations\"") require.NoError(t, err) // migrations table should not exist in either schema count := 0 - err = db.QueryRow("select count(*) from foo.schema_migrations").Scan(&count) + err = db.QueryRow("select count(*) from \"camelFoo\".\"testMigrations\"").Scan(&count) require.Error(t, err) - require.Equal(t, "pq: relation \"foo.schema_migrations\" does not exist", err.Error()) - err = db.QueryRow("select count(*) from public.schema_migrations").Scan(&count) + require.Equal(t, "pq: relation \"camelFoo.testMigrations\" does not exist", err.Error()) + err = db.QueryRow("select count(*) from public.\"testMigrations\"").Scan(&count) require.Error(t, err) - require.Equal(t, "pq: relation \"public.schema_migrations\" does not exist", err.Error()) + require.Equal(t, "pq: relation \"public.testMigrations\" does not exist", err.Error()) // create table err = drv.CreateMigrationsTable(u, db) require.NoError(t, err) - // foo schema should be created, and migrations table should exist only in foo schema - err = db.QueryRow("select count(*) from foo.schema_migrations").Scan(&count) + // camelFoo schema should be created, and migrations table should exist only in camelFoo schema + err = db.QueryRow("select count(*) from \"camelFoo\".\"testMigrations\"").Scan(&count) require.NoError(t, err) - err = db.QueryRow("select count(*) from public.schema_migrations").Scan(&count) + err = db.QueryRow("select count(*) from public.\"testMigrations\"").Scan(&count) require.Error(t, err) - require.Equal(t, "pq: relation \"public.schema_migrations\" does not exist", err.Error()) + require.Equal(t, "pq: relation \"public.testMigrations\" does not exist", err.Error()) + + // create table should be idempotent + err = drv.CreateMigrationsTable(u, db) + require.NoError(t, err) + }) + + t.Run("custom schema", func(t *testing.T) { + drv := testPostgresDriver() + drv.SetMigrationsTableName("camelSchema.testMigrations") + + u, err := url.Parse(postgresTestURL(t).String() + "&search_path=foo") + require.NoError(t, err) + db := prepTestPostgresDB(t, u) + defer mustClose(db) + + // delete schemas + _, err = db.Exec("drop schema if exists foo") + require.NoError(t, err) + _, err = db.Exec("drop schema if exists \"camelSchema\"") + require.NoError(t, err) + + // migrations table should not exist + count := 0 + err = db.QueryRow("select count(*) from \"camelSchema\".\"testMigrations\"").Scan(&count) + require.Error(t, err) + require.Equal(t, "pq: relation \"camelSchema.testMigrations\" does not exist", err.Error()) + + // create table + err = drv.CreateMigrationsTable(u, db) + require.NoError(t, err) + + // camelSchema should be created, and testMigrations table should exist + err = db.QueryRow("select count(*) from \"camelSchema\".\"testMigrations\"").Scan(&count) + require.NoError(t, err) + // testMigrations table should not exist in foo schema because + // schema specified with migrations table name takes priority over search path + err = db.QueryRow("select count(*) from foo.\"testMigrations\"").Scan(&count) + require.Error(t, err) + require.Equal(t, "pq: relation \"foo.testMigrations\" does not exist", err.Error()) // create table should be idempotent err = drv.CreateMigrationsTable(u, db) @@ -264,7 +347,9 @@ func TestPostgresCreateMigrationsTable(t *testing.T) { } func TestPostgresSelectMigrations(t *testing.T) { - drv := PostgresDriver{} + drv := testPostgresDriver() + drv.SetMigrationsTableName("test_migrations") + u := postgresTestURL(t) db := prepTestPostgresDB(t, u) defer mustClose(db) @@ -272,7 +357,7 @@ func TestPostgresSelectMigrations(t *testing.T) { err := drv.CreateMigrationsTable(u, db) require.NoError(t, err) - _, err = db.Exec(`insert into public.schema_migrations (version) + _, err = db.Exec(`insert into public.test_migrations (version) values ('abc2'), ('abc1'), ('abc3')`) require.NoError(t, err) @@ -291,7 +376,9 @@ func TestPostgresSelectMigrations(t *testing.T) { } func TestPostgresInsertMigration(t *testing.T) { - drv := PostgresDriver{} + drv := testPostgresDriver() + drv.SetMigrationsTableName("test_migrations") + u := postgresTestURL(t) db := prepTestPostgresDB(t, u) defer mustClose(db) @@ -300,7 +387,7 @@ func TestPostgresInsertMigration(t *testing.T) { require.NoError(t, err) count := 0 - err = db.QueryRow("select count(*) from public.schema_migrations").Scan(&count) + err = db.QueryRow("select count(*) from public.test_migrations").Scan(&count) require.NoError(t, err) require.Equal(t, 0, count) @@ -308,14 +395,16 @@ func TestPostgresInsertMigration(t *testing.T) { err = drv.InsertMigration(db, "abc1") require.NoError(t, err) - err = db.QueryRow("select count(*) from public.schema_migrations where version = 'abc1'"). + err = db.QueryRow("select count(*) from public.test_migrations where version = 'abc1'"). Scan(&count) require.NoError(t, err) require.Equal(t, 1, count) } func TestPostgresDeleteMigration(t *testing.T) { - drv := PostgresDriver{} + drv := testPostgresDriver() + drv.SetMigrationsTableName("test_migrations") + u := postgresTestURL(t) db := prepTestPostgresDB(t, u) defer mustClose(db) @@ -323,7 +412,7 @@ func TestPostgresDeleteMigration(t *testing.T) { err := drv.CreateMigrationsTable(u, db) require.NoError(t, err) - _, err = db.Exec(`insert into public.schema_migrations (version) + _, err = db.Exec(`insert into public.test_migrations (version) values ('abc1'), ('abc2')`) require.NoError(t, err) @@ -331,13 +420,13 @@ func TestPostgresDeleteMigration(t *testing.T) { require.NoError(t, err) count := 0 - err = db.QueryRow("select count(*) from public.schema_migrations").Scan(&count) + err = db.QueryRow("select count(*) from public.test_migrations").Scan(&count) require.NoError(t, err) require.Equal(t, 1, count) } func TestPostgresPing(t *testing.T) { - drv := PostgresDriver{} + drv := testPostgresDriver() u := postgresTestURL(t) // drop any existing database @@ -355,15 +444,15 @@ func TestPostgresPing(t *testing.T) { require.Contains(t, err.Error(), "connect: connection refused") } -func TestMigrationsTableName(t *testing.T) { - drv := PostgresDriver{} +func TestPostgresQuotedMigrationsTableName(t *testing.T) { + drv := testPostgresDriver() t.Run("default schema", func(t *testing.T) { u := postgresTestURL(t) db := prepTestPostgresDB(t, u) defer mustClose(db) - name, err := drv.migrationsTableName(db) + name, err := drv.quotedMigrationsTableName(db) require.NoError(t, err) require.Equal(t, "public.schema_migrations", name) }) @@ -379,14 +468,14 @@ func TestMigrationsTableName(t *testing.T) { require.NoError(t, err) _, err = db.Exec("drop schema if exists bar") require.NoError(t, err) - name, err := drv.migrationsTableName(db) + name, err := drv.quotedMigrationsTableName(db) require.NoError(t, err) require.Equal(t, "public.schema_migrations", name) // if "foo" schema exists, it should be used _, err = db.Exec("create schema foo") require.NoError(t, err) - name, err = drv.migrationsTableName(db) + name, err = drv.quotedMigrationsTableName(db) require.NoError(t, err) require.Equal(t, "foo.schema_migrations", name) }) @@ -401,8 +490,76 @@ func TestMigrationsTableName(t *testing.T) { _, err := db.Exec("select pg_catalog.set_config('search_path', '', false)") require.NoError(t, err) - name, err := drv.migrationsTableName(db) + name, err := drv.quotedMigrationsTableName(db) require.NoError(t, err) require.Equal(t, "public.schema_migrations", name) }) + + t.Run("custom table name", func(t *testing.T) { + u := postgresTestURL(t) + db := prepTestPostgresDB(t, u) + defer mustClose(db) + + drv.SetMigrationsTableName("simple_name") + name, err := drv.quotedMigrationsTableName(db) + require.NoError(t, err) + require.Equal(t, "public.simple_name", name) + }) + + t.Run("custom table name quoted", func(t *testing.T) { + u := postgresTestURL(t) + db := prepTestPostgresDB(t, u) + defer mustClose(db) + + // this table name will need quoting + drv.SetMigrationsTableName("camelCase") + name, err := drv.quotedMigrationsTableName(db) + require.NoError(t, err) + require.Equal(t, "public.\"camelCase\"", name) + }) + + t.Run("custom table name with custom schema", func(t *testing.T) { + u, err := url.Parse(postgresTestURL(t).String() + "&search_path=foo") + require.NoError(t, err) + db := prepTestPostgresDB(t, u) + defer mustClose(db) + + _, err = db.Exec("create schema if not exists foo") + require.NoError(t, err) + + drv.SetMigrationsTableName("simple_name") + name, err := drv.quotedMigrationsTableName(db) + require.NoError(t, err) + require.Equal(t, "foo.simple_name", name) + }) + + t.Run("custom table name overrides schema", func(t *testing.T) { + u, err := url.Parse(postgresTestURL(t).String() + "&search_path=foo") + require.NoError(t, err) + db := prepTestPostgresDB(t, u) + defer mustClose(db) + + _, err = db.Exec("create schema if not exists foo") + require.NoError(t, err) + _, err = db.Exec("create schema if not exists bar") + require.NoError(t, err) + + // if schema is specified as part of table name, it should override search_path + drv.SetMigrationsTableName("bar.simple_name") + name, err := drv.quotedMigrationsTableName(db) + require.NoError(t, err) + require.Equal(t, "bar.simple_name", name) + + // schema and table name should be quoted if necessary + drv.SetMigrationsTableName("barName.camelTable") + name, err = drv.quotedMigrationsTableName(db) + require.NoError(t, err) + require.Equal(t, "\"barName\".\"camelTable\"", name) + + // more than 2 components is unexpected but we will quote and pass it along anyway + drv.SetMigrationsTableName("whyWould.i.doThis") + name, err = drv.quotedMigrationsTableName(db) + require.NoError(t, err) + require.Equal(t, "\"whyWould\".i.\"doThis\"", name) + }) } diff --git a/pkg/dbmate/sqlite.go b/pkg/dbmate/sqlite.go index 3ef10dd..f762d4c 100644 --- a/pkg/dbmate/sqlite.go +++ b/pkg/dbmate/sqlite.go @@ -11,16 +11,19 @@ import ( "regexp" "strings" + "github.com/lib/pq" _ "github.com/mattn/go-sqlite3" // sqlite driver for database/sql ) func init() { - RegisterDriver(SQLiteDriver{}, "sqlite") - RegisterDriver(SQLiteDriver{}, "sqlite3") + drv := &SQLiteDriver{} + RegisterDriver(drv, "sqlite") + RegisterDriver(drv, "sqlite3") } // SQLiteDriver provides top level database functions type SQLiteDriver struct { + migrationsTableName string } func sqlitePath(u *url.URL) string { @@ -31,13 +34,18 @@ func sqlitePath(u *url.URL) string { return str } +// SetMigrationsTableName sets the schema migrations table name +func (drv *SQLiteDriver) SetMigrationsTableName(name string) { + drv.migrationsTableName = name +} + // Open creates a new database connection -func (drv SQLiteDriver) Open(u *url.URL) (*sql.DB, error) { +func (drv *SQLiteDriver) Open(u *url.URL) (*sql.DB, error) { return sql.Open("sqlite3", sqlitePath(u)) } // CreateDatabase creates the specified database -func (drv SQLiteDriver) CreateDatabase(u *url.URL) error { +func (drv *SQLiteDriver) CreateDatabase(u *url.URL) error { fmt.Printf("Creating: %s\n", sqlitePath(u)) db, err := drv.Open(u) @@ -50,7 +58,7 @@ func (drv SQLiteDriver) CreateDatabase(u *url.URL) error { } // DropDatabase drops the specified database (if it exists) -func (drv SQLiteDriver) DropDatabase(u *url.URL) error { +func (drv *SQLiteDriver) DropDatabase(u *url.URL) error { path := sqlitePath(u) fmt.Printf("Dropping: %s\n", path) @@ -65,36 +73,39 @@ func (drv SQLiteDriver) DropDatabase(u *url.URL) error { return os.Remove(path) } -func sqliteSchemaMigrationsDump(db *sql.DB) ([]byte, error) { +func (drv *SQLiteDriver) schemaMigrationsDump(db *sql.DB) ([]byte, error) { + migrationsTable := drv.quotedMigrationsTableName() + // load applied migrations migrations, err := queryColumn(db, - "select quote(version) from schema_migrations order by version asc") + fmt.Sprintf("select quote(version) from %s order by version asc", migrationsTable)) if err != nil { return nil, err } - // build schema_migrations table data + // build schema migrations table data var buf bytes.Buffer buf.WriteString("-- Dbmate schema migrations\n") if len(migrations) > 0 { - buf.WriteString("INSERT INTO schema_migrations (version) VALUES\n (" + - strings.Join(migrations, "),\n (") + - ");\n") + buf.WriteString( + fmt.Sprintf("INSERT INTO %s (version) VALUES\n (", migrationsTable) + + strings.Join(migrations, "),\n (") + + ");\n") } return buf.Bytes(), nil } // DumpSchema returns the current database schema -func (drv SQLiteDriver) DumpSchema(u *url.URL, db *sql.DB) ([]byte, error) { +func (drv *SQLiteDriver) DumpSchema(u *url.URL, db *sql.DB) ([]byte, error) { path := sqlitePath(u) schema, err := runCommand("sqlite3", path, ".schema") if err != nil { return nil, err } - migrations, err := sqliteSchemaMigrationsDump(db) + migrations, err := drv.schemaMigrationsDump(db) if err != nil { return nil, err } @@ -104,7 +115,7 @@ func (drv SQLiteDriver) DumpSchema(u *url.URL, db *sql.DB) ([]byte, error) { } // DatabaseExists determines whether the database exists -func (drv SQLiteDriver) DatabaseExists(u *url.URL) (bool, error) { +func (drv *SQLiteDriver) DatabaseExists(u *url.URL) (bool, error) { _, err := os.Stat(sqlitePath(u)) if os.IsNotExist(err) { return false, nil @@ -116,18 +127,19 @@ func (drv SQLiteDriver) DatabaseExists(u *url.URL) (bool, error) { return true, nil } -// CreateMigrationsTable creates the schema_migrations table -func (drv SQLiteDriver) CreateMigrationsTable(u *url.URL, db *sql.DB) error { - _, err := db.Exec("create table if not exists schema_migrations " + - "(version varchar(255) primary key)") +// CreateMigrationsTable creates the schema migrations table +func (drv *SQLiteDriver) CreateMigrationsTable(u *url.URL, db *sql.DB) error { + _, err := db.Exec( + fmt.Sprintf("create table if not exists %s ", drv.quotedMigrationsTableName()) + + "(version varchar(255) primary key)") return err } // SelectMigrations returns a list of applied migrations // with an optional limit (in descending order) -func (drv SQLiteDriver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, error) { - query := "select version from schema_migrations order by version desc" +func (drv *SQLiteDriver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, error) { + query := fmt.Sprintf("select version from %s order by version desc", drv.quotedMigrationsTableName()) if limit >= 0 { query = fmt.Sprintf("%s limit %d", query, limit) } @@ -156,15 +168,19 @@ func (drv SQLiteDriver) SelectMigrations(db *sql.DB, limit int) (map[string]bool } // InsertMigration adds a new migration record -func (drv SQLiteDriver) InsertMigration(db Transaction, version string) error { - _, err := db.Exec("insert into schema_migrations (version) values (?)", version) +func (drv *SQLiteDriver) InsertMigration(db Transaction, version string) error { + _, err := db.Exec( + fmt.Sprintf("insert into %s (version) values (?)", drv.quotedMigrationsTableName()), + version) return err } // DeleteMigration removes a migration record -func (drv SQLiteDriver) DeleteMigration(db Transaction, version string) error { - _, err := db.Exec("delete from schema_migrations where version = ?", version) +func (drv *SQLiteDriver) DeleteMigration(db Transaction, version string) error { + _, err := db.Exec( + fmt.Sprintf("delete from %s where version = ?", drv.quotedMigrationsTableName()), + version) return err } @@ -172,7 +188,7 @@ func (drv SQLiteDriver) DeleteMigration(db Transaction, version string) error { // Ping verifies a connection to the database. Due to the way SQLite works, by // testing whether the database is valid, it will automatically create the database // if it does not already exist. -func (drv SQLiteDriver) Ping(u *url.URL) error { +func (drv *SQLiteDriver) Ping(u *url.URL) error { db, err := drv.Open(u) if err != nil { return err @@ -181,3 +197,14 @@ func (drv SQLiteDriver) Ping(u *url.URL) error { return db.Ping() } + +func (drv *SQLiteDriver) quotedMigrationsTableName() string { + return drv.quoteIdentifier(drv.migrationsTableName) +} + +// quoteIdentifier quotes a table or column name +// we fall back to lib/pq implementation since both use ansi standard (double quotes) +// and mattn/go-sqlite3 doesn't provide a sqlite-specific equivalent +func (drv *SQLiteDriver) quoteIdentifier(s string) string { + return pq.QuoteIdentifier(s) +} diff --git a/pkg/dbmate/sqlite_test.go b/pkg/dbmate/sqlite_test.go index 3d32889..5602188 100644 --- a/pkg/dbmate/sqlite_test.go +++ b/pkg/dbmate/sqlite_test.go @@ -18,8 +18,15 @@ func sqliteTestURL(t *testing.T) *url.URL { return u } +func testSQLiteDriver() *SQLiteDriver { + drv := &SQLiteDriver{} + drv.SetMigrationsTableName(DefaultMigrationsTableName) + + return drv +} + func prepTestSQLiteDB(t *testing.T, u *url.URL) *sql.DB { - drv := SQLiteDriver{} + drv := testSQLiteDriver() // drop any existing database err := drv.DropDatabase(u) @@ -37,7 +44,7 @@ func prepTestSQLiteDB(t *testing.T, u *url.URL) *sql.DB { } func TestSQLiteCreateDropDatabase(t *testing.T) { - drv := SQLiteDriver{} + drv := testSQLiteDriver() u := sqliteTestURL(t) path := sqlitePath(u) @@ -64,7 +71,9 @@ func TestSQLiteCreateDropDatabase(t *testing.T) { } func TestSQLiteDumpSchema(t *testing.T) { - drv := SQLiteDriver{} + drv := testSQLiteDriver() + drv.SetMigrationsTableName("test_migrations") + u := sqliteTestURL(t) // prepare database @@ -82,9 +91,9 @@ func TestSQLiteDumpSchema(t *testing.T) { // DumpSchema should return schema schema, err := drv.DumpSchema(u, db) require.NoError(t, err) - require.Contains(t, string(schema), "CREATE TABLE schema_migrations") + require.Contains(t, string(schema), "CREATE TABLE IF NOT EXISTS \"test_migrations\"") require.Contains(t, string(schema), ");\n-- Dbmate schema migrations\n"+ - "INSERT INTO schema_migrations (version) VALUES\n"+ + "INSERT INTO \"test_migrations\" (version) VALUES\n"+ " ('abc1'),\n"+ " ('abc2');\n") @@ -97,7 +106,7 @@ func TestSQLiteDumpSchema(t *testing.T) { } func TestSQLiteDatabaseExists(t *testing.T) { - drv := SQLiteDriver{} + drv := testSQLiteDriver() u := sqliteTestURL(t) // drop any existing database @@ -120,31 +129,61 @@ func TestSQLiteDatabaseExists(t *testing.T) { } func TestSQLiteCreateMigrationsTable(t *testing.T) { - drv := SQLiteDriver{} - u := sqliteTestURL(t) - db := prepTestSQLiteDB(t, u) - defer mustClose(db) + t.Run("default table", func(t *testing.T) { + drv := testSQLiteDriver() + u := sqliteTestURL(t) + db := prepTestSQLiteDB(t, u) + defer mustClose(db) - // migrations table should not exist - count := 0 - err := db.QueryRow("select count(*) from schema_migrations").Scan(&count) - require.Regexp(t, "no such table: schema_migrations", err.Error()) + // migrations table should not exist + count := 0 + err := db.QueryRow("select count(*) from schema_migrations").Scan(&count) + require.Regexp(t, "no such table: schema_migrations", err.Error()) - // create table - err = drv.CreateMigrationsTable(u, db) - require.NoError(t, err) + // create table + err = drv.CreateMigrationsTable(u, db) + require.NoError(t, err) - // migrations table should exist - err = db.QueryRow("select count(*) from schema_migrations").Scan(&count) - require.NoError(t, err) + // migrations table should exist + err = db.QueryRow("select count(*) from schema_migrations").Scan(&count) + require.NoError(t, err) - // create table should be idempotent - err = drv.CreateMigrationsTable(u, db) - require.NoError(t, err) + // create table should be idempotent + err = drv.CreateMigrationsTable(u, db) + require.NoError(t, err) + }) + + t.Run("custom table", func(t *testing.T) { + drv := testSQLiteDriver() + drv.SetMigrationsTableName("test_migrations") + + u := sqliteTestURL(t) + db := prepTestSQLiteDB(t, u) + defer mustClose(db) + + // migrations table should not exist + count := 0 + err := db.QueryRow("select count(*) from test_migrations").Scan(&count) + require.Regexp(t, "no such table: test_migrations", err.Error()) + + // create table + err = drv.CreateMigrationsTable(u, db) + require.NoError(t, err) + + // migrations table should exist + err = db.QueryRow("select count(*) from test_migrations").Scan(&count) + require.NoError(t, err) + + // create table should be idempotent + err = drv.CreateMigrationsTable(u, db) + require.NoError(t, err) + }) } func TestSQLiteSelectMigrations(t *testing.T) { - drv := SQLiteDriver{} + drv := testSQLiteDriver() + drv.SetMigrationsTableName("test_migrations") + u := sqliteTestURL(t) db := prepTestSQLiteDB(t, u) defer mustClose(db) @@ -152,7 +191,7 @@ func TestSQLiteSelectMigrations(t *testing.T) { err := drv.CreateMigrationsTable(u, db) require.NoError(t, err) - _, err = db.Exec(`insert into schema_migrations (version) + _, err = db.Exec(`insert into test_migrations (version) values ('abc2'), ('abc1'), ('abc3')`) require.NoError(t, err) @@ -171,7 +210,9 @@ func TestSQLiteSelectMigrations(t *testing.T) { } func TestSQLiteInsertMigration(t *testing.T) { - drv := SQLiteDriver{} + drv := testSQLiteDriver() + drv.SetMigrationsTableName("test_migrations") + u := sqliteTestURL(t) db := prepTestSQLiteDB(t, u) defer mustClose(db) @@ -180,7 +221,7 @@ func TestSQLiteInsertMigration(t *testing.T) { require.NoError(t, err) count := 0 - err = db.QueryRow("select count(*) from schema_migrations").Scan(&count) + err = db.QueryRow("select count(*) from test_migrations").Scan(&count) require.NoError(t, err) require.Equal(t, 0, count) @@ -188,14 +229,16 @@ func TestSQLiteInsertMigration(t *testing.T) { err = drv.InsertMigration(db, "abc1") require.NoError(t, err) - err = db.QueryRow("select count(*) from schema_migrations where version = 'abc1'"). + err = db.QueryRow("select count(*) from test_migrations where version = 'abc1'"). Scan(&count) require.NoError(t, err) require.Equal(t, 1, count) } func TestSQLiteDeleteMigration(t *testing.T) { - drv := SQLiteDriver{} + drv := testSQLiteDriver() + drv.SetMigrationsTableName("test_migrations") + u := sqliteTestURL(t) db := prepTestSQLiteDB(t, u) defer mustClose(db) @@ -203,7 +246,7 @@ func TestSQLiteDeleteMigration(t *testing.T) { err := drv.CreateMigrationsTable(u, db) require.NoError(t, err) - _, err = db.Exec(`insert into schema_migrations (version) + _, err = db.Exec(`insert into test_migrations (version) values ('abc1'), ('abc2')`) require.NoError(t, err) @@ -211,13 +254,13 @@ func TestSQLiteDeleteMigration(t *testing.T) { require.NoError(t, err) count := 0 - err = db.QueryRow("select count(*) from schema_migrations").Scan(&count) + err = db.QueryRow("select count(*) from test_migrations").Scan(&count) require.NoError(t, err) require.Equal(t, 1, count) } func TestSQLitePing(t *testing.T) { - drv := SQLiteDriver{} + drv := testSQLiteDriver() u := sqliteTestURL(t) path := sqlitePath(u) @@ -249,3 +292,19 @@ func TestSQLitePing(t *testing.T) { err = drv.Ping(u) require.EqualError(t, err, "unable to open database file: is a directory") } + +func TestSQLiteQuotedMigrationsTableName(t *testing.T) { + t.Run("default name", func(t *testing.T) { + drv := testSQLiteDriver() + name := drv.quotedMigrationsTableName() + require.Equal(t, `"schema_migrations"`, name) + }) + + t.Run("custom name", func(t *testing.T) { + drv := testSQLiteDriver() + drv.SetMigrationsTableName("fooMigrations") + + name := drv.quotedMigrationsTableName() + require.Equal(t, `"fooMigrations"`, name) + }) +} diff --git a/pkg/dbmate/utils.go b/pkg/dbmate/utils.go index cee8476..41913fc 100644 --- a/pkg/dbmate/utils.go +++ b/pkg/dbmate/utils.go @@ -104,8 +104,8 @@ func trimLeadingSQLComments(data []byte) ([]byte, error) { // queryColumn runs a SQL statement and returns a slice of strings // it is assumed that the statement returns only one column // e.g. schema_migrations table -func queryColumn(db Transaction, query string) ([]string, error) { - rows, err := db.Query(query) +func queryColumn(db Transaction, query string, args ...interface{}) ([]string, error) { + rows, err := db.Query(query, args...) if err != nil { return nil, err } @@ -128,12 +128,12 @@ func queryColumn(db Transaction, query string) ([]string, error) { return result, nil } -// queryRow runs a SQL statement and returns a single string +// queryValue runs a SQL statement and returns a single string // it is assumed that the statement returns only one row and one column // sql NULL is returned as empty string -func queryRow(db Transaction, query string) (string, error) { +func queryValue(db Transaction, query string, args ...interface{}) (string, error) { var result sql.NullString - err := db.QueryRow(query).Scan(&result) + err := db.QueryRow(query, args...).Scan(&result) if err != nil || !result.Valid { return "", err } diff --git a/pkg/dbmate/utils_test.go b/pkg/dbmate/utils_test.go index e8a86a7..63db33c 100644 --- a/pkg/dbmate/utils_test.go +++ b/pkg/dbmate/utils_test.go @@ -1,9 +1,11 @@ package dbmate import ( + "database/sql" "net/url" "testing" + "github.com/lib/pq" "github.com/stretchr/testify/require" ) @@ -33,3 +35,24 @@ func TestTrimLeadingSQLComments(t *testing.T) { require.NoError(t, err) require.Equal(t, "real stuff\n-- end\n", string(out)) } + +func TestQueryColumn(t *testing.T) { + u := postgresTestURL(t) + db, err := sql.Open("postgres", u.String()) + require.NoError(t, err) + + val, err := queryColumn(db, "select concat('foo_', unnest($1::text[]))", + pq.Array([]string{"hi", "there"})) + require.NoError(t, err) + require.Equal(t, []string{"foo_hi", "foo_there"}, val) +} + +func TestQueryValue(t *testing.T) { + u := postgresTestURL(t) + db, err := sql.Open("postgres", u.String()) + require.NoError(t, err) + + val, err := queryValue(db, "select $1::int + $2::int", "5", 2) + require.NoError(t, err) + require.Equal(t, "7", val) +} From 61771e386d75a18943149b14c03a9cfa304d363c Mon Sep 17 00:00:00 2001 From: Adrian Macneil Date: Thu, 19 Nov 2020 15:04:42 +1300 Subject: [PATCH 29/55] Refactor drivers into separate packages (#179) `dbmate` package was starting to get a bit polluted. This PR migrates each driver into a separate package, with clean separation between each. In addition: * Drivers are now initialized with a URL, avoiding the need to pass `*url.URL` to every method * Sqlite supports a cleaner syntax for relative paths * Driver tests now load their test URL from environment variables Public API of `dbmate` package has not changed (no changes to `main` package). --- .golangci.yml | 4 + Makefile | 12 +- README.md | 8 +- docker-compose.yml | 7 +- go.sum | 1 + main.go | 4 + pkg/dbmate/db.go | 283 ++++++++------ pkg/dbmate/db_test.go | 353 +++++++++--------- pkg/dbmate/driver.go | 65 ++-- pkg/dbmate/driver_test.go | 27 -- pkg/dbmate/{migrations.go => migration.go} | 0 .../{migrations_test.go => migration_test.go} | 0 pkg/dbmate/utils_test.go | 58 --- pkg/{dbmate/utils.go => dbutil/dbutil.go} | 64 ++-- pkg/dbutil/dbutil_test.go | 58 +++ .../clickhouse}/clickhouse.go | 104 +++--- .../clickhouse}/clickhouse_test.go | 207 +++++----- pkg/{dbmate => driver/mysql}/mysql.go | 110 +++--- pkg/{dbmate => driver/mysql}/mysql_test.go | 229 ++++++------ pkg/{dbmate => driver/postgres}/postgres.go | 119 +++--- .../postgres}/postgres_test.go | 278 +++++++------- pkg/{dbmate => driver/sqlite}/sqlite.go | 96 ++--- pkg/{dbmate => driver/sqlite}/sqlite_test.go | 186 +++++---- 23 files changed, 1195 insertions(+), 1078 deletions(-) delete mode 100644 pkg/dbmate/driver_test.go rename pkg/dbmate/{migrations.go => migration.go} (100%) rename pkg/dbmate/{migrations_test.go => migration_test.go} (100%) delete mode 100644 pkg/dbmate/utils_test.go rename pkg/{dbmate/utils.go => dbutil/dbutil.go} (65%) create mode 100644 pkg/dbutil/dbutil_test.go rename pkg/{dbmate => driver/clickhouse}/clickhouse.go (69%) rename pkg/{dbmate => driver/clickhouse}/clickhouse_test.go (61%) rename pkg/{dbmate => driver/mysql}/mysql.go (64%) rename pkg/{dbmate => driver/mysql}/mysql_test.go (53%) rename pkg/{dbmate => driver/postgres}/postgres.go (70%) rename pkg/{dbmate => driver/postgres}/postgres_test.go (71%) rename pkg/{dbmate => driver/sqlite}/sqlite.go (59%) rename pkg/{dbmate => driver/sqlite}/sqlite_test.go (60%) diff --git a/.golangci.yml b/.golangci.yml index fcad107..ac76638 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -26,3 +26,7 @@ linters-settings: local-prefixes: github.com/amacneil/dbmate misspell: locale: US + +issues: + include: + - EXC0002 diff --git a/Makefile b/Makefile index ef9dfc0..70963e0 100644 --- a/Makefile +++ b/Makefile @@ -3,14 +3,14 @@ LDFLAGS := -ldflags '-s' # statically link binaries (to support alpine + scratch containers) STATICLDFLAGS := -ldflags '-s -extldflags "-static"' # avoid building code that is incompatible with static linking -TAGS := -tags netgo,osusergo,sqlite_omit_load_extension +TAGS := -tags netgo,osusergo,sqlite_omit_load_extension,sqlite_json .PHONY: all -all: build lint test +all: build test lint .PHONY: test test: - go test -v $(TAGS) $(STATICLDFLAGS) ./... + go test -p 1 $(TAGS) $(STATICLDFLAGS) ./... .PHONY: fix fix: @@ -22,9 +22,9 @@ lint: .PHONY: wait wait: - dist/dbmate-linux-amd64 -e MYSQL_URL wait - dist/dbmate-linux-amd64 -e POSTGRESQL_URL wait - dist/dbmate-linux-amd64 -e CLICKHOUSE_URL wait + dist/dbmate-linux-amd64 -e CLICKHOUSE_TEST_URL wait + dist/dbmate-linux-amd64 -e MYSQL_TEST_URL wait + dist/dbmate-linux-amd64 -e POSTGRES_TEST_URL wait .PHONY: clean clean: diff --git a/README.md b/README.md index 4abbba4..cca110d 100644 --- a/README.md +++ b/README.md @@ -152,16 +152,16 @@ DATABASE_URL="postgres://username:password@127.0.0.1:5432/database_name?search_p **SQLite** -SQLite databases are stored on the filesystem, so you do not need to specify a host. By default, files are relative to the current directory. For example, the following will create a database at `./db/database_name.sqlite3`: +SQLite databases are stored on the filesystem, so you do not need to specify a host. By default, files are relative to the current directory. For example, the following will create a database at `./db/database.sqlite3`: ```sh -DATABASE_URL="sqlite:///db/database_name.sqlite3" +DATABASE_URL="sqlite:db/database.sqlite3" ``` -To specify an absolute path, add an additional forward slash to the path. The following will create a database at `/tmp/database_name.sqlite3`: +To specify an absolute path, add a forward slash to the path. The following will create a database at `/tmp/database.sqlite3`: ```sh -DATABASE_URL="sqlite:////tmp/database_name.sqlite3" +DATABASE_URL="sqlite:/tmp/database.sqlite3" ``` **ClickHouse** diff --git a/docker-compose.yml b/docker-compose.yml index 09d8331..f2a4754 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,9 +11,10 @@ services: - postgres - clickhouse environment: - MYSQL_URL: mysql://root:root@mysql/dbmate - POSTGRESQL_URL: postgres://postgres:postgres@postgres/dbmate?sslmode=disable - CLICKHOUSE_URL: clickhouse://clickhouse:9000?database=dbmate + CLICKHOUSE_TEST_URL: clickhouse://clickhouse:9000?database=dbmate_test + MYSQL_TEST_URL: mysql://root:root@mysql/dbmate_test + POSTGRES_TEST_URL: postgres://postgres:postgres@postgres/dbmate_test?sslmode=disable + SQLITE_TEST_URL: sqlite3:/tmp/dbmate_test.sqlite3 dbmate: build: diff --git a/go.sum b/go.sum index 142d1fc..3fd6f97 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,4 @@ +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/ClickHouse/clickhouse-go v1.4.3 h1:iAFMa2UrQdR5bHJ2/yaSLffZkxpcOYQMCUuKeNXGdqc= github.com/ClickHouse/clickhouse-go v1.4.3/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI= diff --git a/main.go b/main.go index 04a777f..2ed5875 100644 --- a/main.go +++ b/main.go @@ -11,6 +11,10 @@ import ( "github.com/urfave/cli/v2" "github.com/amacneil/dbmate/pkg/dbmate" + _ "github.com/amacneil/dbmate/pkg/driver/clickhouse" + _ "github.com/amacneil/dbmate/pkg/driver/mysql" + _ "github.com/amacneil/dbmate/pkg/driver/postgres" + _ "github.com/amacneil/dbmate/pkg/driver/sqlite" ) func main() { diff --git a/pkg/dbmate/db.go b/pkg/dbmate/db.go index 2ddc0be..3350c02 100644 --- a/pkg/dbmate/db.go +++ b/pkg/dbmate/db.go @@ -2,6 +2,7 @@ package dbmate import ( "database/sql" + "errors" "fmt" "io/ioutil" "net/url" @@ -10,6 +11,8 @@ import ( "regexp" "sort" "time" + + "github.com/amacneil/dbmate/pkg/dbutil" ) // DefaultMigrationsDir specifies default directory to find migration files @@ -43,9 +46,10 @@ type DB struct { // migrationFileRegexp pattern for valid migration files var migrationFileRegexp = regexp.MustCompile(`^\d.*\.sql$`) -type statusResult struct { - filename string - applied bool +// StatusResult represents an available migration status +type StatusResult struct { + Filename string + Applied bool } // New initializes a new dbmate database @@ -62,16 +66,23 @@ func New(databaseURL *url.URL) *DB { } } -// GetDriver loads the required database driver +// GetDriver initializes the appropriate database driver func (db *DB) GetDriver() (Driver, error) { - drv, err := getDriver(db.DatabaseURL.Scheme) - if err != nil { - return nil, err + if db.DatabaseURL == nil || db.DatabaseURL.Scheme == "" { + return nil, errors.New("invalid url") } - drv.SetMigrationsTableName(db.MigrationsTableName) + driverFunc := drivers[db.DatabaseURL.Scheme] + if driverFunc == nil { + return nil, fmt.Errorf("unsupported driver: %s", db.DatabaseURL.Scheme) + } - return drv, err + config := DriverConfig{ + DatabaseURL: db.DatabaseURL, + MigrationsTableName: db.MigrationsTableName, + } + + return driverFunc(config), nil } // Wait blocks until the database server is available. It does not verify that @@ -82,8 +93,12 @@ func (db *DB) Wait() error { return err } + return db.wait(drv) +} + +func (db *DB) wait(drv Driver) error { // attempt connection to database server - err = drv.Ping(db.DatabaseURL) + err := drv.Ping() if err == nil { // connection successful return nil @@ -95,7 +110,7 @@ func (db *DB) Wait() error { time.Sleep(db.WaitInterval) // attempt connection to database server - err = drv.Ping(db.DatabaseURL) + err = drv.Ping() if err == nil { // connection successful fmt.Print("\n") @@ -110,82 +125,91 @@ func (db *DB) Wait() error { // CreateAndMigrate creates the database (if necessary) and runs migrations func (db *DB) CreateAndMigrate() error { - if db.WaitBefore { - err := db.Wait() - if err != nil { - return err - } - } - drv, err := db.GetDriver() if err != nil { return err } + if db.WaitBefore { + err := db.wait(drv) + if err != nil { + return err + } + } + // create database if it does not already exist // skip this step if we cannot determine status // (e.g. user does not have list database permission) - exists, err := drv.DatabaseExists(db.DatabaseURL) + exists, err := drv.DatabaseExists() if err == nil && !exists { - if err := drv.CreateDatabase(db.DatabaseURL); err != nil { + if err := drv.CreateDatabase(); err != nil { return err } } // migrate - return db.Migrate() + return db.migrate(drv) } // Create creates the current database func (db *DB) Create() error { - if db.WaitBefore { - err := db.Wait() - if err != nil { - return err - } - } - drv, err := db.GetDriver() if err != nil { return err } - return drv.CreateDatabase(db.DatabaseURL) + if db.WaitBefore { + err := db.wait(drv) + if err != nil { + return err + } + } + + return drv.CreateDatabase() } // Drop drops the current database (if it exists) func (db *DB) Drop() error { - if db.WaitBefore { - err := db.Wait() - if err != nil { - return err - } - } - drv, err := db.GetDriver() if err != nil { return err } - return drv.DropDatabase(db.DatabaseURL) -} - -// DumpSchema writes the current database schema to a file -func (db *DB) DumpSchema() error { if db.WaitBefore { - err := db.Wait() + err := db.wait(drv) if err != nil { return err } } - drv, sqlDB, err := db.openDatabaseForMigration() + return drv.DropDatabase() +} + +// DumpSchema writes the current database schema to a file +func (db *DB) DumpSchema() error { + drv, err := db.GetDriver() if err != nil { return err } - defer mustClose(sqlDB) - schema, err := drv.DumpSchema(db.DatabaseURL, sqlDB) + return db.dumpSchema(drv) +} + +func (db *DB) dumpSchema(drv Driver) error { + if db.WaitBefore { + err := db.wait(drv) + if err != nil { + return err + } + } + + sqlDB, err := db.openDatabaseForMigration(drv) + if err != nil { + return err + } + defer dbutil.MustClose(sqlDB) + + schema, err := drv.DumpSchema(sqlDB) if err != nil { return err } @@ -201,6 +225,15 @@ func (db *DB) DumpSchema() error { return ioutil.WriteFile(db.SchemaFile, schema, 0644) } +// ensureDir creates a directory if it does not already exist +func ensureDir(dir string) error { + if err := os.MkdirAll(dir, 0755); err != nil { + return fmt.Errorf("unable to create directory `%s`", dir) + } + + return nil +} + const migrationTemplate = "-- migrate:up\n\n\n-- migrate:down\n\n" // NewMigration creates a new migration file @@ -231,13 +264,13 @@ func (db *DB) NewMigration(name string) error { return err } - defer mustClose(file) + defer dbutil.MustClose(file) _, err = file.WriteString(migrationTemplate) return err } -func doTransaction(db *sql.DB, txFunc func(Transaction) error) error { - tx, err := db.Begin() +func doTransaction(sqlDB *sql.DB, txFunc func(dbutil.Transaction) error) error { + tx, err := sqlDB.Begin() if err != nil { return err } @@ -253,27 +286,31 @@ func doTransaction(db *sql.DB, txFunc func(Transaction) error) error { return tx.Commit() } -func (db *DB) openDatabaseForMigration() (Driver, *sql.DB, error) { - drv, err := db.GetDriver() +func (db *DB) openDatabaseForMigration(drv Driver) (*sql.DB, error) { + sqlDB, err := drv.Open() if err != nil { - return nil, nil, err + return nil, err } - sqlDB, err := drv.Open(db.DatabaseURL) - if err != nil { - return nil, nil, err + if err := drv.CreateMigrationsTable(sqlDB); err != nil { + dbutil.MustClose(sqlDB) + return nil, err } - if err := drv.CreateMigrationsTable(db.DatabaseURL, sqlDB); err != nil { - mustClose(sqlDB) - return nil, nil, err - } - - return drv, sqlDB, nil + return sqlDB, nil } // Migrate migrates database to the latest version func (db *DB) Migrate() error { + drv, err := db.GetDriver() + if err != nil { + return err + } + + return db.migrate(drv) +} + +func (db *DB) migrate(drv Driver) error { files, err := findMigrationFiles(db.MigrationsDir, migrationFileRegexp) if err != nil { return err @@ -284,17 +321,17 @@ func (db *DB) Migrate() error { } if db.WaitBefore { - err := db.Wait() + err := db.wait(drv) if err != nil { return err } } - drv, sqlDB, err := db.openDatabaseForMigration() + sqlDB, err := db.openDatabaseForMigration(drv) if err != nil { return err } - defer mustClose(sqlDB) + defer dbutil.MustClose(sqlDB) applied, err := drv.SelectMigrations(sqlDB, -1) if err != nil { @@ -315,7 +352,7 @@ func (db *DB) Migrate() error { return err } - execMigration := func(tx Transaction) error { + execMigration := func(tx dbutil.Transaction) error { // run actual migration result, err := tx.Exec(up.Contents) if err != nil { @@ -343,12 +380,23 @@ func (db *DB) Migrate() error { // automatically update schema file, silence errors if db.AutoDumpSchema { - _ = db.DumpSchema() + _ = db.dumpSchema(drv) } return nil } +func printVerbose(result sql.Result) { + lastInsertID, err := result.LastInsertId() + if err == nil { + fmt.Printf("Last insert ID: %d\n", lastInsertID) + } + rowsAffected, err := result.RowsAffected() + if err == nil { + fmt.Printf("Rows affected: %d\n", rowsAffected) + } +} + func findMigrationFiles(dir string, re *regexp.Regexp) ([]string, error) { files, err := ioutil.ReadDir(dir) if err != nil { @@ -400,18 +448,23 @@ func migrationVersion(filename string) string { // Rollback rolls back the most recent migration func (db *DB) Rollback() error { + drv, err := db.GetDriver() + if err != nil { + return err + } + if db.WaitBefore { - err := db.Wait() + err := db.wait(drv) if err != nil { return err } } - drv, sqlDB, err := db.openDatabaseForMigration() + sqlDB, err := db.openDatabaseForMigration(drv) if err != nil { return err } - defer mustClose(sqlDB) + defer dbutil.MustClose(sqlDB) applied, err := drv.SelectMigrations(sqlDB, 1) if err != nil { @@ -439,7 +492,7 @@ func (db *DB) Rollback() error { return err } - execMigration := func(tx Transaction) error { + execMigration := func(tx dbutil.Transaction) error { // rollback migration result, err := tx.Exec(down.Contents) if err != nil { @@ -466,53 +519,20 @@ func (db *DB) Rollback() error { // automatically update schema file, silence errors if db.AutoDumpSchema { - _ = db.DumpSchema() + _ = db.dumpSchema(drv) } return nil } -func checkMigrationsStatus(db *DB) ([]statusResult, error) { - files, err := findMigrationFiles(db.MigrationsDir, migrationFileRegexp) - if err != nil { - return nil, err - } - - if len(files) == 0 { - return nil, fmt.Errorf("no migration files found") - } - - drv, sqlDB, err := db.openDatabaseForMigration() - if err != nil { - return nil, err - } - defer mustClose(sqlDB) - - applied, err := drv.SelectMigrations(sqlDB, -1) - if err != nil { - return nil, err - } - - var results []statusResult - - for _, filename := range files { - ver := migrationVersion(filename) - res := statusResult{filename: filename} - if ok := applied[ver]; ok { - res.applied = true - } else { - res.applied = false - } - - results = append(results, res) - } - - return results, nil -} - // Status shows the status of all migrations func (db *DB) Status(quiet bool) (int, error) { - results, err := checkMigrationsStatus(db) + drv, err := db.GetDriver() + if err != nil { + return -1, err + } + + results, err := db.CheckMigrationsStatus(drv) if err != nil { return -1, err } @@ -521,11 +541,11 @@ func (db *DB) Status(quiet bool) (int, error) { var line string for _, res := range results { - if res.applied { - line = fmt.Sprintf("[X] %s", res.filename) + if res.Applied { + line = fmt.Sprintf("[X] %s", res.Filename) totalApplied++ } else { - line = fmt.Sprintf("[ ] %s", res.filename) + line = fmt.Sprintf("[ ] %s", res.Filename) } if !quiet { fmt.Println(line) @@ -541,3 +561,42 @@ func (db *DB) Status(quiet bool) (int, error) { return totalPending, nil } + +// CheckMigrationsStatus returns the status of all available mgirations +func (db *DB) CheckMigrationsStatus(drv Driver) ([]StatusResult, error) { + files, err := findMigrationFiles(db.MigrationsDir, migrationFileRegexp) + if err != nil { + return nil, err + } + + if len(files) == 0 { + return nil, fmt.Errorf("no migration files found") + } + + sqlDB, err := db.openDatabaseForMigration(drv) + if err != nil { + return nil, err + } + defer dbutil.MustClose(sqlDB) + + applied, err := drv.SelectMigrations(sqlDB, -1) + if err != nil { + return nil, err + } + + var results []StatusResult + + for _, filename := range files { + ver := migrationVersion(filename) + res := StatusResult{Filename: filename} + if ok := applied[ver]; ok { + res.Applied = true + } else { + res.Applied = false + } + + results = append(results, res) + } + + return results, nil +} diff --git a/pkg/dbmate/db_test.go b/pkg/dbmate/db_test.go index f3c8c5e..0c07457 100644 --- a/pkg/dbmate/db_test.go +++ b/pkg/dbmate/db_test.go @@ -1,4 +1,4 @@ -package dbmate +package dbmate_test import ( "io/ioutil" @@ -8,13 +8,19 @@ import ( "testing" "time" + "github.com/amacneil/dbmate/pkg/dbmate" + "github.com/amacneil/dbmate/pkg/dbutil" + _ "github.com/amacneil/dbmate/pkg/driver/mysql" + _ "github.com/amacneil/dbmate/pkg/driver/postgres" + _ "github.com/amacneil/dbmate/pkg/driver/sqlite" + "github.com/kami-zh/go-capturer" "github.com/stretchr/testify/require" ) var testdataDir string -func newTestDB(t *testing.T, u *url.URL) *DB { +func newTestDB(t *testing.T, u *url.URL) *dbmate.DB { var err error // only chdir once, because testdata is relative to current directory @@ -26,17 +32,16 @@ func newTestDB(t *testing.T, u *url.URL) *DB { require.NoError(t, err) } - db := New(u) + db := dbmate.New(u) db.AutoDumpSchema = false return db } func TestNew(t *testing.T) { - u := postgresTestURL(t) - db := New(u) + db := dbmate.New(dbutil.MustParseURL("foo:test")) require.True(t, db.AutoDumpSchema) - require.Equal(t, u.String(), db.DatabaseURL.String()) + require.Equal(t, "foo:test", db.DatabaseURL.String()) require.Equal(t, "./db/migrations", db.MigrationsDir) require.Equal(t, "schema_migrations", db.MigrationsTableName) require.Equal(t, "./db/schema.sql", db.SchemaFile) @@ -46,20 +51,30 @@ func TestNew(t *testing.T) { } func TestGetDriver(t *testing.T) { - u := postgresTestURL(t) - db := New(u) + t.Run("missing URL", func(t *testing.T) { + db := dbmate.New(nil) + drv, err := db.GetDriver() + require.Nil(t, drv) + require.EqualError(t, err, "invalid url") + }) - drv, err := db.GetDriver() - require.NoError(t, err) + t.Run("missing schema", func(t *testing.T) { + db := dbmate.New(dbutil.MustParseURL("//hi")) + drv, err := db.GetDriver() + require.Nil(t, drv) + require.EqualError(t, err, "invalid url") + }) - // driver should have default migrations table set - pgDrv, ok := drv.(*PostgresDriver) - require.True(t, ok) - require.Equal(t, "schema_migrations", pgDrv.migrationsTableName) + t.Run("invalid driver", func(t *testing.T) { + db := dbmate.New(dbutil.MustParseURL("foo://bar")) + drv, err := db.GetDriver() + require.EqualError(t, err, "unsupported driver: foo") + require.Nil(t, drv) + }) } func TestWait(t *testing.T) { - u := postgresTestURL(t) + u := dbutil.MustParseURL(os.Getenv("POSTGRES_TEST_URL")) db := newTestDB(t, u) // speed up our retry loop for testing @@ -83,7 +98,7 @@ func TestWait(t *testing.T) { } func TestDumpSchema(t *testing.T) { - u := postgresTestURL(t) + u := dbutil.MustParseURL(os.Getenv("POSTGRES_TEST_URL")) db := newTestDB(t, u) // create custom schema file directory @@ -120,7 +135,7 @@ func TestDumpSchema(t *testing.T) { } func TestAutoDumpSchema(t *testing.T) { - u := postgresTestURL(t) + u := dbutil.MustParseURL(os.Getenv("POSTGRES_TEST_URL")) db := newTestDB(t, u) db.AutoDumpSchema = true @@ -177,7 +192,7 @@ func checkWaitCalled(t *testing.T, u *url.URL, command func() error) { } func testWaitBefore(t *testing.T, verbose bool) { - u := postgresTestURL(t) + u := dbutil.MustParseURL(os.Getenv("POSTGRES_TEST_URL")) db := newTestDB(t, u) db.Verbose = verbose db.WaitBefore = true @@ -234,173 +249,173 @@ Rows affected: 0`) Rows affected: 0`) } -func testURLs(t *testing.T) []*url.URL { +func testURLs() []*url.URL { return []*url.URL{ - postgresTestURL(t), - mySQLTestURL(t), - sqliteTestURL(t), + dbutil.MustParseURL(os.Getenv("MYSQL_TEST_URL")), + dbutil.MustParseURL(os.Getenv("POSTGRES_TEST_URL")), + dbutil.MustParseURL(os.Getenv("SQLITE_TEST_URL")), } } -func testMigrateURL(t *testing.T, u *url.URL) { - db := newTestDB(t, u) - - // drop and recreate database - err := db.Drop() - require.NoError(t, err) - err = db.Create() - require.NoError(t, err) - - // migrate - err = db.Migrate() - require.NoError(t, err) - - // verify results - sqlDB, err := getDriverOpen(u) - require.NoError(t, err) - defer mustClose(sqlDB) - - count := 0 - err = sqlDB.QueryRow(`select count(*) from schema_migrations - where version = '20151129054053'`).Scan(&count) - require.NoError(t, err) - require.Equal(t, 1, count) - - err = sqlDB.QueryRow("select count(*) from users").Scan(&count) - require.NoError(t, err) - require.Equal(t, 1, count) -} - func TestMigrate(t *testing.T) { - for _, u := range testURLs(t) { - testMigrateURL(t, u) + for _, u := range testURLs() { + t.Run(u.Scheme, func(t *testing.T) { + db := newTestDB(t, u) + drv, err := db.GetDriver() + require.NoError(t, err) + + // drop and recreate database + err = db.Drop() + require.NoError(t, err) + err = db.Create() + require.NoError(t, err) + + // migrate + err = db.Migrate() + require.NoError(t, err) + + // verify results + sqlDB, err := drv.Open() + require.NoError(t, err) + defer dbutil.MustClose(sqlDB) + + count := 0 + err = sqlDB.QueryRow(`select count(*) from schema_migrations + where version = '20151129054053'`).Scan(&count) + require.NoError(t, err) + require.Equal(t, 1, count) + + err = sqlDB.QueryRow("select count(*) from users").Scan(&count) + require.NoError(t, err) + require.Equal(t, 1, count) + }) } } -func testUpURL(t *testing.T, u *url.URL) { - db := newTestDB(t, u) - - // drop database - err := db.Drop() - require.NoError(t, err) - - // create and migrate - err = db.CreateAndMigrate() - require.NoError(t, err) - - // verify results - sqlDB, err := getDriverOpen(u) - require.NoError(t, err) - defer mustClose(sqlDB) - - count := 0 - err = sqlDB.QueryRow(`select count(*) from schema_migrations - where version = '20151129054053'`).Scan(&count) - require.NoError(t, err) - require.Equal(t, 1, count) - - err = sqlDB.QueryRow("select count(*) from users").Scan(&count) - require.NoError(t, err) - require.Equal(t, 1, count) -} - func TestUp(t *testing.T) { - for _, u := range testURLs(t) { - testUpURL(t, u) + for _, u := range testURLs() { + t.Run(u.Scheme, func(t *testing.T) { + db := newTestDB(t, u) + drv, err := db.GetDriver() + require.NoError(t, err) + + // drop database + err = db.Drop() + require.NoError(t, err) + + // create and migrate + err = db.CreateAndMigrate() + require.NoError(t, err) + + // verify results + sqlDB, err := drv.Open() + require.NoError(t, err) + defer dbutil.MustClose(sqlDB) + + count := 0 + err = sqlDB.QueryRow(`select count(*) from schema_migrations + where version = '20151129054053'`).Scan(&count) + require.NoError(t, err) + require.Equal(t, 1, count) + + err = sqlDB.QueryRow("select count(*) from users").Scan(&count) + require.NoError(t, err) + require.Equal(t, 1, count) + }) } } -func testRollbackURL(t *testing.T, u *url.URL) { - db := newTestDB(t, u) - - // drop, recreate, and migrate database - err := db.Drop() - require.NoError(t, err) - err = db.Create() - require.NoError(t, err) - err = db.Migrate() - require.NoError(t, err) - - // verify migration - sqlDB, err := getDriverOpen(u) - require.NoError(t, err) - defer mustClose(sqlDB) - - count := 0 - err = sqlDB.QueryRow(`select count(*) from schema_migrations - where version = '20151129054053'`).Scan(&count) - require.NoError(t, err) - require.Equal(t, 1, count) - - err = sqlDB.QueryRow("select count(*) from posts").Scan(&count) - require.Nil(t, err) - - // rollback - err = db.Rollback() - require.NoError(t, err) - - // verify rollback - err = sqlDB.QueryRow("select count(*) from schema_migrations").Scan(&count) - require.NoError(t, err) - require.Equal(t, 1, count) - - err = sqlDB.QueryRow("select count(*) from posts").Scan(&count) - require.NotNil(t, err) - require.Regexp(t, "(does not exist|doesn't exist|no such table)", err.Error()) -} - func TestRollback(t *testing.T) { - for _, u := range testURLs(t) { - testRollbackURL(t, u) + for _, u := range testURLs() { + t.Run(u.Scheme, func(t *testing.T) { + db := newTestDB(t, u) + drv, err := db.GetDriver() + require.NoError(t, err) + + // drop, recreate, and migrate database + err = db.Drop() + require.NoError(t, err) + err = db.Create() + require.NoError(t, err) + err = db.Migrate() + require.NoError(t, err) + + // verify migration + sqlDB, err := drv.Open() + require.NoError(t, err) + defer dbutil.MustClose(sqlDB) + + count := 0 + err = sqlDB.QueryRow(`select count(*) from schema_migrations + where version = '20151129054053'`).Scan(&count) + require.NoError(t, err) + require.Equal(t, 1, count) + + err = sqlDB.QueryRow("select count(*) from posts").Scan(&count) + require.Nil(t, err) + + // rollback + err = db.Rollback() + require.NoError(t, err) + + // verify rollback + err = sqlDB.QueryRow("select count(*) from schema_migrations").Scan(&count) + require.NoError(t, err) + require.Equal(t, 1, count) + + err = sqlDB.QueryRow("select count(*) from posts").Scan(&count) + require.NotNil(t, err) + require.Regexp(t, "(does not exist|doesn't exist|no such table)", err.Error()) + }) } } -func testStatusURL(t *testing.T, u *url.URL) { - db := newTestDB(t, u) - - // drop, recreate, and migrate database - err := db.Drop() - require.NoError(t, err) - err = db.Create() - require.NoError(t, err) - - // verify migration - sqlDB, err := getDriverOpen(u) - require.NoError(t, err) - defer mustClose(sqlDB) - - // two pending - results, err := checkMigrationsStatus(db) - require.NoError(t, err) - require.Len(t, results, 2) - require.False(t, results[0].applied) - require.False(t, results[1].applied) - - // run migrations - err = db.Migrate() - require.NoError(t, err) - - // two applied - results, err = checkMigrationsStatus(db) - require.NoError(t, err) - require.Len(t, results, 2) - require.True(t, results[0].applied) - require.True(t, results[1].applied) - - // rollback last migration - err = db.Rollback() - require.NoError(t, err) - - // one applied, one pending - results, err = checkMigrationsStatus(db) - require.NoError(t, err) - require.Len(t, results, 2) - require.True(t, results[0].applied) - require.False(t, results[1].applied) -} - func TestStatus(t *testing.T) { - for _, u := range testURLs(t) { - testStatusURL(t, u) + for _, u := range testURLs() { + t.Run(u.Scheme, func(t *testing.T) { + db := newTestDB(t, u) + drv, err := db.GetDriver() + require.NoError(t, err) + + // drop, recreate, and migrate database + err = db.Drop() + require.NoError(t, err) + err = db.Create() + require.NoError(t, err) + + // verify migration + sqlDB, err := drv.Open() + require.NoError(t, err) + defer dbutil.MustClose(sqlDB) + + // two pending + results, err := db.CheckMigrationsStatus(drv) + require.NoError(t, err) + require.Len(t, results, 2) + require.False(t, results[0].Applied) + require.False(t, results[1].Applied) + + // run migrations + err = db.Migrate() + require.NoError(t, err) + + // two applied + results, err = db.CheckMigrationsStatus(drv) + require.NoError(t, err) + require.Len(t, results, 2) + require.True(t, results[0].Applied) + require.True(t, results[1].Applied) + + // rollback last migration + err = db.Rollback() + require.NoError(t, err) + + // one applied, one pending + results, err = db.CheckMigrationsStatus(drv) + require.NoError(t, err) + require.Len(t, results, 2) + require.True(t, results[0].Applied) + require.False(t, results[1].Applied) + }) } } diff --git a/pkg/dbmate/driver.go b/pkg/dbmate/driver.go index f5674ad..09d2503 100644 --- a/pkg/dbmate/driver.go +++ b/pkg/dbmate/driver.go @@ -2,56 +2,37 @@ package dbmate import ( "database/sql" - "fmt" "net/url" + + "github.com/amacneil/dbmate/pkg/dbutil" ) // Driver provides top level database functions type Driver interface { - Open(*url.URL) (*sql.DB, error) - DatabaseExists(*url.URL) (bool, error) - CreateDatabase(*url.URL) error - DropDatabase(*url.URL) error - DumpSchema(*url.URL, *sql.DB) ([]byte, error) - SetMigrationsTableName(string) - CreateMigrationsTable(*url.URL, *sql.DB) error + Open() (*sql.DB, error) + DatabaseExists() (bool, error) + CreateDatabase() error + DropDatabase() error + DumpSchema(*sql.DB) ([]byte, error) + CreateMigrationsTable(*sql.DB) error SelectMigrations(*sql.DB, int) (map[string]bool, error) - InsertMigration(Transaction, string) error - DeleteMigration(Transaction, string) error - Ping(*url.URL) error + InsertMigration(dbutil.Transaction, string) error + DeleteMigration(dbutil.Transaction, string) error + Ping() error } -var drivers = map[string]Driver{} - -// RegisterDriver registers a driver for a URL scheme -func RegisterDriver(drv Driver, scheme string) { - drivers[scheme] = drv +// DriverConfig holds configuration passed to driver constructors +type DriverConfig struct { + DatabaseURL *url.URL + MigrationsTableName string } -// Transaction can represent a database or open transaction -type Transaction interface { - Exec(query string, args ...interface{}) (sql.Result, error) - Query(query string, args ...interface{}) (*sql.Rows, error) - QueryRow(query string, args ...interface{}) *sql.Row -} - -// getDriver loads a database driver by name -func getDriver(name string) (Driver, error) { - if drv, ok := drivers[name]; ok { - drv.SetMigrationsTableName(DefaultMigrationsTableName) - - return drv, nil - } - - return nil, fmt.Errorf("unsupported driver: %s", name) -} - -// getDriverOpen is a shortcut for GetDriver(u.Scheme).Open(u) -func getDriverOpen(u *url.URL) (*sql.DB, error) { - drv, err := getDriver(u.Scheme) - if err != nil { - return nil, err - } - - return drv.Open(u) +// DriverFunc represents a driver constructor +type DriverFunc func(DriverConfig) Driver + +var drivers = map[string]DriverFunc{} + +// RegisterDriver registers a driver constructor for a given URL scheme +func RegisterDriver(f DriverFunc, scheme string) { + drivers[scheme] = f } diff --git a/pkg/dbmate/driver_test.go b/pkg/dbmate/driver_test.go deleted file mode 100644 index 0a23eee..0000000 --- a/pkg/dbmate/driver_test.go +++ /dev/null @@ -1,27 +0,0 @@ -package dbmate - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -func TestGetDriver_Postgres(t *testing.T) { - drv, err := getDriver("postgres") - require.NoError(t, err) - _, ok := drv.(*PostgresDriver) - require.Equal(t, true, ok) -} - -func TestGetDriver_MySQL(t *testing.T) { - drv, err := getDriver("mysql") - require.NoError(t, err) - _, ok := drv.(*MySQLDriver) - require.Equal(t, true, ok) -} - -func TestGetDriver_Error(t *testing.T) { - drv, err := getDriver("foo") - require.EqualError(t, err, "unsupported driver: foo") - require.Nil(t, drv) -} diff --git a/pkg/dbmate/migrations.go b/pkg/dbmate/migration.go similarity index 100% rename from pkg/dbmate/migrations.go rename to pkg/dbmate/migration.go diff --git a/pkg/dbmate/migrations_test.go b/pkg/dbmate/migration_test.go similarity index 100% rename from pkg/dbmate/migrations_test.go rename to pkg/dbmate/migration_test.go diff --git a/pkg/dbmate/utils_test.go b/pkg/dbmate/utils_test.go deleted file mode 100644 index 63db33c..0000000 --- a/pkg/dbmate/utils_test.go +++ /dev/null @@ -1,58 +0,0 @@ -package dbmate - -import ( - "database/sql" - "net/url" - "testing" - - "github.com/lib/pq" - "github.com/stretchr/testify/require" -) - -func TestDatabaseName(t *testing.T) { - u, err := url.Parse("ignore://localhost/foo?query") - require.NoError(t, err) - - name := databaseName(u) - require.Equal(t, "foo", name) -} - -func TestDatabaseName_Empty(t *testing.T) { - u, err := url.Parse("ignore://localhost") - require.NoError(t, err) - - name := databaseName(u) - require.Equal(t, "", name) -} - -func TestTrimLeadingSQLComments(t *testing.T) { - in := "--\n" + - "-- foo\n\n" + - "-- bar\n\n" + - "real stuff\n" + - "-- end\n" - out, err := trimLeadingSQLComments([]byte(in)) - require.NoError(t, err) - require.Equal(t, "real stuff\n-- end\n", string(out)) -} - -func TestQueryColumn(t *testing.T) { - u := postgresTestURL(t) - db, err := sql.Open("postgres", u.String()) - require.NoError(t, err) - - val, err := queryColumn(db, "select concat('foo_', unnest($1::text[]))", - pq.Array([]string{"hi", "there"})) - require.NoError(t, err) - require.Equal(t, []string{"foo_hi", "foo_there"}, val) -} - -func TestQueryValue(t *testing.T) { - u := postgresTestURL(t) - db, err := sql.Open("postgres", u.String()) - require.NoError(t, err) - - val, err := queryValue(db, "select $1::int + $2::int", "5", 2) - require.NoError(t, err) - require.Equal(t, "7", val) -} diff --git a/pkg/dbmate/utils.go b/pkg/dbutil/dbutil.go similarity index 65% rename from pkg/dbmate/utils.go rename to pkg/dbutil/dbutil.go index 41913fc..e0bb13a 100644 --- a/pkg/dbmate/utils.go +++ b/pkg/dbutil/dbutil.go @@ -1,21 +1,26 @@ -package dbmate +package dbutil import ( "bufio" "bytes" "database/sql" "errors" - "fmt" "io" "net/url" - "os" "os/exec" "strings" "unicode" ) -// databaseName returns the database name from a URL -func databaseName(u *url.URL) string { +// Transaction can represent a database or open transaction +type Transaction interface { + Exec(query string, args ...interface{}) (sql.Result, error) + Query(query string, args ...interface{}) (*sql.Rows, error) + QueryRow(query string, args ...interface{}) *sql.Row +} + +// DatabaseName returns the database name from a URL +func DatabaseName(u *url.URL) string { name := u.Path if len(name) > 0 && name[:1] == "/" { name = name[1:] @@ -24,24 +29,15 @@ func databaseName(u *url.URL) string { return name } -// mustClose ensures a stream is closed -func mustClose(c io.Closer) { +// MustClose ensures a stream is closed +func MustClose(c io.Closer) { if err := c.Close(); err != nil { panic(err) } } -// ensureDir creates a directory if it does not already exist -func ensureDir(dir string) error { - if err := os.MkdirAll(dir, 0755); err != nil { - return fmt.Errorf("unable to create directory `%s`", dir) - } - - return nil -} - -// runCommand runs a command and returns the stdout if successful -func runCommand(name string, args ...string) ([]byte, error) { +// RunCommand runs a command and returns the stdout if successful +func RunCommand(name string, args ...string) ([]byte, error) { var stdout, stderr bytes.Buffer cmd := exec.Command(name, args...) cmd.Stdout = &stdout @@ -61,10 +57,10 @@ func runCommand(name string, args ...string) ([]byte, error) { return stdout.Bytes(), nil } -// trimLeadingSQLComments removes sql comments and blank lines from the beginning of text +// TrimLeadingSQLComments removes sql comments and blank lines from the beginning of text // generally when performing sql dumps these contain host-specific information such as // client/server version numbers -func trimLeadingSQLComments(data []byte) ([]byte, error) { +func TrimLeadingSQLComments(data []byte) ([]byte, error) { // create decent size buffer out := bytes.NewBuffer(make([]byte, 0, len(data))) @@ -101,15 +97,15 @@ func trimLeadingSQLComments(data []byte) ([]byte, error) { return out.Bytes(), nil } -// queryColumn runs a SQL statement and returns a slice of strings +// QueryColumn runs a SQL statement and returns a slice of strings // it is assumed that the statement returns only one column // e.g. schema_migrations table -func queryColumn(db Transaction, query string, args ...interface{}) ([]string, error) { +func QueryColumn(db Transaction, query string, args ...interface{}) ([]string, error) { rows, err := db.Query(query, args...) if err != nil { return nil, err } - defer mustClose(rows) + defer MustClose(rows) // read into slice var result []string @@ -128,10 +124,10 @@ func queryColumn(db Transaction, query string, args ...interface{}) ([]string, e return result, nil } -// queryValue runs a SQL statement and returns a single string +// QueryValue runs a SQL statement and returns a single string // it is assumed that the statement returns only one row and one column // sql NULL is returned as empty string -func queryValue(db Transaction, query string, args ...interface{}) (string, error) { +func QueryValue(db Transaction, query string, args ...interface{}) (string, error) { var result sql.NullString err := db.QueryRow(query, args...).Scan(&result) if err != nil || !result.Valid { @@ -141,13 +137,17 @@ func queryValue(db Transaction, query string, args ...interface{}) (string, erro return result.String, nil } -func printVerbose(result sql.Result) { - lastInsertID, err := result.LastInsertId() - if err == nil { - fmt.Printf("Last insert ID: %d\n", lastInsertID) +// MustParseURL parses a URL from string, and panics if it fails. +// It is used during testing and in cases where we are parsing a generated URL. +func MustParseURL(s string) *url.URL { + if s == "" { + panic("missing url") } - rowsAffected, err := result.RowsAffected() - if err == nil { - fmt.Printf("Rows affected: %d\n", rowsAffected) + + u, err := url.Parse(s) + if err != nil { + panic(err) } + + return u } diff --git a/pkg/dbutil/dbutil_test.go b/pkg/dbutil/dbutil_test.go new file mode 100644 index 0000000..113b933 --- /dev/null +++ b/pkg/dbutil/dbutil_test.go @@ -0,0 +1,58 @@ +package dbutil_test + +import ( + "database/sql" + "testing" + + "github.com/amacneil/dbmate/pkg/dbutil" + + _ "github.com/mattn/go-sqlite3" // database/sql driver + "github.com/stretchr/testify/require" +) + +func TestDatabaseName(t *testing.T) { + t.Run("valid", func(t *testing.T) { + u := dbutil.MustParseURL("foo://host/dbname?query") + name := dbutil.DatabaseName(u) + require.Equal(t, "dbname", name) + }) + + t.Run("empty", func(t *testing.T) { + u := dbutil.MustParseURL("foo://host") + name := dbutil.DatabaseName(u) + require.Equal(t, "", name) + }) +} + +func TestTrimLeadingSQLComments(t *testing.T) { + in := "--\n" + + "-- foo\n\n" + + "-- bar\n\n" + + "real stuff\n" + + "-- end\n" + out, err := dbutil.TrimLeadingSQLComments([]byte(in)) + require.NoError(t, err) + require.Equal(t, "real stuff\n-- end\n", string(out)) +} + +// connect to in-memory sqlite database for testing +const sqliteMemoryDB = "file:dbutil.sqlite3?mode=memory&cache=shared" + +func TestQueryColumn(t *testing.T) { + db, err := sql.Open("sqlite3", sqliteMemoryDB) + require.NoError(t, err) + + val, err := dbutil.QueryColumn(db, "select 'foo_' || val from (select ? as val union select ?)", + "hi", "there") + require.NoError(t, err) + require.Equal(t, []string{"foo_hi", "foo_there"}, val) +} + +func TestQueryValue(t *testing.T) { + db, err := sql.Open("sqlite3", sqliteMemoryDB) + require.NoError(t, err) + + val, err := dbutil.QueryValue(db, "select $1 + $2", "5", 2) + require.NoError(t, err) + require.Equal(t, "7", val) +} diff --git a/pkg/dbmate/clickhouse.go b/pkg/driver/clickhouse/clickhouse.go similarity index 69% rename from pkg/dbmate/clickhouse.go rename to pkg/driver/clickhouse/clickhouse.go index f0acd3b..0204a58 100644 --- a/pkg/dbmate/clickhouse.go +++ b/pkg/driver/clickhouse/clickhouse.go @@ -1,4 +1,4 @@ -package dbmate +package clickhouse import ( "bytes" @@ -9,19 +9,31 @@ import ( "sort" "strings" + "github.com/amacneil/dbmate/pkg/dbmate" + "github.com/amacneil/dbmate/pkg/dbutil" + "github.com/ClickHouse/clickhouse-go" ) func init() { - RegisterDriver(&ClickHouseDriver{}, "clickhouse") + dbmate.RegisterDriver(NewDriver, "clickhouse") } -// ClickHouseDriver provides top level database functions -type ClickHouseDriver struct { +// Driver provides top level database functions +type Driver struct { migrationsTableName string + databaseURL *url.URL } -func normalizeClickHouseURL(initialURL *url.URL) *url.URL { +// NewDriver initializes the driver +func NewDriver(config dbmate.DriverConfig) dbmate.Driver { + return &Driver{ + migrationsTableName: config.MigrationsTableName, + databaseURL: config.DatabaseURL, + } +} + +func connectionString(initialURL *url.URL) string { u := *initialURL u.Scheme = "tcp" @@ -50,31 +62,31 @@ func normalizeClickHouseURL(initialURL *url.URL) *url.URL { } u.RawQuery = query.Encode() - return &u -} - -// SetMigrationsTableName sets the schema migrations table name -func (drv *ClickHouseDriver) SetMigrationsTableName(name string) { - drv.migrationsTableName = name + return u.String() } // Open creates a new database connection -func (drv *ClickHouseDriver) Open(u *url.URL) (*sql.DB, error) { - return sql.Open("clickhouse", normalizeClickHouseURL(u).String()) +func (drv *Driver) Open() (*sql.DB, error) { + return sql.Open("clickhouse", connectionString(drv.databaseURL)) } -func (drv *ClickHouseDriver) openClickHouseDB(u *url.URL) (*sql.DB, error) { +func (drv *Driver) openClickHouseDB() (*sql.DB, error) { + // clone databaseURL + clickhouseURL, err := url.Parse(connectionString(drv.databaseURL)) + if err != nil { + return nil, err + } + // connect to clickhouse database - clickhouseURL := normalizeClickHouseURL(u) values := clickhouseURL.Query() values.Set("database", "default") clickhouseURL.RawQuery = values.Encode() - return drv.Open(clickhouseURL) + return sql.Open("clickhouse", clickhouseURL.String()) } -func (drv *ClickHouseDriver) databaseName(u *url.URL) string { - name := normalizeClickHouseURL(u).Query().Get("database") +func (drv *Driver) databaseName() string { + name := dbutil.MustParseURL(connectionString(drv.databaseURL)).Query().Get("database") if name == "" { name = "default" } @@ -83,7 +95,7 @@ func (drv *ClickHouseDriver) databaseName(u *url.URL) string { var clickhouseValidIdentifier = regexp.MustCompile(`^[a-zA-Z_][0-9a-zA-Z_]*$`) -func (drv *ClickHouseDriver) quoteIdentifier(str string) string { +func (drv *Driver) quoteIdentifier(str string) string { if clickhouseValidIdentifier.MatchString(str) { return str } @@ -94,15 +106,15 @@ func (drv *ClickHouseDriver) quoteIdentifier(str string) string { } // CreateDatabase creates the specified database -func (drv *ClickHouseDriver) CreateDatabase(u *url.URL) error { - name := drv.databaseName(u) +func (drv *Driver) CreateDatabase() error { + name := drv.databaseName() fmt.Printf("Creating: %s\n", name) - db, err := drv.openClickHouseDB(u) + db, err := drv.openClickHouseDB() if err != nil { return err } - defer mustClose(db) + defer dbutil.MustClose(db) _, err = db.Exec("create database " + drv.quoteIdentifier(name)) @@ -110,27 +122,27 @@ func (drv *ClickHouseDriver) CreateDatabase(u *url.URL) error { } // DropDatabase drops the specified database (if it exists) -func (drv *ClickHouseDriver) DropDatabase(u *url.URL) error { - name := drv.databaseName(u) +func (drv *Driver) DropDatabase() error { + name := drv.databaseName() fmt.Printf("Dropping: %s\n", name) - db, err := drv.openClickHouseDB(u) + db, err := drv.openClickHouseDB() if err != nil { return err } - defer mustClose(db) + defer dbutil.MustClose(db) _, err = db.Exec("drop database if exists " + drv.quoteIdentifier(name)) return err } -func (drv *ClickHouseDriver) schemaDump(db *sql.DB, buf *bytes.Buffer, databaseName string) error { +func (drv *Driver) schemaDump(db *sql.DB, buf *bytes.Buffer, databaseName string) error { buf.WriteString("\n--\n-- Database schema\n--\n\n") buf.WriteString("CREATE DATABASE " + drv.quoteIdentifier(databaseName) + " IF NOT EXISTS;\n\n") - tables, err := queryColumn(db, "show tables") + tables, err := dbutil.QueryColumn(db, "show tables") if err != nil { return err } @@ -147,11 +159,11 @@ func (drv *ClickHouseDriver) schemaDump(db *sql.DB, buf *bytes.Buffer, databaseN return nil } -func (drv *ClickHouseDriver) schemaMigrationsDump(db *sql.DB, buf *bytes.Buffer) error { +func (drv *Driver) schemaMigrationsDump(db *sql.DB, buf *bytes.Buffer) error { migrationsTable := drv.quotedMigrationsTableName() // load applied migrations - migrations, err := queryColumn(db, + migrations, err := dbutil.QueryColumn(db, fmt.Sprintf("select version from %s final ", migrationsTable)+ "where applied order by version asc", ) @@ -178,11 +190,11 @@ func (drv *ClickHouseDriver) schemaMigrationsDump(db *sql.DB, buf *bytes.Buffer) } // DumpSchema returns the current database schema -func (drv *ClickHouseDriver) DumpSchema(u *url.URL, db *sql.DB) ([]byte, error) { +func (drv *Driver) DumpSchema(db *sql.DB) ([]byte, error) { var buf bytes.Buffer var err error - err = drv.schemaDump(db, &buf, drv.databaseName(u)) + err = drv.schemaDump(db, &buf, drv.databaseName()) if err != nil { return nil, err } @@ -196,14 +208,14 @@ func (drv *ClickHouseDriver) DumpSchema(u *url.URL, db *sql.DB) ([]byte, error) } // DatabaseExists determines whether the database exists -func (drv *ClickHouseDriver) DatabaseExists(u *url.URL) (bool, error) { - name := drv.databaseName(u) +func (drv *Driver) DatabaseExists() (bool, error) { + name := drv.databaseName() - db, err := drv.openClickHouseDB(u) + db, err := drv.openClickHouseDB() if err != nil { return false, err } - defer mustClose(db) + defer dbutil.MustClose(db) exists := false err = db.QueryRow("SELECT 1 FROM system.databases where name = ?", name). @@ -216,7 +228,7 @@ func (drv *ClickHouseDriver) DatabaseExists(u *url.URL) (bool, error) { } // CreateMigrationsTable creates the schema migrations table -func (drv *ClickHouseDriver) CreateMigrationsTable(u *url.URL, db *sql.DB) error { +func (drv *Driver) CreateMigrationsTable(db *sql.DB) error { _, err := db.Exec(fmt.Sprintf(` create table if not exists %s ( version String, @@ -232,7 +244,7 @@ func (drv *ClickHouseDriver) CreateMigrationsTable(u *url.URL, db *sql.DB) error // SelectMigrations returns a list of applied migrations // with an optional limit (in descending order) -func (drv *ClickHouseDriver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, error) { +func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, error) { query := fmt.Sprintf("select version from %s final where applied order by version desc", drv.quotedMigrationsTableName()) @@ -244,7 +256,7 @@ func (drv *ClickHouseDriver) SelectMigrations(db *sql.DB, limit int) (map[string return nil, err } - defer mustClose(rows) + defer dbutil.MustClose(rows) migrations := map[string]bool{} for rows.Next() { @@ -264,7 +276,7 @@ func (drv *ClickHouseDriver) SelectMigrations(db *sql.DB, limit int) (map[string } // InsertMigration adds a new migration record -func (drv *ClickHouseDriver) InsertMigration(db Transaction, version string) error { +func (drv *Driver) InsertMigration(db dbutil.Transaction, version string) error { _, err := db.Exec( fmt.Sprintf("insert into %s (version) values (?)", drv.quotedMigrationsTableName()), version) @@ -273,7 +285,7 @@ func (drv *ClickHouseDriver) InsertMigration(db Transaction, version string) err } // DeleteMigration removes a migration record -func (drv *ClickHouseDriver) DeleteMigration(db Transaction, version string) error { +func (drv *Driver) DeleteMigration(db dbutil.Transaction, version string) error { _, err := db.Exec( fmt.Sprintf("insert into %s (version, applied) values (?, ?)", drv.quotedMigrationsTableName()), @@ -285,15 +297,15 @@ func (drv *ClickHouseDriver) DeleteMigration(db Transaction, version string) err // Ping verifies a connection to the database server. It does not verify whether the // specified database exists. -func (drv *ClickHouseDriver) Ping(u *url.URL) error { +func (drv *Driver) Ping() error { // attempt connection to primary database, not "clickhouse" database // to support servers with no "clickhouse" database // (see https://github.com/amacneil/dbmate/issues/78) - db, err := drv.Open(u) + db, err := drv.Open() if err != nil { return err } - defer mustClose(db) + defer dbutil.MustClose(db) err = db.Ping() if err == nil { @@ -309,6 +321,6 @@ func (drv *ClickHouseDriver) Ping(u *url.URL) error { return err } -func (drv *ClickHouseDriver) quotedMigrationsTableName() string { +func (drv *Driver) quotedMigrationsTableName() string { return drv.quoteIdentifier(drv.migrationsTableName) } diff --git a/pkg/dbmate/clickhouse_test.go b/pkg/driver/clickhouse/clickhouse_test.go similarity index 61% rename from pkg/dbmate/clickhouse_test.go rename to pkg/driver/clickhouse/clickhouse_test.go index c1aeefd..6c85c58 100644 --- a/pkg/dbmate/clickhouse_test.go +++ b/pkg/driver/clickhouse/clickhouse_test.go @@ -1,108 +1,117 @@ -package dbmate +package clickhouse import ( "database/sql" "net/url" + "os" "testing" + "github.com/amacneil/dbmate/pkg/dbmate" + "github.com/amacneil/dbmate/pkg/dbutil" + "github.com/stretchr/testify/require" ) -func clickhouseTestURL(t *testing.T) *url.URL { - u, err := url.Parse("clickhouse://clickhouse:9000?database=dbmate") +func testClickHouseDriver(t *testing.T) *Driver { + u := dbutil.MustParseURL(os.Getenv("CLICKHOUSE_TEST_URL")) + drv, err := dbmate.New(u).GetDriver() require.NoError(t, err) - return u + return drv.(*Driver) } -func testClickHouseDriver() *ClickHouseDriver { - drv := &ClickHouseDriver{} - drv.SetMigrationsTableName(DefaultMigrationsTableName) - - return drv -} - -func prepTestClickHouseDB(t *testing.T, u *url.URL) *sql.DB { - drv := testClickHouseDriver() +func prepTestClickHouseDB(t *testing.T) *sql.DB { + drv := testClickHouseDriver(t) // drop any existing database - err := drv.DropDatabase(u) + err := drv.DropDatabase() require.NoError(t, err) // create database - err = drv.CreateDatabase(u) + err = drv.CreateDatabase() require.NoError(t, err) // connect database - db, err := sql.Open("clickhouse", u.String()) + db, err := sql.Open("clickhouse", drv.databaseURL.String()) require.NoError(t, err) return db } -func TestNormalizeClickHouseURLSimplified(t *testing.T) { - u, err := url.Parse("clickhouse://user:pass@host/db") +func TestGetDriver(t *testing.T) { + db := dbmate.New(dbutil.MustParseURL("clickhouse://")) + drvInterface, err := db.GetDriver() require.NoError(t, err) - s := normalizeClickHouseURL(u).String() - require.Equal(t, "tcp://host:9000?database=db&password=pass&username=user", s) + // driver should have URL and default migrations table set + drv, ok := drvInterface.(*Driver) + require.True(t, ok) + require.Equal(t, db.DatabaseURL.String(), drv.databaseURL.String()) + require.Equal(t, "schema_migrations", drv.migrationsTableName) } -func TestNormalizeClickHouseURLCanonical(t *testing.T) { - u, err := url.Parse("clickhouse://host:9000?database=db&password=pass&username=user") - require.NoError(t, err) +func TestConnectionString(t *testing.T) { + t.Run("simple", func(t *testing.T) { + u, err := url.Parse("clickhouse://user:pass@host/db") + require.NoError(t, err) - s := normalizeClickHouseURL(u).String() - require.Equal(t, "tcp://host:9000?database=db&password=pass&username=user", s) + s := connectionString(u) + require.Equal(t, "tcp://host:9000?database=db&password=pass&username=user", s) + }) + + t.Run("canonical", func(t *testing.T) { + u, err := url.Parse("clickhouse://host:9000?database=db&password=pass&username=user") + require.NoError(t, err) + + s := connectionString(u) + require.Equal(t, "tcp://host:9000?database=db&password=pass&username=user", s) + }) } func TestClickHouseCreateDropDatabase(t *testing.T) { - drv := testClickHouseDriver() - u := clickhouseTestURL(t) + drv := testClickHouseDriver(t) // drop any existing database - err := drv.DropDatabase(u) + err := drv.DropDatabase() require.NoError(t, err) // create database - err = drv.CreateDatabase(u) + err = drv.CreateDatabase() require.NoError(t, err) // check that database exists and we can connect to it func() { - db, err := sql.Open("clickhouse", u.String()) + db, err := sql.Open("clickhouse", drv.databaseURL.String()) require.NoError(t, err) - defer mustClose(db) + defer dbutil.MustClose(db) err = db.Ping() require.NoError(t, err) }() // drop the database - err = drv.DropDatabase(u) + err = drv.DropDatabase() require.NoError(t, err) // check that database no longer exists func() { - db, err := sql.Open("clickhouse", u.String()) + db, err := sql.Open("clickhouse", drv.databaseURL.String()) require.NoError(t, err) - defer mustClose(db) + defer dbutil.MustClose(db) err = db.Ping() - require.EqualError(t, err, "code: 81, message: Database dbmate doesn't exist") + require.EqualError(t, err, "code: 81, message: Database dbmate_test doesn't exist") }() } func TestClickHouseDumpSchema(t *testing.T) { - drv := testClickHouseDriver() - drv.SetMigrationsTableName("test_migrations") - - u := clickhouseTestURL(t) + drv := testClickHouseDriver(t) + drv.migrationsTableName = "test_migrations" // prepare database - db := prepTestClickHouseDB(t, u) - defer mustClose(db) - err := drv.CreateMigrationsTable(u, db) + db := prepTestClickHouseDB(t) + defer dbutil.MustClose(db) + err := drv.CreateMigrationsTable(db) require.NoError(t, err) // insert migration @@ -120,9 +129,9 @@ func TestClickHouseDumpSchema(t *testing.T) { require.NoError(t, err) // DumpSchema should return schema - schema, err := drv.DumpSchema(u, db) + schema, err := drv.DumpSchema(db) require.NoError(t, err) - require.Contains(t, string(schema), "CREATE TABLE "+drv.databaseName(u)+".test_migrations") + require.Contains(t, string(schema), "CREATE TABLE "+drv.databaseName()+".test_migrations") require.Contains(t, string(schema), "--\n"+ "-- Dbmate schema migrations\n"+ "--\n\n"+ @@ -131,66 +140,63 @@ func TestClickHouseDumpSchema(t *testing.T) { " ('abc2');\n") // DumpSchema should return error if command fails - values := u.Query() + values := drv.databaseURL.Query() values.Set("database", "fakedb") - u.RawQuery = values.Encode() - db, err = sql.Open("clickhouse", u.String()) + drv.databaseURL.RawQuery = values.Encode() + db, err = sql.Open("clickhouse", drv.databaseURL.String()) require.NoError(t, err) - schema, err = drv.DumpSchema(u, db) + schema, err = drv.DumpSchema(db) require.Nil(t, schema) require.EqualError(t, err, "code: 81, message: Database fakedb doesn't exist") } func TestClickHouseDatabaseExists(t *testing.T) { - drv := testClickHouseDriver() - u := clickhouseTestURL(t) + drv := testClickHouseDriver(t) // drop any existing database - err := drv.DropDatabase(u) + err := drv.DropDatabase() require.NoError(t, err) // DatabaseExists should return false - exists, err := drv.DatabaseExists(u) + exists, err := drv.DatabaseExists() require.NoError(t, err) require.Equal(t, false, exists) // create database - err = drv.CreateDatabase(u) + err = drv.CreateDatabase() require.NoError(t, err) // DatabaseExists should return true - exists, err = drv.DatabaseExists(u) + exists, err = drv.DatabaseExists() require.NoError(t, err) require.Equal(t, true, exists) } func TestClickHouseDatabaseExists_Error(t *testing.T) { - drv := testClickHouseDriver() - u := clickhouseTestURL(t) - values := u.Query() + drv := testClickHouseDriver(t) + values := drv.databaseURL.Query() values.Set("username", "invalid") - u.RawQuery = values.Encode() + drv.databaseURL.RawQuery = values.Encode() - exists, err := drv.DatabaseExists(u) + exists, err := drv.DatabaseExists() require.EqualError(t, err, "code: 192, message: Unknown user invalid") require.Equal(t, false, exists) } func TestClickHouseCreateMigrationsTable(t *testing.T) { t.Run("default table", func(t *testing.T) { - drv := testClickHouseDriver() - u := clickhouseTestURL(t) - db := prepTestClickHouseDB(t, u) - defer mustClose(db) + drv := testClickHouseDriver(t) + db := prepTestClickHouseDB(t) + defer dbutil.MustClose(db) // migrations table should not exist count := 0 err := db.QueryRow("select count(*) from schema_migrations").Scan(&count) - require.EqualError(t, err, "code: 60, message: Table dbmate.schema_migrations doesn't exist.") + require.EqualError(t, err, "code: 60, message: Table dbmate_test.schema_migrations doesn't exist.") // create table - err = drv.CreateMigrationsTable(u, db) + err = drv.CreateMigrationsTable(db) require.NoError(t, err) // migrations table should exist @@ -198,25 +204,24 @@ func TestClickHouseCreateMigrationsTable(t *testing.T) { require.NoError(t, err) // create table should be idempotent - err = drv.CreateMigrationsTable(u, db) + err = drv.CreateMigrationsTable(db) require.NoError(t, err) }) t.Run("custom table", func(t *testing.T) { - drv := testClickHouseDriver() - drv.SetMigrationsTableName("testMigrations") + drv := testClickHouseDriver(t) + drv.migrationsTableName = "testMigrations" - u := clickhouseTestURL(t) - db := prepTestClickHouseDB(t, u) - defer mustClose(db) + db := prepTestClickHouseDB(t) + defer dbutil.MustClose(db) // migrations table should not exist count := 0 err := db.QueryRow("select count(*) from \"testMigrations\"").Scan(&count) - require.EqualError(t, err, "code: 60, message: Table dbmate.testMigrations doesn't exist.") + require.EqualError(t, err, "code: 60, message: Table dbmate_test.testMigrations doesn't exist.") // create table - err = drv.CreateMigrationsTable(u, db) + err = drv.CreateMigrationsTable(db) require.NoError(t, err) // migrations table should exist @@ -224,20 +229,19 @@ func TestClickHouseCreateMigrationsTable(t *testing.T) { require.NoError(t, err) // create table should be idempotent - err = drv.CreateMigrationsTable(u, db) + err = drv.CreateMigrationsTable(db) require.NoError(t, err) }) } func TestClickHouseSelectMigrations(t *testing.T) { - drv := testClickHouseDriver() - drv.SetMigrationsTableName("test_migrations") + drv := testClickHouseDriver(t) + drv.migrationsTableName = "test_migrations" - u := clickhouseTestURL(t) - db := prepTestClickHouseDB(t, u) - defer mustClose(db) + db := prepTestClickHouseDB(t) + defer dbutil.MustClose(db) - err := drv.CreateMigrationsTable(u, db) + err := drv.CreateMigrationsTable(db) require.NoError(t, err) tx, err := db.Begin() @@ -268,14 +272,13 @@ func TestClickHouseSelectMigrations(t *testing.T) { } func TestClickHouseInsertMigration(t *testing.T) { - drv := testClickHouseDriver() - drv.SetMigrationsTableName("test_migrations") + drv := testClickHouseDriver(t) + drv.migrationsTableName = "test_migrations" - u := clickhouseTestURL(t) - db := prepTestClickHouseDB(t, u) - defer mustClose(db) + db := prepTestClickHouseDB(t) + defer dbutil.MustClose(db) - err := drv.CreateMigrationsTable(u, db) + err := drv.CreateMigrationsTable(db) require.NoError(t, err) count := 0 @@ -297,14 +300,13 @@ func TestClickHouseInsertMigration(t *testing.T) { } func TestClickHouseDeleteMigration(t *testing.T) { - drv := testClickHouseDriver() - drv.SetMigrationsTableName("test_migrations") + drv := testClickHouseDriver(t) + drv.migrationsTableName = "test_migrations" - u := clickhouseTestURL(t) - db := prepTestClickHouseDB(t, u) - defer mustClose(db) + db := prepTestClickHouseDB(t) + defer dbutil.MustClose(db) - err := drv.CreateMigrationsTable(u, db) + err := drv.CreateMigrationsTable(db) require.NoError(t, err) tx, err := db.Begin() @@ -332,42 +334,41 @@ func TestClickHouseDeleteMigration(t *testing.T) { } func TestClickHousePing(t *testing.T) { - drv := testClickHouseDriver() - u := clickhouseTestURL(t) + drv := testClickHouseDriver(t) // drop any existing database - err := drv.DropDatabase(u) + err := drv.DropDatabase() require.NoError(t, err) // ping database - err = drv.Ping(u) + err = drv.Ping() require.NoError(t, err) // ping invalid host should return error - u.Host = "clickhouse:404" - err = drv.Ping(u) + drv.databaseURL.Host = "clickhouse:404" + err = drv.Ping() require.Error(t, err) require.Contains(t, err.Error(), "connect: connection refused") } func TestClickHouseQuotedMigrationsTableName(t *testing.T) { t.Run("default name", func(t *testing.T) { - drv := testClickHouseDriver() + drv := testClickHouseDriver(t) name := drv.quotedMigrationsTableName() require.Equal(t, "schema_migrations", name) }) t.Run("custom name", func(t *testing.T) { - drv := testClickHouseDriver() - drv.SetMigrationsTableName("fooMigrations") + drv := testClickHouseDriver(t) + drv.migrationsTableName = "fooMigrations" name := drv.quotedMigrationsTableName() require.Equal(t, "fooMigrations", name) }) t.Run("quoted name", func(t *testing.T) { - drv := testClickHouseDriver() - drv.SetMigrationsTableName("bizarre\"$name") + drv := testClickHouseDriver(t) + drv.migrationsTableName = "bizarre\"$name" name := drv.quotedMigrationsTableName() require.Equal(t, `"bizarre""$name"`, name) diff --git a/pkg/dbmate/mysql.go b/pkg/driver/mysql/mysql.go similarity index 64% rename from pkg/dbmate/mysql.go rename to pkg/driver/mysql/mysql.go index 6443351..a19b74b 100644 --- a/pkg/dbmate/mysql.go +++ b/pkg/driver/mysql/mysql.go @@ -1,4 +1,4 @@ -package dbmate +package mysql import ( "bytes" @@ -7,19 +7,31 @@ import ( "net/url" "strings" - _ "github.com/go-sql-driver/mysql" // mysql driver for database/sql + "github.com/amacneil/dbmate/pkg/dbmate" + "github.com/amacneil/dbmate/pkg/dbutil" + + _ "github.com/go-sql-driver/mysql" // database/sql driver ) func init() { - RegisterDriver(&MySQLDriver{}, "mysql") + dbmate.RegisterDriver(NewDriver, "mysql") } -// MySQLDriver provides top level database functions -type MySQLDriver struct { +// Driver provides top level database functions +type Driver struct { migrationsTableName string + databaseURL *url.URL } -func normalizeMySQLURL(u *url.URL) string { +// NewDriver initializes the driver +func NewDriver(config dbmate.DriverConfig) dbmate.Driver { + return &Driver{ + migrationsTableName: config.MigrationsTableName, + databaseURL: config.DatabaseURL, + } +} + +func connectionString(u *url.URL) string { query := u.Query() query.Set("multiStatements", "true") @@ -53,40 +65,40 @@ func normalizeMySQLURL(u *url.URL) string { return normalizedString } -// SetMigrationsTableName sets the schema migrations table name -func (drv *MySQLDriver) SetMigrationsTableName(name string) { - drv.migrationsTableName = name -} - // Open creates a new database connection -func (drv *MySQLDriver) Open(u *url.URL) (*sql.DB, error) { - return sql.Open("mysql", normalizeMySQLURL(u)) +func (drv *Driver) Open() (*sql.DB, error) { + return sql.Open("mysql", connectionString(drv.databaseURL)) } -func (drv *MySQLDriver) openRootDB(u *url.URL) (*sql.DB, error) { +func (drv *Driver) openRootDB() (*sql.DB, error) { + // clone databaseURL + rootURL, err := url.Parse(drv.databaseURL.String()) + if err != nil { + return nil, err + } + // connect to no particular database - rootURL := *u rootURL.Path = "/" - return drv.Open(&rootURL) + return sql.Open("mysql", connectionString(rootURL)) } -func (drv *MySQLDriver) quoteIdentifier(str string) string { +func (drv *Driver) quoteIdentifier(str string) string { str = strings.Replace(str, "`", "\\`", -1) return fmt.Sprintf("`%s`", str) } // CreateDatabase creates the specified database -func (drv *MySQLDriver) CreateDatabase(u *url.URL) error { - name := databaseName(u) +func (drv *Driver) CreateDatabase() error { + name := dbutil.DatabaseName(drv.databaseURL) fmt.Printf("Creating: %s\n", name) - db, err := drv.openRootDB(u) + db, err := drv.openRootDB() if err != nil { return err } - defer mustClose(db) + defer dbutil.MustClose(db) _, err = db.Exec(fmt.Sprintf("create database %s", drv.quoteIdentifier(name))) @@ -95,15 +107,15 @@ func (drv *MySQLDriver) CreateDatabase(u *url.URL) error { } // DropDatabase drops the specified database (if it exists) -func (drv *MySQLDriver) DropDatabase(u *url.URL) error { - name := databaseName(u) +func (drv *Driver) DropDatabase() error { + name := dbutil.DatabaseName(drv.databaseURL) fmt.Printf("Dropping: %s\n", name) - db, err := drv.openRootDB(u) + db, err := drv.openRootDB() if err != nil { return err } - defer mustClose(db) + defer dbutil.MustClose(db) _, err = db.Exec(fmt.Sprintf("drop database if exists %s", drv.quoteIdentifier(name))) @@ -111,37 +123,37 @@ func (drv *MySQLDriver) DropDatabase(u *url.URL) error { return err } -func (drv *MySQLDriver) mysqldumpArgs(u *url.URL) []string { +func (drv *Driver) mysqldumpArgs() []string { // generate CLI arguments args := []string{"--opt", "--routines", "--no-data", "--skip-dump-date", "--skip-add-drop-table"} - if hostname := u.Hostname(); hostname != "" { + if hostname := drv.databaseURL.Hostname(); hostname != "" { args = append(args, "--host="+hostname) } - if port := u.Port(); port != "" { + if port := drv.databaseURL.Port(); port != "" { args = append(args, "--port="+port) } - if username := u.User.Username(); username != "" { + if username := drv.databaseURL.User.Username(); username != "" { args = append(args, "--user="+username) } // mysql recommends against using environment variables to supply password // https://dev.mysql.com/doc/refman/5.7/en/password-security-user.html - if password, set := u.User.Password(); set { + if password, set := drv.databaseURL.User.Password(); set { args = append(args, "--password="+password) } // add database name - args = append(args, strings.TrimLeft(u.Path, "/")) + args = append(args, dbutil.DatabaseName(drv.databaseURL)) return args } -func (drv *MySQLDriver) schemaMigrationsDump(db *sql.DB) ([]byte, error) { +func (drv *Driver) schemaMigrationsDump(db *sql.DB) ([]byte, error) { migrationsTable := drv.quotedMigrationsTableName() // load applied migrations - migrations, err := queryColumn(db, + migrations, err := dbutil.QueryColumn(db, fmt.Sprintf("select quote(version) from %s order by version asc", migrationsTable)) if err != nil { return nil, err @@ -165,8 +177,8 @@ func (drv *MySQLDriver) schemaMigrationsDump(db *sql.DB) ([]byte, error) { } // DumpSchema returns the current database schema -func (drv *MySQLDriver) DumpSchema(u *url.URL, db *sql.DB) ([]byte, error) { - schema, err := runCommand("mysqldump", drv.mysqldumpArgs(u)...) +func (drv *Driver) DumpSchema(db *sql.DB) ([]byte, error) { + schema, err := dbutil.RunCommand("mysqldump", drv.mysqldumpArgs()...) if err != nil { return nil, err } @@ -177,18 +189,18 @@ func (drv *MySQLDriver) DumpSchema(u *url.URL, db *sql.DB) ([]byte, error) { } schema = append(schema, migrations...) - return trimLeadingSQLComments(schema) + return dbutil.TrimLeadingSQLComments(schema) } // DatabaseExists determines whether the database exists -func (drv *MySQLDriver) DatabaseExists(u *url.URL) (bool, error) { - name := databaseName(u) +func (drv *Driver) DatabaseExists() (bool, error) { + name := dbutil.DatabaseName(drv.databaseURL) - db, err := drv.openRootDB(u) + db, err := drv.openRootDB() if err != nil { return false, err } - defer mustClose(db) + defer dbutil.MustClose(db) exists := false err = db.QueryRow("select true from information_schema.schemata "+ @@ -201,7 +213,7 @@ func (drv *MySQLDriver) DatabaseExists(u *url.URL) (bool, error) { } // CreateMigrationsTable creates the schema_migrations table -func (drv *MySQLDriver) CreateMigrationsTable(u *url.URL, db *sql.DB) error { +func (drv *Driver) CreateMigrationsTable(db *sql.DB) error { _, err := db.Exec(fmt.Sprintf("create table if not exists %s "+ "(version varchar(255) primary key) character set latin1 collate latin1_bin", drv.quotedMigrationsTableName())) @@ -211,7 +223,7 @@ func (drv *MySQLDriver) CreateMigrationsTable(u *url.URL, db *sql.DB) error { // SelectMigrations returns a list of applied migrations // with an optional limit (in descending order) -func (drv *MySQLDriver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, error) { +func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, error) { query := fmt.Sprintf("select version from %s order by version desc", drv.quotedMigrationsTableName()) if limit >= 0 { query = fmt.Sprintf("%s limit %d", query, limit) @@ -221,7 +233,7 @@ func (drv *MySQLDriver) SelectMigrations(db *sql.DB, limit int) (map[string]bool return nil, err } - defer mustClose(rows) + defer dbutil.MustClose(rows) migrations := map[string]bool{} for rows.Next() { @@ -241,7 +253,7 @@ func (drv *MySQLDriver) SelectMigrations(db *sql.DB, limit int) (map[string]bool } // InsertMigration adds a new migration record -func (drv *MySQLDriver) InsertMigration(db Transaction, version string) error { +func (drv *Driver) InsertMigration(db dbutil.Transaction, version string) error { _, err := db.Exec( fmt.Sprintf("insert into %s (version) values (?)", drv.quotedMigrationsTableName()), version) @@ -250,7 +262,7 @@ func (drv *MySQLDriver) InsertMigration(db Transaction, version string) error { } // DeleteMigration removes a migration record -func (drv *MySQLDriver) DeleteMigration(db Transaction, version string) error { +func (drv *Driver) DeleteMigration(db dbutil.Transaction, version string) error { _, err := db.Exec( fmt.Sprintf("delete from %s where version = ?", drv.quotedMigrationsTableName()), version) @@ -260,16 +272,16 @@ func (drv *MySQLDriver) DeleteMigration(db Transaction, version string) error { // Ping verifies a connection to the database server. It does not verify whether the // specified database exists. -func (drv *MySQLDriver) Ping(u *url.URL) error { - db, err := drv.openRootDB(u) +func (drv *Driver) Ping() error { + db, err := drv.openRootDB() if err != nil { return err } - defer mustClose(db) + defer dbutil.MustClose(db) return db.Ping() } -func (drv *MySQLDriver) quotedMigrationsTableName() string { +func (drv *Driver) quotedMigrationsTableName() string { return drv.quoteIdentifier(drv.migrationsTableName) } diff --git a/pkg/dbmate/mysql_test.go b/pkg/driver/mysql/mysql_test.go similarity index 53% rename from pkg/dbmate/mysql_test.go rename to pkg/driver/mysql/mysql_test.go index c5483cc..0ff97d0 100644 --- a/pkg/dbmate/mysql_test.go +++ b/pkg/driver/mysql/mysql_test.go @@ -1,137 +1,146 @@ -package dbmate +package mysql import ( "database/sql" "net/url" + "os" "testing" + "github.com/amacneil/dbmate/pkg/dbmate" + "github.com/amacneil/dbmate/pkg/dbutil" + "github.com/stretchr/testify/require" ) -func mySQLTestURL(t *testing.T) *url.URL { - u, err := url.Parse("mysql://root:root@mysql/dbmate") +func testMySQLDriver(t *testing.T) *Driver { + u := dbutil.MustParseURL(os.Getenv("MYSQL_TEST_URL")) + drv, err := dbmate.New(u).GetDriver() require.NoError(t, err) - return u + return drv.(*Driver) } -func testMySQLDriver() *MySQLDriver { - drv := &MySQLDriver{} - drv.SetMigrationsTableName(DefaultMigrationsTableName) - - return drv -} - -func prepTestMySQLDB(t *testing.T, u *url.URL) *sql.DB { - drv := testMySQLDriver() +func prepTestMySQLDB(t *testing.T) *sql.DB { + drv := testMySQLDriver(t) // drop any existing database - err := drv.DropDatabase(u) + err := drv.DropDatabase() require.NoError(t, err) // create database - err = drv.CreateDatabase(u) + err = drv.CreateDatabase() require.NoError(t, err) // connect database - db, err := drv.Open(u) + db, err := drv.Open() require.NoError(t, err) return db } -func TestNormalizeMySQLURLDefaults(t *testing.T) { - u, err := url.Parse("mysql://host/foo") +func TestGetDriver(t *testing.T) { + db := dbmate.New(dbutil.MustParseURL("mysql://")) + drvInterface, err := db.GetDriver() require.NoError(t, err) - require.Equal(t, "", u.Port()) - s := normalizeMySQLURL(u) - require.Equal(t, "tcp(host:3306)/foo?multiStatements=true", s) + // driver should have URL and default migrations table set + drv, ok := drvInterface.(*Driver) + require.True(t, ok) + require.Equal(t, db.DatabaseURL.String(), drv.databaseURL.String()) + require.Equal(t, "schema_migrations", drv.migrationsTableName) } -func TestNormalizeMySQLURLCustom(t *testing.T) { - u, err := url.Parse("mysql://bob:secret@host:123/foo?flag=on") - require.NoError(t, err) - require.Equal(t, "123", u.Port()) +func TestConnectionString(t *testing.T) { + t.Run("defaults", func(t *testing.T) { + u, err := url.Parse("mysql://host/foo") + require.NoError(t, err) + require.Equal(t, "", u.Port()) - s := normalizeMySQLURL(u) - require.Equal(t, "bob:secret@tcp(host:123)/foo?flag=on&multiStatements=true", s) -} + s := connectionString(u) + require.Equal(t, "tcp(host:3306)/foo?multiStatements=true", s) + }) -func TestNormalizeMySQLURLCustomSpecialChars(t *testing.T) { - u, err := url.Parse("mysql://duhfsd7s:123!@123!@@host:123/foo?flag=on") - require.NoError(t, err) - require.Equal(t, "123", u.Port()) + t.Run("custom", func(t *testing.T) { + u, err := url.Parse("mysql://bob:secret@host:123/foo?flag=on") + require.NoError(t, err) + require.Equal(t, "123", u.Port()) - s := normalizeMySQLURL(u) - require.Equal(t, "duhfsd7s:123!@123!@@tcp(host:123)/foo?flag=on&multiStatements=true", s) -} + s := connectionString(u) + require.Equal(t, "bob:secret@tcp(host:123)/foo?flag=on&multiStatements=true", s) + }) -func TestNormalizeMySQLURLSocket(t *testing.T) { - // test with no user/pass - u, err := url.Parse("mysql:///foo?socket=/var/run/mysqld/mysqld.sock&flag=on") - require.NoError(t, err) - require.Equal(t, "", u.Host) + t.Run("special chars", func(t *testing.T) { + u, err := url.Parse("mysql://duhfsd7s:123!@123!@@host:123/foo?flag=on") + require.NoError(t, err) + require.Equal(t, "123", u.Port()) - s := normalizeMySQLURL(u) - require.Equal(t, "unix(/var/run/mysqld/mysqld.sock)/foo?flag=on&multiStatements=true", s) + s := connectionString(u) + require.Equal(t, "duhfsd7s:123!@123!@@tcp(host:123)/foo?flag=on&multiStatements=true", s) + }) - // test with user/pass - u, err = url.Parse("mysql://bob:secret@fakehost/foo?socket=/var/run/mysqld/mysqld.sock&flag=on") - require.NoError(t, err) + t.Run("socket", func(t *testing.T) { + // test with no user/pass + u, err := url.Parse("mysql:///foo?socket=/var/run/mysqld/mysqld.sock&flag=on") + require.NoError(t, err) + require.Equal(t, "", u.Host) - s = normalizeMySQLURL(u) - require.Equal(t, "bob:secret@unix(/var/run/mysqld/mysqld.sock)/foo?flag=on&multiStatements=true", s) + s := connectionString(u) + require.Equal(t, "unix(/var/run/mysqld/mysqld.sock)/foo?flag=on&multiStatements=true", s) + + // test with user/pass + u, err = url.Parse("mysql://bob:secret@fakehost/foo?socket=/var/run/mysqld/mysqld.sock&flag=on") + require.NoError(t, err) + + s = connectionString(u) + require.Equal(t, "bob:secret@unix(/var/run/mysqld/mysqld.sock)/foo?flag=on&multiStatements=true", s) + }) } func TestMySQLCreateDropDatabase(t *testing.T) { - drv := testMySQLDriver() - u := mySQLTestURL(t) + drv := testMySQLDriver(t) // drop any existing database - err := drv.DropDatabase(u) + err := drv.DropDatabase() require.NoError(t, err) // create database - err = drv.CreateDatabase(u) + err = drv.CreateDatabase() require.NoError(t, err) // check that database exists and we can connect to it func() { - db, err := drv.Open(u) + db, err := drv.Open() require.NoError(t, err) - defer mustClose(db) + defer dbutil.MustClose(db) err = db.Ping() require.NoError(t, err) }() // drop the database - err = drv.DropDatabase(u) + err = drv.DropDatabase() require.NoError(t, err) // check that database no longer exists func() { - db, err := drv.Open(u) + db, err := drv.Open() require.NoError(t, err) - defer mustClose(db) + defer dbutil.MustClose(db) err = db.Ping() - require.NotNil(t, err) - require.Regexp(t, "Unknown database 'dbmate'", err.Error()) + require.Error(t, err) + require.Regexp(t, "Unknown database 'dbmate_test'", err.Error()) }() } func TestMySQLDumpSchema(t *testing.T) { - drv := testMySQLDriver() - drv.SetMigrationsTableName("test_migrations") - - u := mySQLTestURL(t) + drv := testMySQLDriver(t) + drv.migrationsTableName = "test_migrations" // prepare database - db := prepTestMySQLDB(t, u) - defer mustClose(db) - err := drv.CreateMigrationsTable(u, db) + db := prepTestMySQLDB(t) + defer dbutil.MustClose(db) + err := drv.CreateMigrationsTable(db) require.NoError(t, err) // insert migration @@ -141,7 +150,7 @@ func TestMySQLDumpSchema(t *testing.T) { require.NoError(t, err) // DumpSchema should return schema - schema, err := drv.DumpSchema(u, db) + schema, err := drv.DumpSchema(db) require.NoError(t, err) require.Contains(t, string(schema), "CREATE TABLE `test_migrations`") require.Contains(t, string(schema), "\n-- Dump completed\n\n"+ @@ -155,8 +164,8 @@ func TestMySQLDumpSchema(t *testing.T) { "UNLOCK TABLES;\n") // DumpSchema should return error if command fails - u.Path = "/fakedb" - schema, err = drv.DumpSchema(u, db) + drv.databaseURL.Path = "/fakedb" + schema, err = drv.DumpSchema(db) require.Nil(t, schema) require.EqualError(t, err, "mysqldump: [Warning] Using a password "+ "on the command line interface can be insecure.\n"+ @@ -165,54 +174,52 @@ func TestMySQLDumpSchema(t *testing.T) { } func TestMySQLDatabaseExists(t *testing.T) { - drv := testMySQLDriver() - u := mySQLTestURL(t) + drv := testMySQLDriver(t) // drop any existing database - err := drv.DropDatabase(u) + err := drv.DropDatabase() require.NoError(t, err) // DatabaseExists should return false - exists, err := drv.DatabaseExists(u) + exists, err := drv.DatabaseExists() require.NoError(t, err) require.Equal(t, false, exists) // create database - err = drv.CreateDatabase(u) + err = drv.CreateDatabase() require.NoError(t, err) // DatabaseExists should return true - exists, err = drv.DatabaseExists(u) + exists, err = drv.DatabaseExists() require.NoError(t, err) require.Equal(t, true, exists) } func TestMySQLDatabaseExists_Error(t *testing.T) { - drv := testMySQLDriver() - u := mySQLTestURL(t) - u.User = url.User("invalid") + drv := testMySQLDriver(t) + drv.databaseURL.User = url.User("invalid") - exists, err := drv.DatabaseExists(u) + exists, err := drv.DatabaseExists() + require.Error(t, err) require.Regexp(t, "Access denied for user 'invalid'@", err.Error()) require.Equal(t, false, exists) } func TestMySQLCreateMigrationsTable(t *testing.T) { - drv := testMySQLDriver() - drv.SetMigrationsTableName("test_migrations") + drv := testMySQLDriver(t) + drv.migrationsTableName = "test_migrations" - u := mySQLTestURL(t) - db := prepTestMySQLDB(t, u) - defer mustClose(db) + db := prepTestMySQLDB(t) + defer dbutil.MustClose(db) // migrations table should not exist count := 0 err := db.QueryRow("select count(*) from test_migrations").Scan(&count) require.Error(t, err) - require.Regexp(t, "Table 'dbmate.test_migrations' doesn't exist", err.Error()) + require.Regexp(t, "Table 'dbmate_test.test_migrations' doesn't exist", err.Error()) // create table - err = drv.CreateMigrationsTable(u, db) + err = drv.CreateMigrationsTable(db) require.NoError(t, err) // migrations table should exist @@ -220,19 +227,18 @@ func TestMySQLCreateMigrationsTable(t *testing.T) { require.NoError(t, err) // create table should be idempotent - err = drv.CreateMigrationsTable(u, db) + err = drv.CreateMigrationsTable(db) require.NoError(t, err) } func TestMySQLSelectMigrations(t *testing.T) { - drv := testMySQLDriver() - drv.SetMigrationsTableName("test_migrations") + drv := testMySQLDriver(t) + drv.migrationsTableName = "test_migrations" - u := mySQLTestURL(t) - db := prepTestMySQLDB(t, u) - defer mustClose(db) + db := prepTestMySQLDB(t) + defer dbutil.MustClose(db) - err := drv.CreateMigrationsTable(u, db) + err := drv.CreateMigrationsTable(db) require.NoError(t, err) _, err = db.Exec(`insert into test_migrations (version) @@ -254,14 +260,13 @@ func TestMySQLSelectMigrations(t *testing.T) { } func TestMySQLInsertMigration(t *testing.T) { - drv := testMySQLDriver() - drv.SetMigrationsTableName("test_migrations") + drv := testMySQLDriver(t) + drv.migrationsTableName = "test_migrations" - u := mySQLTestURL(t) - db := prepTestMySQLDB(t, u) - defer mustClose(db) + db := prepTestMySQLDB(t) + defer dbutil.MustClose(db) - err := drv.CreateMigrationsTable(u, db) + err := drv.CreateMigrationsTable(db) require.NoError(t, err) count := 0 @@ -280,14 +285,13 @@ func TestMySQLInsertMigration(t *testing.T) { } func TestMySQLDeleteMigration(t *testing.T) { - drv := testMySQLDriver() - drv.SetMigrationsTableName("test_migrations") + drv := testMySQLDriver(t) + drv.migrationsTableName = "test_migrations" - u := mySQLTestURL(t) - db := prepTestMySQLDB(t, u) - defer mustClose(db) + db := prepTestMySQLDB(t) + defer dbutil.MustClose(db) - err := drv.CreateMigrationsTable(u, db) + err := drv.CreateMigrationsTable(db) require.NoError(t, err) _, err = db.Exec(`insert into test_migrations (version) @@ -304,34 +308,33 @@ func TestMySQLDeleteMigration(t *testing.T) { } func TestMySQLPing(t *testing.T) { - drv := testMySQLDriver() - u := mySQLTestURL(t) + drv := testMySQLDriver(t) // drop any existing database - err := drv.DropDatabase(u) + err := drv.DropDatabase() require.NoError(t, err) // ping database - err = drv.Ping(u) + err = drv.Ping() require.NoError(t, err) // ping invalid host should return error - u.Host = "mysql:404" - err = drv.Ping(u) + drv.databaseURL.Host = "mysql:404" + err = drv.Ping() require.Error(t, err) require.Contains(t, err.Error(), "connect: connection refused") } func TestMySQLQuotedMigrationsTableName(t *testing.T) { t.Run("default name", func(t *testing.T) { - drv := testMySQLDriver() + drv := testMySQLDriver(t) name := drv.quotedMigrationsTableName() require.Equal(t, "`schema_migrations`", name) }) t.Run("custom name", func(t *testing.T) { - drv := testMySQLDriver() - drv.SetMigrationsTableName("fooMigrations") + drv := testMySQLDriver(t) + drv.migrationsTableName = "fooMigrations" name := drv.quotedMigrationsTableName() require.Equal(t, "`fooMigrations`", name) diff --git a/pkg/dbmate/postgres.go b/pkg/driver/postgres/postgres.go similarity index 70% rename from pkg/dbmate/postgres.go rename to pkg/driver/postgres/postgres.go index 172ccc3..9624ea9 100644 --- a/pkg/dbmate/postgres.go +++ b/pkg/driver/postgres/postgres.go @@ -1,4 +1,4 @@ -package dbmate +package postgres import ( "bytes" @@ -7,21 +7,32 @@ import ( "net/url" "strings" + "github.com/amacneil/dbmate/pkg/dbmate" + "github.com/amacneil/dbmate/pkg/dbutil" + "github.com/lib/pq" ) func init() { - drv := &PostgresDriver{} - RegisterDriver(drv, "postgres") - RegisterDriver(drv, "postgresql") + dbmate.RegisterDriver(NewDriver, "postgres") + dbmate.RegisterDriver(NewDriver, "postgresql") } -// PostgresDriver provides top level database functions -type PostgresDriver struct { +// Driver provides top level database functions +type Driver struct { migrationsTableName string + databaseURL *url.URL } -func normalizePostgresURL(u *url.URL) *url.URL { +// NewDriver initializes the driver +func NewDriver(config dbmate.DriverConfig) dbmate.Driver { + return &Driver{ + migrationsTableName: config.MigrationsTableName, + databaseURL: config.DatabaseURL, + } +} + +func connectionString(u *url.URL) string { hostname := u.Hostname() port := u.Port() query := u.Query() @@ -56,11 +67,11 @@ func normalizePostgresURL(u *url.URL) *url.URL { out.Host = fmt.Sprintf("%s:%s", hostname, port) out.RawQuery = query.Encode() - return out + return out.String() } -func normalizePostgresURLForDump(u *url.URL) []string { - u = normalizePostgresURL(u) +func connectionArgsForDump(u *url.URL) []string { + u = dbutil.MustParseURL(connectionString(u)) // find schemas from search_path query := u.Query() @@ -80,34 +91,34 @@ func normalizePostgresURLForDump(u *url.URL) []string { return out } -// SetMigrationsTableName sets the schema migrations table name -func (drv *PostgresDriver) SetMigrationsTableName(name string) { - drv.migrationsTableName = name -} - // Open creates a new database connection -func (drv *PostgresDriver) Open(u *url.URL) (*sql.DB, error) { - return sql.Open("postgres", normalizePostgresURL(u).String()) +func (drv *Driver) Open() (*sql.DB, error) { + return sql.Open("postgres", connectionString(drv.databaseURL)) } -func (drv *PostgresDriver) openPostgresDB(u *url.URL) (*sql.DB, error) { +func (drv *Driver) openPostgresDB() (*sql.DB, error) { + // clone databaseURL + postgresURL, err := url.Parse(connectionString(drv.databaseURL)) + if err != nil { + return nil, err + } + // connect to postgres database - postgresURL := *u postgresURL.Path = "postgres" - return drv.Open(&postgresURL) + return sql.Open("postgres", postgresURL.String()) } // CreateDatabase creates the specified database -func (drv *PostgresDriver) CreateDatabase(u *url.URL) error { - name := databaseName(u) +func (drv *Driver) CreateDatabase() error { + name := dbutil.DatabaseName(drv.databaseURL) fmt.Printf("Creating: %s\n", name) - db, err := drv.openPostgresDB(u) + db, err := drv.openPostgresDB() if err != nil { return err } - defer mustClose(db) + defer dbutil.MustClose(db) _, err = db.Exec(fmt.Sprintf("create database %s", pq.QuoteIdentifier(name))) @@ -116,15 +127,15 @@ func (drv *PostgresDriver) CreateDatabase(u *url.URL) error { } // DropDatabase drops the specified database (if it exists) -func (drv *PostgresDriver) DropDatabase(u *url.URL) error { - name := databaseName(u) +func (drv *Driver) DropDatabase() error { + name := dbutil.DatabaseName(drv.databaseURL) fmt.Printf("Dropping: %s\n", name) - db, err := drv.openPostgresDB(u) + db, err := drv.openPostgresDB() if err != nil { return err } - defer mustClose(db) + defer dbutil.MustClose(db) _, err = db.Exec(fmt.Sprintf("drop database if exists %s", pq.QuoteIdentifier(name))) @@ -132,14 +143,14 @@ func (drv *PostgresDriver) DropDatabase(u *url.URL) error { return err } -func (drv *PostgresDriver) schemaMigrationsDump(db *sql.DB) ([]byte, error) { +func (drv *Driver) schemaMigrationsDump(db *sql.DB) ([]byte, error) { migrationsTable, err := drv.quotedMigrationsTableName(db) if err != nil { return nil, err } // load applied migrations - migrations, err := queryColumn(db, + migrations, err := dbutil.QueryColumn(db, "select quote_literal(version) from "+migrationsTable+" order by version asc") if err != nil { return nil, err @@ -159,11 +170,11 @@ func (drv *PostgresDriver) schemaMigrationsDump(db *sql.DB) ([]byte, error) { } // DumpSchema returns the current database schema -func (drv *PostgresDriver) DumpSchema(u *url.URL, db *sql.DB) ([]byte, error) { +func (drv *Driver) DumpSchema(db *sql.DB) ([]byte, error) { // load schema args := append([]string{"--format=plain", "--encoding=UTF8", "--schema-only", - "--no-privileges", "--no-owner"}, normalizePostgresURLForDump(u)...) - schema, err := runCommand("pg_dump", args...) + "--no-privileges", "--no-owner"}, connectionArgsForDump(drv.databaseURL)...) + schema, err := dbutil.RunCommand("pg_dump", args...) if err != nil { return nil, err } @@ -174,18 +185,18 @@ func (drv *PostgresDriver) DumpSchema(u *url.URL, db *sql.DB) ([]byte, error) { } schema = append(schema, migrations...) - return trimLeadingSQLComments(schema) + return dbutil.TrimLeadingSQLComments(schema) } // DatabaseExists determines whether the database exists -func (drv *PostgresDriver) DatabaseExists(u *url.URL) (bool, error) { - name := databaseName(u) +func (drv *Driver) DatabaseExists() (bool, error) { + name := dbutil.DatabaseName(drv.databaseURL) - db, err := drv.openPostgresDB(u) + db, err := drv.openPostgresDB() if err != nil { return false, err } - defer mustClose(db) + defer dbutil.MustClose(db) exists := false err = db.QueryRow("select true from pg_database where datname = $1", name). @@ -198,8 +209,8 @@ func (drv *PostgresDriver) DatabaseExists(u *url.URL) (bool, error) { } // CreateMigrationsTable creates the schema_migrations table -func (drv *PostgresDriver) CreateMigrationsTable(u *url.URL, db *sql.DB) error { - schema, migrationsTable, err := drv.quotedMigrationsTableNameParts(db, u) +func (drv *Driver) CreateMigrationsTable(db *sql.DB) error { + schema, migrationsTable, err := drv.quotedMigrationsTableNameParts(db) if err != nil { return err } @@ -235,7 +246,7 @@ func (drv *PostgresDriver) CreateMigrationsTable(u *url.URL, db *sql.DB) error { // SelectMigrations returns a list of applied migrations // with an optional limit (in descending order) -func (drv *PostgresDriver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, error) { +func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, error) { migrationsTable, err := drv.quotedMigrationsTableName(db) if err != nil { return nil, err @@ -250,7 +261,7 @@ func (drv *PostgresDriver) SelectMigrations(db *sql.DB, limit int) (map[string]b return nil, err } - defer mustClose(rows) + defer dbutil.MustClose(rows) migrations := map[string]bool{} for rows.Next() { @@ -270,7 +281,7 @@ func (drv *PostgresDriver) SelectMigrations(db *sql.DB, limit int) (map[string]b } // InsertMigration adds a new migration record -func (drv *PostgresDriver) InsertMigration(db Transaction, version string) error { +func (drv *Driver) InsertMigration(db dbutil.Transaction, version string) error { migrationsTable, err := drv.quotedMigrationsTableName(db) if err != nil { return err @@ -282,7 +293,7 @@ func (drv *PostgresDriver) InsertMigration(db Transaction, version string) error } // DeleteMigration removes a migration record -func (drv *PostgresDriver) DeleteMigration(db Transaction, version string) error { +func (drv *Driver) DeleteMigration(db dbutil.Transaction, version string) error { migrationsTable, err := drv.quotedMigrationsTableName(db) if err != nil { return err @@ -295,15 +306,15 @@ func (drv *PostgresDriver) DeleteMigration(db Transaction, version string) error // Ping verifies a connection to the database server. It does not verify whether the // specified database exists. -func (drv *PostgresDriver) Ping(u *url.URL) error { +func (drv *Driver) Ping() error { // attempt connection to primary database, not "postgres" database // to support servers with no "postgres" database // (see https://github.com/amacneil/dbmate/issues/78) - db, err := drv.Open(u) + db, err := drv.Open() if err != nil { return err } - defer mustClose(db) + defer dbutil.MustClose(db) err = db.Ping() if err == nil { @@ -319,8 +330,8 @@ func (drv *PostgresDriver) Ping(u *url.URL) error { return err } -func (drv *PostgresDriver) quotedMigrationsTableName(db Transaction) (string, error) { - schema, name, err := drv.quotedMigrationsTableNameParts(db, nil) +func (drv *Driver) quotedMigrationsTableName(db dbutil.Transaction) (string, error) { + schema, name, err := drv.quotedMigrationsTableNameParts(db) if err != nil { return "", err } @@ -328,7 +339,7 @@ func (drv *PostgresDriver) quotedMigrationsTableName(db Transaction) (string, er return schema + "." + name, nil } -func (drv *PostgresDriver) quotedMigrationsTableNameParts(db Transaction, u *url.URL) (string, string, error) { +func (drv *Driver) quotedMigrationsTableNameParts(db dbutil.Transaction) (string, string, error) { schema := "" tableNameParts := strings.Split(drv.migrationsTableName, ".") if len(tableNameParts) > 1 { @@ -336,9 +347,9 @@ func (drv *PostgresDriver) quotedMigrationsTableNameParts(db Transaction, u *url schema, tableNameParts = tableNameParts[0], tableNameParts[1:] } - if schema == "" && u != nil { + if schema == "" { // no schema specified with table name, try URL search path if available - searchPath := strings.Split(u.Query().Get("search_path"), ",") + searchPath := strings.Split(drv.databaseURL.Query().Get("search_path"), ",") schema = strings.TrimSpace(searchPath[0]) } @@ -346,7 +357,7 @@ func (drv *PostgresDriver) quotedMigrationsTableNameParts(db Transaction, u *url if schema == "" { // if no URL available, use current schema // this is a hack because we don't always have the URL context available - schema, err = queryValue(db, "select current_schema()") + schema, err = dbutil.QueryValue(db, "select current_schema()") if err != nil { return "", "", err } @@ -361,7 +372,7 @@ func (drv *PostgresDriver) quotedMigrationsTableNameParts(db Transaction, u *url // use server rather than client to do this to avoid unnecessary quotes // (which would change schema.sql diff) tableNameParts = append([]string{schema}, tableNameParts...) - quotedNameParts, err := queryColumn(db, "select quote_ident(unnest($1::text[]))", pq.Array(tableNameParts)) + quotedNameParts, err := dbutil.QueryColumn(db, "select quote_ident(unnest($1::text[]))", pq.Array(tableNameParts)) if err != nil { return "", "", err } diff --git a/pkg/dbmate/postgres_test.go b/pkg/driver/postgres/postgres_test.go similarity index 71% rename from pkg/dbmate/postgres_test.go rename to pkg/driver/postgres/postgres_test.go index a2e63d0..f3f8793 100644 --- a/pkg/dbmate/postgres_test.go +++ b/pkg/driver/postgres/postgres_test.go @@ -1,46 +1,56 @@ -package dbmate +package postgres import ( "database/sql" "net/url" + "os" "testing" + "github.com/amacneil/dbmate/pkg/dbmate" + "github.com/amacneil/dbmate/pkg/dbutil" + "github.com/stretchr/testify/require" ) -func postgresTestURL(t *testing.T) *url.URL { - u, err := url.Parse("postgres://postgres:postgres@postgres/dbmate?sslmode=disable") +func testPostgresDriver(t *testing.T) *Driver { + u := dbutil.MustParseURL(os.Getenv("POSTGRES_TEST_URL")) + drv, err := dbmate.New(u).GetDriver() require.NoError(t, err) - return u + return drv.(*Driver) } -func testPostgresDriver() *PostgresDriver { - drv := &PostgresDriver{} - drv.SetMigrationsTableName(DefaultMigrationsTableName) - - return drv -} - -func prepTestPostgresDB(t *testing.T, u *url.URL) *sql.DB { - drv := testPostgresDriver() +func prepTestPostgresDB(t *testing.T) *sql.DB { + drv := testPostgresDriver(t) // drop any existing database - err := drv.DropDatabase(u) + err := drv.DropDatabase() require.NoError(t, err) // create database - err = drv.CreateDatabase(u) + err = drv.CreateDatabase() require.NoError(t, err) // connect database - db, err := sql.Open("postgres", u.String()) + db, err := sql.Open("postgres", drv.databaseURL.String()) require.NoError(t, err) return db } -func TestNormalizePostgresURL(t *testing.T) { +func TestGetDriver(t *testing.T) { + db := dbmate.New(dbutil.MustParseURL("postgres://")) + drvInterface, err := db.GetDriver() + require.NoError(t, err) + + // driver should have URL and default migrations table set + drv, ok := drvInterface.(*Driver) + require.True(t, ok) + require.Equal(t, db.DatabaseURL.String(), drv.databaseURL.String()) + require.Equal(t, "schema_migrations", drv.migrationsTableName) +} + +func TestConnectionString(t *testing.T) { cases := []struct { input string expected string @@ -63,13 +73,13 @@ func TestNormalizePostgresURL(t *testing.T) { u, err := url.Parse(c.input) require.NoError(t, err) - actual := normalizePostgresURL(u).String() + actual := connectionString(u) require.Equal(t, c.expected, actual) }) } } -func TestNormalizePostgresURLForDump(t *testing.T) { +func TestConnectionArgsForDump(t *testing.T) { cases := []struct { input string expected []string @@ -87,59 +97,57 @@ func TestNormalizePostgresURLForDump(t *testing.T) { u, err := url.Parse(c.input) require.NoError(t, err) - actual := normalizePostgresURLForDump(u) + actual := connectionArgsForDump(u) require.Equal(t, c.expected, actual) }) } } func TestPostgresCreateDropDatabase(t *testing.T) { - drv := testPostgresDriver() - u := postgresTestURL(t) + drv := testPostgresDriver(t) // drop any existing database - err := drv.DropDatabase(u) + err := drv.DropDatabase() require.NoError(t, err) // create database - err = drv.CreateDatabase(u) + err = drv.CreateDatabase() require.NoError(t, err) // check that database exists and we can connect to it func() { - db, err := sql.Open("postgres", u.String()) + db, err := sql.Open("postgres", drv.databaseURL.String()) require.NoError(t, err) - defer mustClose(db) + defer dbutil.MustClose(db) err = db.Ping() require.NoError(t, err) }() // drop the database - err = drv.DropDatabase(u) + err = drv.DropDatabase() require.NoError(t, err) // check that database no longer exists func() { - db, err := sql.Open("postgres", u.String()) + db, err := sql.Open("postgres", drv.databaseURL.String()) require.NoError(t, err) - defer mustClose(db) + defer dbutil.MustClose(db) err = db.Ping() require.Error(t, err) - require.Equal(t, "pq: database \"dbmate\" does not exist", err.Error()) + require.Equal(t, "pq: database \"dbmate_test\" does not exist", err.Error()) }() } func TestPostgresDumpSchema(t *testing.T) { t.Run("default migrations table", func(t *testing.T) { - drv := testPostgresDriver() - u := postgresTestURL(t) + drv := testPostgresDriver(t) // prepare database - db := prepTestPostgresDB(t, u) - defer mustClose(db) - err := drv.CreateMigrationsTable(u, db) + db := prepTestPostgresDB(t) + defer dbutil.MustClose(db) + err := drv.CreateMigrationsTable(db) require.NoError(t, err) // insert migration @@ -149,7 +157,7 @@ func TestPostgresDumpSchema(t *testing.T) { require.NoError(t, err) // DumpSchema should return schema - schema, err := drv.DumpSchema(u, db) + schema, err := drv.DumpSchema(db) require.NoError(t, err) require.Contains(t, string(schema), "CREATE TABLE public.schema_migrations") require.Contains(t, string(schema), "\n--\n"+ @@ -163,23 +171,21 @@ func TestPostgresDumpSchema(t *testing.T) { " ('abc2');\n") // DumpSchema should return error if command fails - u.Path = "/fakedb" - schema, err = drv.DumpSchema(u, db) + drv.databaseURL.Path = "/fakedb" + schema, err = drv.DumpSchema(db) require.Nil(t, schema) require.EqualError(t, err, "pg_dump: [archiver (db)] connection to database "+ "\"fakedb\" failed: FATAL: database \"fakedb\" does not exist") }) t.Run("custom migrations table with schema", func(t *testing.T) { - drv := testPostgresDriver() - drv.SetMigrationsTableName("camelSchema.testMigrations") - - u := postgresTestURL(t) + drv := testPostgresDriver(t) + drv.migrationsTableName = "camelSchema.testMigrations" // prepare database - db := prepTestPostgresDB(t, u) - defer mustClose(db) - err := drv.CreateMigrationsTable(u, db) + db := prepTestPostgresDB(t) + defer dbutil.MustClose(db) + err := drv.CreateMigrationsTable(db) require.NoError(t, err) // insert migration @@ -189,7 +195,7 @@ func TestPostgresDumpSchema(t *testing.T) { require.NoError(t, err) // DumpSchema should return schema - schema, err := drv.DumpSchema(u, db) + schema, err := drv.DumpSchema(db) require.NoError(t, err) require.Contains(t, string(schema), "CREATE TABLE \"camelSchema\".\"testMigrations\"") require.Contains(t, string(schema), "\n--\n"+ @@ -205,34 +211,32 @@ func TestPostgresDumpSchema(t *testing.T) { } func TestPostgresDatabaseExists(t *testing.T) { - drv := testPostgresDriver() - u := postgresTestURL(t) + drv := testPostgresDriver(t) // drop any existing database - err := drv.DropDatabase(u) + err := drv.DropDatabase() require.NoError(t, err) // DatabaseExists should return false - exists, err := drv.DatabaseExists(u) + exists, err := drv.DatabaseExists() require.NoError(t, err) require.Equal(t, false, exists) // create database - err = drv.CreateDatabase(u) + err = drv.CreateDatabase() require.NoError(t, err) // DatabaseExists should return true - exists, err = drv.DatabaseExists(u) + exists, err = drv.DatabaseExists() require.NoError(t, err) require.Equal(t, true, exists) } func TestPostgresDatabaseExists_Error(t *testing.T) { - drv := testPostgresDriver() - u := postgresTestURL(t) - u.User = url.User("invalid") + drv := testPostgresDriver(t) + drv.databaseURL.User = url.User("invalid") - exists, err := drv.DatabaseExists(u) + exists, err := drv.DatabaseExists() require.Error(t, err) require.Equal(t, "pq: password authentication failed for user \"invalid\"", err.Error()) require.Equal(t, false, exists) @@ -240,10 +244,9 @@ func TestPostgresDatabaseExists_Error(t *testing.T) { func TestPostgresCreateMigrationsTable(t *testing.T) { t.Run("default schema", func(t *testing.T) { - drv := testPostgresDriver() - u := postgresTestURL(t) - db := prepTestPostgresDB(t, u) - defer mustClose(db) + drv := testPostgresDriver(t) + db := prepTestPostgresDB(t) + defer dbutil.MustClose(db) // migrations table should not exist count := 0 @@ -252,7 +255,7 @@ func TestPostgresCreateMigrationsTable(t *testing.T) { require.Equal(t, "pq: relation \"public.schema_migrations\" does not exist", err.Error()) // create table - err = drv.CreateMigrationsTable(u, db) + err = drv.CreateMigrationsTable(db) require.NoError(t, err) // migrations table should exist @@ -260,18 +263,20 @@ func TestPostgresCreateMigrationsTable(t *testing.T) { require.NoError(t, err) // create table should be idempotent - err = drv.CreateMigrationsTable(u, db) + err = drv.CreateMigrationsTable(db) require.NoError(t, err) }) t.Run("custom search path", func(t *testing.T) { - drv := testPostgresDriver() - drv.SetMigrationsTableName("testMigrations") + drv := testPostgresDriver(t) + drv.migrationsTableName = "testMigrations" - u, err := url.Parse(postgresTestURL(t).String() + "&search_path=camelFoo") + u, err := url.Parse(drv.databaseURL.String() + "&search_path=camelFoo") require.NoError(t, err) - db := prepTestPostgresDB(t, u) - defer mustClose(db) + drv.databaseURL = u + + db := prepTestPostgresDB(t) + defer dbutil.MustClose(db) // delete schema _, err = db.Exec("drop schema if exists \"camelFoo\"") @@ -291,7 +296,7 @@ func TestPostgresCreateMigrationsTable(t *testing.T) { require.Equal(t, "pq: relation \"public.testMigrations\" does not exist", err.Error()) // create table - err = drv.CreateMigrationsTable(u, db) + err = drv.CreateMigrationsTable(db) require.NoError(t, err) // camelFoo schema should be created, and migrations table should exist only in camelFoo schema @@ -302,18 +307,20 @@ func TestPostgresCreateMigrationsTable(t *testing.T) { require.Equal(t, "pq: relation \"public.testMigrations\" does not exist", err.Error()) // create table should be idempotent - err = drv.CreateMigrationsTable(u, db) + err = drv.CreateMigrationsTable(db) require.NoError(t, err) }) t.Run("custom schema", func(t *testing.T) { - drv := testPostgresDriver() - drv.SetMigrationsTableName("camelSchema.testMigrations") + drv := testPostgresDriver(t) + drv.migrationsTableName = "camelSchema.testMigrations" - u, err := url.Parse(postgresTestURL(t).String() + "&search_path=foo") + u, err := url.Parse(drv.databaseURL.String() + "&search_path=foo") require.NoError(t, err) - db := prepTestPostgresDB(t, u) - defer mustClose(db) + drv.databaseURL = u + + db := prepTestPostgresDB(t) + defer dbutil.MustClose(db) // delete schemas _, err = db.Exec("drop schema if exists foo") @@ -328,7 +335,7 @@ func TestPostgresCreateMigrationsTable(t *testing.T) { require.Equal(t, "pq: relation \"camelSchema.testMigrations\" does not exist", err.Error()) // create table - err = drv.CreateMigrationsTable(u, db) + err = drv.CreateMigrationsTable(db) require.NoError(t, err) // camelSchema should be created, and testMigrations table should exist @@ -341,20 +348,19 @@ func TestPostgresCreateMigrationsTable(t *testing.T) { require.Equal(t, "pq: relation \"foo.testMigrations\" does not exist", err.Error()) // create table should be idempotent - err = drv.CreateMigrationsTable(u, db) + err = drv.CreateMigrationsTable(db) require.NoError(t, err) }) } func TestPostgresSelectMigrations(t *testing.T) { - drv := testPostgresDriver() - drv.SetMigrationsTableName("test_migrations") + drv := testPostgresDriver(t) + drv.migrationsTableName = "test_migrations" - u := postgresTestURL(t) - db := prepTestPostgresDB(t, u) - defer mustClose(db) + db := prepTestPostgresDB(t) + defer dbutil.MustClose(db) - err := drv.CreateMigrationsTable(u, db) + err := drv.CreateMigrationsTable(db) require.NoError(t, err) _, err = db.Exec(`insert into public.test_migrations (version) @@ -376,14 +382,13 @@ func TestPostgresSelectMigrations(t *testing.T) { } func TestPostgresInsertMigration(t *testing.T) { - drv := testPostgresDriver() - drv.SetMigrationsTableName("test_migrations") + drv := testPostgresDriver(t) + drv.migrationsTableName = "test_migrations" - u := postgresTestURL(t) - db := prepTestPostgresDB(t, u) - defer mustClose(db) + db := prepTestPostgresDB(t) + defer dbutil.MustClose(db) - err := drv.CreateMigrationsTable(u, db) + err := drv.CreateMigrationsTable(db) require.NoError(t, err) count := 0 @@ -402,14 +407,13 @@ func TestPostgresInsertMigration(t *testing.T) { } func TestPostgresDeleteMigration(t *testing.T) { - drv := testPostgresDriver() - drv.SetMigrationsTableName("test_migrations") + drv := testPostgresDriver(t) + drv.migrationsTableName = "test_migrations" - u := postgresTestURL(t) - db := prepTestPostgresDB(t, u) - defer mustClose(db) + db := prepTestPostgresDB(t) + defer dbutil.MustClose(db) - err := drv.CreateMigrationsTable(u, db) + err := drv.CreateMigrationsTable(db) require.NoError(t, err) _, err = db.Exec(`insert into public.test_migrations (version) @@ -426,31 +430,28 @@ func TestPostgresDeleteMigration(t *testing.T) { } func TestPostgresPing(t *testing.T) { - drv := testPostgresDriver() - u := postgresTestURL(t) + drv := testPostgresDriver(t) // drop any existing database - err := drv.DropDatabase(u) + err := drv.DropDatabase() require.NoError(t, err) // ping database - err = drv.Ping(u) + err = drv.Ping() require.NoError(t, err) // ping invalid host should return error - u.Host = "postgres:404" - err = drv.Ping(u) + drv.databaseURL.Host = "postgres:404" + err = drv.Ping() require.Error(t, err) require.Contains(t, err.Error(), "connect: connection refused") } func TestPostgresQuotedMigrationsTableName(t *testing.T) { - drv := testPostgresDriver() - t.Run("default schema", func(t *testing.T) { - u := postgresTestURL(t) - db := prepTestPostgresDB(t, u) - defer mustClose(db) + drv := testPostgresDriver(t) + db := prepTestPostgresDB(t) + defer dbutil.MustClose(db) name, err := drv.quotedMigrationsTableName(db) require.NoError(t, err) @@ -458,32 +459,29 @@ func TestPostgresQuotedMigrationsTableName(t *testing.T) { }) t.Run("custom schema", func(t *testing.T) { - u, err := url.Parse(postgresTestURL(t).String() + "&search_path=foo,bar,public") + drv := testPostgresDriver(t) + u, err := url.Parse(drv.databaseURL.String() + "&search_path=foo,bar,public") require.NoError(t, err) - db := prepTestPostgresDB(t, u) - defer mustClose(db) + drv.databaseURL = u + + db := prepTestPostgresDB(t) + defer dbutil.MustClose(db) - // if "foo" schema does not exist, current schema should be "public" _, err = db.Exec("drop schema if exists foo") require.NoError(t, err) _, err = db.Exec("drop schema if exists bar") require.NoError(t, err) - name, err := drv.quotedMigrationsTableName(db) - require.NoError(t, err) - require.Equal(t, "public.schema_migrations", name) - // if "foo" schema exists, it should be used - _, err = db.Exec("create schema foo") - require.NoError(t, err) - name, err = drv.quotedMigrationsTableName(db) + // should use first schema from search path + name, err := drv.quotedMigrationsTableName(db) require.NoError(t, err) require.Equal(t, "foo.schema_migrations", name) }) t.Run("no schema", func(t *testing.T) { - u := postgresTestURL(t) - db := prepTestPostgresDB(t, u) - defer mustClose(db) + drv := testPostgresDriver(t) + db := prepTestPostgresDB(t) + defer dbutil.MustClose(db) // this is an unlikely edge case, but if for some reason there is // no current schema then we should default to "public" @@ -496,48 +494,54 @@ func TestPostgresQuotedMigrationsTableName(t *testing.T) { }) t.Run("custom table name", func(t *testing.T) { - u := postgresTestURL(t) - db := prepTestPostgresDB(t, u) - defer mustClose(db) + drv := testPostgresDriver(t) + db := prepTestPostgresDB(t) + defer dbutil.MustClose(db) - drv.SetMigrationsTableName("simple_name") + drv.migrationsTableName = "simple_name" name, err := drv.quotedMigrationsTableName(db) require.NoError(t, err) require.Equal(t, "public.simple_name", name) }) t.Run("custom table name quoted", func(t *testing.T) { - u := postgresTestURL(t) - db := prepTestPostgresDB(t, u) - defer mustClose(db) + drv := testPostgresDriver(t) + db := prepTestPostgresDB(t) + defer dbutil.MustClose(db) // this table name will need quoting - drv.SetMigrationsTableName("camelCase") + drv.migrationsTableName = "camelCase" name, err := drv.quotedMigrationsTableName(db) require.NoError(t, err) require.Equal(t, "public.\"camelCase\"", name) }) t.Run("custom table name with custom schema", func(t *testing.T) { - u, err := url.Parse(postgresTestURL(t).String() + "&search_path=foo") + drv := testPostgresDriver(t) + u, err := url.Parse(drv.databaseURL.String() + "&search_path=foo") require.NoError(t, err) - db := prepTestPostgresDB(t, u) - defer mustClose(db) + drv.databaseURL = u + + db := prepTestPostgresDB(t) + defer dbutil.MustClose(db) _, err = db.Exec("create schema if not exists foo") require.NoError(t, err) - drv.SetMigrationsTableName("simple_name") + drv.migrationsTableName = "simple_name" name, err := drv.quotedMigrationsTableName(db) require.NoError(t, err) require.Equal(t, "foo.simple_name", name) }) t.Run("custom table name overrides schema", func(t *testing.T) { - u, err := url.Parse(postgresTestURL(t).String() + "&search_path=foo") + drv := testPostgresDriver(t) + u, err := url.Parse(drv.databaseURL.String() + "&search_path=foo") require.NoError(t, err) - db := prepTestPostgresDB(t, u) - defer mustClose(db) + drv.databaseURL = u + + db := prepTestPostgresDB(t) + defer dbutil.MustClose(db) _, err = db.Exec("create schema if not exists foo") require.NoError(t, err) @@ -545,19 +549,19 @@ func TestPostgresQuotedMigrationsTableName(t *testing.T) { require.NoError(t, err) // if schema is specified as part of table name, it should override search_path - drv.SetMigrationsTableName("bar.simple_name") + drv.migrationsTableName = "bar.simple_name" name, err := drv.quotedMigrationsTableName(db) require.NoError(t, err) require.Equal(t, "bar.simple_name", name) // schema and table name should be quoted if necessary - drv.SetMigrationsTableName("barName.camelTable") + drv.migrationsTableName = "barName.camelTable" name, err = drv.quotedMigrationsTableName(db) require.NoError(t, err) require.Equal(t, "\"barName\".\"camelTable\"", name) // more than 2 components is unexpected but we will quote and pass it along anyway - drv.SetMigrationsTableName("whyWould.i.doThis") + drv.migrationsTableName = "whyWould.i.doThis" name, err = drv.quotedMigrationsTableName(db) require.NoError(t, err) require.Equal(t, "\"whyWould\".i.\"doThis\"", name) diff --git a/pkg/dbmate/sqlite.go b/pkg/driver/sqlite/sqlite.go similarity index 59% rename from pkg/dbmate/sqlite.go rename to pkg/driver/sqlite/sqlite.go index f762d4c..99ffe3d 100644 --- a/pkg/dbmate/sqlite.go +++ b/pkg/driver/sqlite/sqlite.go @@ -1,6 +1,6 @@ // +build cgo -package dbmate +package sqlite import ( "bytes" @@ -11,58 +11,68 @@ import ( "regexp" "strings" + "github.com/amacneil/dbmate/pkg/dbmate" + "github.com/amacneil/dbmate/pkg/dbutil" + "github.com/lib/pq" - _ "github.com/mattn/go-sqlite3" // sqlite driver for database/sql + _ "github.com/mattn/go-sqlite3" // database/sql driver ) func init() { - drv := &SQLiteDriver{} - RegisterDriver(drv, "sqlite") - RegisterDriver(drv, "sqlite3") + dbmate.RegisterDriver(NewDriver, "sqlite") + dbmate.RegisterDriver(NewDriver, "sqlite3") } -// SQLiteDriver provides top level database functions -type SQLiteDriver struct { +// Driver provides top level database functions +type Driver struct { migrationsTableName string + databaseURL *url.URL } -func sqlitePath(u *url.URL) string { - // strip one leading slash - // absolute URLs can be specified as sqlite:////tmp/foo.sqlite3 - str := regexp.MustCompile("^/").ReplaceAllString(u.Path, "") +// NewDriver initializes the driver +func NewDriver(config dbmate.DriverConfig) dbmate.Driver { + return &Driver{ + migrationsTableName: config.MigrationsTableName, + databaseURL: config.DatabaseURL, + } +} + +// ConnectionString converts a URL into a valid connection string +func ConnectionString(u *url.URL) string { + // duplicate URL and remove scheme + newURL := *u + newURL.Scheme = "" + + // trim duplicate leading slashes + str := regexp.MustCompile("^//+").ReplaceAllString(newURL.String(), "/") return str } -// SetMigrationsTableName sets the schema migrations table name -func (drv *SQLiteDriver) SetMigrationsTableName(name string) { - drv.migrationsTableName = name -} - // Open creates a new database connection -func (drv *SQLiteDriver) Open(u *url.URL) (*sql.DB, error) { - return sql.Open("sqlite3", sqlitePath(u)) +func (drv *Driver) Open() (*sql.DB, error) { + return sql.Open("sqlite3", ConnectionString(drv.databaseURL)) } // CreateDatabase creates the specified database -func (drv *SQLiteDriver) CreateDatabase(u *url.URL) error { - fmt.Printf("Creating: %s\n", sqlitePath(u)) +func (drv *Driver) CreateDatabase() error { + fmt.Printf("Creating: %s\n", ConnectionString(drv.databaseURL)) - db, err := drv.Open(u) + db, err := drv.Open() if err != nil { return err } - defer mustClose(db) + defer dbutil.MustClose(db) return db.Ping() } // DropDatabase drops the specified database (if it exists) -func (drv *SQLiteDriver) DropDatabase(u *url.URL) error { - path := sqlitePath(u) +func (drv *Driver) DropDatabase() error { + path := ConnectionString(drv.databaseURL) fmt.Printf("Dropping: %s\n", path) - exists, err := drv.DatabaseExists(u) + exists, err := drv.DatabaseExists() if err != nil { return err } @@ -73,11 +83,11 @@ func (drv *SQLiteDriver) DropDatabase(u *url.URL) error { return os.Remove(path) } -func (drv *SQLiteDriver) schemaMigrationsDump(db *sql.DB) ([]byte, error) { +func (drv *Driver) schemaMigrationsDump(db *sql.DB) ([]byte, error) { migrationsTable := drv.quotedMigrationsTableName() // load applied migrations - migrations, err := queryColumn(db, + migrations, err := dbutil.QueryColumn(db, fmt.Sprintf("select quote(version) from %s order by version asc", migrationsTable)) if err != nil { return nil, err @@ -98,9 +108,9 @@ func (drv *SQLiteDriver) schemaMigrationsDump(db *sql.DB) ([]byte, error) { } // DumpSchema returns the current database schema -func (drv *SQLiteDriver) DumpSchema(u *url.URL, db *sql.DB) ([]byte, error) { - path := sqlitePath(u) - schema, err := runCommand("sqlite3", path, ".schema") +func (drv *Driver) DumpSchema(db *sql.DB) ([]byte, error) { + path := ConnectionString(drv.databaseURL) + schema, err := dbutil.RunCommand("sqlite3", path, ".schema") if err != nil { return nil, err } @@ -111,12 +121,12 @@ func (drv *SQLiteDriver) DumpSchema(u *url.URL, db *sql.DB) ([]byte, error) { } schema = append(schema, migrations...) - return trimLeadingSQLComments(schema) + return dbutil.TrimLeadingSQLComments(schema) } // DatabaseExists determines whether the database exists -func (drv *SQLiteDriver) DatabaseExists(u *url.URL) (bool, error) { - _, err := os.Stat(sqlitePath(u)) +func (drv *Driver) DatabaseExists() (bool, error) { + _, err := os.Stat(ConnectionString(drv.databaseURL)) if os.IsNotExist(err) { return false, nil } @@ -128,7 +138,7 @@ func (drv *SQLiteDriver) DatabaseExists(u *url.URL) (bool, error) { } // CreateMigrationsTable creates the schema migrations table -func (drv *SQLiteDriver) CreateMigrationsTable(u *url.URL, db *sql.DB) error { +func (drv *Driver) CreateMigrationsTable(db *sql.DB) error { _, err := db.Exec( fmt.Sprintf("create table if not exists %s ", drv.quotedMigrationsTableName()) + "(version varchar(255) primary key)") @@ -138,7 +148,7 @@ func (drv *SQLiteDriver) CreateMigrationsTable(u *url.URL, db *sql.DB) error { // SelectMigrations returns a list of applied migrations // with an optional limit (in descending order) -func (drv *SQLiteDriver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, error) { +func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, error) { query := fmt.Sprintf("select version from %s order by version desc", drv.quotedMigrationsTableName()) if limit >= 0 { query = fmt.Sprintf("%s limit %d", query, limit) @@ -148,7 +158,7 @@ func (drv *SQLiteDriver) SelectMigrations(db *sql.DB, limit int) (map[string]boo return nil, err } - defer mustClose(rows) + defer dbutil.MustClose(rows) migrations := map[string]bool{} for rows.Next() { @@ -168,7 +178,7 @@ func (drv *SQLiteDriver) SelectMigrations(db *sql.DB, limit int) (map[string]boo } // InsertMigration adds a new migration record -func (drv *SQLiteDriver) InsertMigration(db Transaction, version string) error { +func (drv *Driver) InsertMigration(db dbutil.Transaction, version string) error { _, err := db.Exec( fmt.Sprintf("insert into %s (version) values (?)", drv.quotedMigrationsTableName()), version) @@ -177,7 +187,7 @@ func (drv *SQLiteDriver) InsertMigration(db Transaction, version string) error { } // DeleteMigration removes a migration record -func (drv *SQLiteDriver) DeleteMigration(db Transaction, version string) error { +func (drv *Driver) DeleteMigration(db dbutil.Transaction, version string) error { _, err := db.Exec( fmt.Sprintf("delete from %s where version = ?", drv.quotedMigrationsTableName()), version) @@ -188,23 +198,23 @@ func (drv *SQLiteDriver) DeleteMigration(db Transaction, version string) error { // Ping verifies a connection to the database. Due to the way SQLite works, by // testing whether the database is valid, it will automatically create the database // if it does not already exist. -func (drv *SQLiteDriver) Ping(u *url.URL) error { - db, err := drv.Open(u) +func (drv *Driver) Ping() error { + db, err := drv.Open() if err != nil { return err } - defer mustClose(db) + defer dbutil.MustClose(db) return db.Ping() } -func (drv *SQLiteDriver) quotedMigrationsTableName() string { +func (drv *Driver) quotedMigrationsTableName() string { return drv.quoteIdentifier(drv.migrationsTableName) } // quoteIdentifier quotes a table or column name // we fall back to lib/pq implementation since both use ansi standard (double quotes) // and mattn/go-sqlite3 doesn't provide a sqlite-specific equivalent -func (drv *SQLiteDriver) quoteIdentifier(s string) string { +func (drv *Driver) quoteIdentifier(s string) string { return pq.QuoteIdentifier(s) } diff --git a/pkg/dbmate/sqlite_test.go b/pkg/driver/sqlite/sqlite_test.go similarity index 60% rename from pkg/dbmate/sqlite_test.go rename to pkg/driver/sqlite/sqlite_test.go index 5602188..f4638f1 100644 --- a/pkg/dbmate/sqlite_test.go +++ b/pkg/driver/sqlite/sqlite_test.go @@ -1,59 +1,91 @@ // +build cgo -package dbmate +package sqlite import ( "database/sql" - "net/url" "os" "testing" + "github.com/amacneil/dbmate/pkg/dbmate" + "github.com/amacneil/dbmate/pkg/dbutil" + "github.com/stretchr/testify/require" ) -func sqliteTestURL(t *testing.T) *url.URL { - u, err := url.Parse("sqlite3:////tmp/dbmate.sqlite3") +func testSQLiteDriver(t *testing.T) *Driver { + u := dbutil.MustParseURL(os.Getenv("SQLITE_TEST_URL")) + drv, err := dbmate.New(u).GetDriver() require.NoError(t, err) - return u + return drv.(*Driver) } -func testSQLiteDriver() *SQLiteDriver { - drv := &SQLiteDriver{} - drv.SetMigrationsTableName(DefaultMigrationsTableName) - - return drv -} - -func prepTestSQLiteDB(t *testing.T, u *url.URL) *sql.DB { - drv := testSQLiteDriver() +func prepTestSQLiteDB(t *testing.T) *sql.DB { + drv := testSQLiteDriver(t) // drop any existing database - err := drv.DropDatabase(u) + err := drv.DropDatabase() require.NoError(t, err) // create database - err = drv.CreateDatabase(u) + err = drv.CreateDatabase() require.NoError(t, err) // connect database - db, err := drv.Open(u) + db, err := drv.Open() require.NoError(t, err) return db } +func TestGetDriver(t *testing.T) { + db := dbmate.New(dbutil.MustParseURL("sqlite://")) + drvInterface, err := db.GetDriver() + require.NoError(t, err) + + // driver should have URL and default migrations table set + drv, ok := drvInterface.(*Driver) + require.True(t, ok) + require.Equal(t, db.DatabaseURL.String(), drv.databaseURL.String()) + require.Equal(t, "schema_migrations", drv.migrationsTableName) +} + +func TestConnectionString(t *testing.T) { + t.Run("relative", func(t *testing.T) { + u := dbutil.MustParseURL("sqlite:foo/bar.sqlite3?mode=ro") + require.Equal(t, "foo/bar.sqlite3?mode=ro", ConnectionString(u)) + }) + + t.Run("absolute", func(t *testing.T) { + u := dbutil.MustParseURL("sqlite:/tmp/foo.sqlite3?mode=ro") + require.Equal(t, "/tmp/foo.sqlite3?mode=ro", ConnectionString(u)) + }) + + t.Run("three slashes", func(t *testing.T) { + // interpreted as absolute path + u := dbutil.MustParseURL("sqlite:///tmp/foo.sqlite3?mode=ro") + require.Equal(t, "/tmp/foo.sqlite3?mode=ro", ConnectionString(u)) + }) + + t.Run("four slashes", func(t *testing.T) { + // interpreted as absolute path + // supported for backwards compatibility + u := dbutil.MustParseURL("sqlite:////tmp/foo.sqlite3?mode=ro") + require.Equal(t, "/tmp/foo.sqlite3?mode=ro", ConnectionString(u)) + }) +} + func TestSQLiteCreateDropDatabase(t *testing.T) { - drv := testSQLiteDriver() - u := sqliteTestURL(t) - path := sqlitePath(u) + drv := testSQLiteDriver(t) + path := ConnectionString(drv.databaseURL) // drop any existing database - err := drv.DropDatabase(u) + err := drv.DropDatabase() require.NoError(t, err) // create database - err = drv.CreateDatabase(u) + err = drv.CreateDatabase() require.NoError(t, err) // check that database exists @@ -61,7 +93,7 @@ func TestSQLiteCreateDropDatabase(t *testing.T) { require.NoError(t, err) // drop the database - err = drv.DropDatabase(u) + err = drv.DropDatabase() require.NoError(t, err) // check that database no longer exists @@ -71,15 +103,13 @@ func TestSQLiteCreateDropDatabase(t *testing.T) { } func TestSQLiteDumpSchema(t *testing.T) { - drv := testSQLiteDriver() - drv.SetMigrationsTableName("test_migrations") - - u := sqliteTestURL(t) + drv := testSQLiteDriver(t) + drv.migrationsTableName = "test_migrations" // prepare database - db := prepTestSQLiteDB(t, u) - defer mustClose(db) - err := drv.CreateMigrationsTable(u, db) + db := prepTestSQLiteDB(t) + defer dbutil.MustClose(db) + err := drv.CreateMigrationsTable(db) require.NoError(t, err) // insert migration @@ -89,7 +119,7 @@ func TestSQLiteDumpSchema(t *testing.T) { require.NoError(t, err) // DumpSchema should return schema - schema, err := drv.DumpSchema(u, db) + schema, err := drv.DumpSchema(db) require.NoError(t, err) require.Contains(t, string(schema), "CREATE TABLE IF NOT EXISTS \"test_migrations\"") require.Contains(t, string(schema), ");\n-- Dbmate schema migrations\n"+ @@ -98,50 +128,50 @@ func TestSQLiteDumpSchema(t *testing.T) { " ('abc2');\n") // DumpSchema should return error if command fails - u.Path = "/." - schema, err = drv.DumpSchema(u, db) + drv.databaseURL = dbutil.MustParseURL(".") + schema, err = drv.DumpSchema(db) require.Nil(t, schema) + require.Error(t, err) require.EqualError(t, err, "Error: unable to open database \".\": "+ "unable to open database file") } func TestSQLiteDatabaseExists(t *testing.T) { - drv := testSQLiteDriver() - u := sqliteTestURL(t) + drv := testSQLiteDriver(t) // drop any existing database - err := drv.DropDatabase(u) + err := drv.DropDatabase() require.NoError(t, err) // DatabaseExists should return false - exists, err := drv.DatabaseExists(u) + exists, err := drv.DatabaseExists() require.NoError(t, err) require.Equal(t, false, exists) // create database - err = drv.CreateDatabase(u) + err = drv.CreateDatabase() require.NoError(t, err) // DatabaseExists should return true - exists, err = drv.DatabaseExists(u) + exists, err = drv.DatabaseExists() require.NoError(t, err) require.Equal(t, true, exists) } func TestSQLiteCreateMigrationsTable(t *testing.T) { t.Run("default table", func(t *testing.T) { - drv := testSQLiteDriver() - u := sqliteTestURL(t) - db := prepTestSQLiteDB(t, u) - defer mustClose(db) + drv := testSQLiteDriver(t) + db := prepTestSQLiteDB(t) + defer dbutil.MustClose(db) // migrations table should not exist count := 0 err := db.QueryRow("select count(*) from schema_migrations").Scan(&count) + require.Error(t, err) require.Regexp(t, "no such table: schema_migrations", err.Error()) // create table - err = drv.CreateMigrationsTable(u, db) + err = drv.CreateMigrationsTable(db) require.NoError(t, err) // migrations table should exist @@ -149,25 +179,25 @@ func TestSQLiteCreateMigrationsTable(t *testing.T) { require.NoError(t, err) // create table should be idempotent - err = drv.CreateMigrationsTable(u, db) + err = drv.CreateMigrationsTable(db) require.NoError(t, err) }) t.Run("custom table", func(t *testing.T) { - drv := testSQLiteDriver() - drv.SetMigrationsTableName("test_migrations") + drv := testSQLiteDriver(t) + drv.migrationsTableName = "test_migrations" - u := sqliteTestURL(t) - db := prepTestSQLiteDB(t, u) - defer mustClose(db) + db := prepTestSQLiteDB(t) + defer dbutil.MustClose(db) // migrations table should not exist count := 0 err := db.QueryRow("select count(*) from test_migrations").Scan(&count) + require.Error(t, err) require.Regexp(t, "no such table: test_migrations", err.Error()) // create table - err = drv.CreateMigrationsTable(u, db) + err = drv.CreateMigrationsTable(db) require.NoError(t, err) // migrations table should exist @@ -175,20 +205,19 @@ func TestSQLiteCreateMigrationsTable(t *testing.T) { require.NoError(t, err) // create table should be idempotent - err = drv.CreateMigrationsTable(u, db) + err = drv.CreateMigrationsTable(db) require.NoError(t, err) }) } func TestSQLiteSelectMigrations(t *testing.T) { - drv := testSQLiteDriver() - drv.SetMigrationsTableName("test_migrations") + drv := testSQLiteDriver(t) + drv.migrationsTableName = "test_migrations" - u := sqliteTestURL(t) - db := prepTestSQLiteDB(t, u) - defer mustClose(db) + db := prepTestSQLiteDB(t) + defer dbutil.MustClose(db) - err := drv.CreateMigrationsTable(u, db) + err := drv.CreateMigrationsTable(db) require.NoError(t, err) _, err = db.Exec(`insert into test_migrations (version) @@ -210,14 +239,13 @@ func TestSQLiteSelectMigrations(t *testing.T) { } func TestSQLiteInsertMigration(t *testing.T) { - drv := testSQLiteDriver() - drv.SetMigrationsTableName("test_migrations") + drv := testSQLiteDriver(t) + drv.migrationsTableName = "test_migrations" - u := sqliteTestURL(t) - db := prepTestSQLiteDB(t, u) - defer mustClose(db) + db := prepTestSQLiteDB(t) + defer dbutil.MustClose(db) - err := drv.CreateMigrationsTable(u, db) + err := drv.CreateMigrationsTable(db) require.NoError(t, err) count := 0 @@ -236,14 +264,13 @@ func TestSQLiteInsertMigration(t *testing.T) { } func TestSQLiteDeleteMigration(t *testing.T) { - drv := testSQLiteDriver() - drv.SetMigrationsTableName("test_migrations") + drv := testSQLiteDriver(t) + drv.migrationsTableName = "test_migrations" - u := sqliteTestURL(t) - db := prepTestSQLiteDB(t, u) - defer mustClose(db) + db := prepTestSQLiteDB(t) + defer dbutil.MustClose(db) - err := drv.CreateMigrationsTable(u, db) + err := drv.CreateMigrationsTable(db) require.NoError(t, err) _, err = db.Exec(`insert into test_migrations (version) @@ -260,16 +287,15 @@ func TestSQLiteDeleteMigration(t *testing.T) { } func TestSQLitePing(t *testing.T) { - drv := testSQLiteDriver() - u := sqliteTestURL(t) - path := sqlitePath(u) + drv := testSQLiteDriver(t) + path := ConnectionString(drv.databaseURL) // drop any existing database - err := drv.DropDatabase(u) + err := drv.DropDatabase() require.NoError(t, err) // ping database - err = drv.Ping(u) + err = drv.Ping() require.NoError(t, err) // check that the database was created (sqlite-only behavior) @@ -277,7 +303,7 @@ func TestSQLitePing(t *testing.T) { require.NoError(t, err) // drop the database - err = drv.DropDatabase(u) + err = drv.DropDatabase() require.NoError(t, err) // create directory where database file is expected @@ -289,20 +315,20 @@ func TestSQLitePing(t *testing.T) { }() // ping database should fail - err = drv.Ping(u) + err = drv.Ping() require.EqualError(t, err, "unable to open database file: is a directory") } func TestSQLiteQuotedMigrationsTableName(t *testing.T) { t.Run("default name", func(t *testing.T) { - drv := testSQLiteDriver() + drv := testSQLiteDriver(t) name := drv.quotedMigrationsTableName() require.Equal(t, `"schema_migrations"`, name) }) t.Run("custom name", func(t *testing.T) { - drv := testSQLiteDriver() - drv.SetMigrationsTableName("fooMigrations") + drv := testSQLiteDriver(t) + drv.migrationsTableName = "fooMigrations" name := drv.quotedMigrationsTableName() require.Equal(t, `"fooMigrations"`, name) From 454f93a000efe61719a52215e082ef3bd0c40785 Mon Sep 17 00:00:00 2001 From: Adrian Macneil Date: Sat, 12 Dec 2020 16:54:02 -0800 Subject: [PATCH 30/55] Document --migrations--table flag (#186) Also reordered some of the readme sections and added a table of contents. Closes https://github.com/amacneil/dbmate/issues/185 --- README.md | 172 ++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 108 insertions(+), 64 deletions(-) diff --git a/README.md b/README.md index cca110d..1064461 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,29 @@ It is a standalone command line tool, which can be used with Go, Node.js, Python For a comparison between dbmate and other popular database schema migration tools, please see the [Alternatives](#alternatives) table. +## Table of Contents + +* [Features](#features) +* [Installation](#installation) +* [Commands](#commands) + * [Command Line Options](#command-line-options) +* [Usage](#usage) + * [Connecting to the Database](#connecting-to-the-database) + * [PostgreSQL](#postgresql) + * [MySQL](#mysql) + * [SQLite](#sqlite) + * [ClickHouse](#clickhouse) + * [Creating Migrations](#creating-migrations) + * [Running Migrations](#running-migrations) + * [Rolling Back Migrations](#rolling-back-migrations) + * [Migration Options](#migration-options) + * [Waiting For The Database](#waiting-for-the-database) + * [Exporting Schema File](#exporting-schema-file) +* [Internals](#internals) + * [schema_migrations table](#schema_migrations-table) +* [Alternatives](#alternatives) +* [Contributing](#contributing) + ## Features * Supports MySQL, PostgreSQL, SQLite, and ClickHouse. @@ -77,7 +100,7 @@ $ heroku run bin/dbmate up ## Commands ```sh -dbmate # print help +dbmate --help # print usage help dbmate new # generate a new migration file dbmate up # create the database (if it does not already exist) and run any pending migrations dbmate create # create the database @@ -90,8 +113,23 @@ dbmate dump # write the database schema.sql file dbmate wait # wait for the database server to become available ``` +### Command Line Options + +The following options are available with all commands. You must use command line arguments in the order `dbmate [global options] command [command options]`. Most options can also be configured via environment variables (and loaded from your `.env` file, which is helpful to share configuration between team members). + +* `--url, -u "protocol://host:port/dbname"` - specify the database url directly. _(env: `$DATABASE_URL`)_ +* `--env, -e "DATABASE_URL"` - specify an environment variable to read the database connection URL from. +* `--migrations-dir, -d "./db/migrations"` - where to keep the migration files. _(env: `$DBMATE_MIGRATIONS_DIR`)_ +* `--migrations-table "schema_migrations"` - database table to record migrations in. _(env: `$DBMATE_MIGRATIONS_TABLE`)_ +* `--schema-file, -s "./db/schema.sql"` - a path to keep the schema.sql file. _(env: `$DBMATE_SCHEMA_FILE`)_ +* `--no-dump-schema` - don't auto-update the schema.sql file on migrate/rollback _(env: `$DBMATE_NO_DUMP_SCHEMA`)_ +* `--wait` - wait for the db to become available before executing the subsequent command _(env: `$DBMATE_WAIT`)_ +* `--wait-timeout 60s` - timeout for --wait flag _(env: `$DBMATE_WAIT_TIMEOUT`)_ + ## Usage +### Connecting to the Database + Dbmate locates your database using the `DATABASE_URL` environment variable by default. If you are writing a [twelve-factor app](http://12factor.net/), you should be storing all connection strings in environment variables. To make this easy in development, dbmate looks for a `.env` file in the current directory, and treats any variables listed there as if they were specified in the current environment (existing environment variables take preference, however). @@ -113,19 +151,33 @@ protocol://username:password@host:port/database_name?options * `host` can be either a hostname or IP address * `options` are driver-specific (refer to the underlying Go SQL drivers if you wish to use these) -**MySQL** +Dbmate can also load the connection URL from a different environment variable. For example, before running your test suite, you may wish to drop and recreate the test database. One easy way to do this is to store your test database connection URL in the `TEST_DATABASE_URL` environment variable: ```sh -DATABASE_URL="mysql://username:password@127.0.0.1:3306/database_name" +$ cat .env +DATABASE_URL="postgres://postgres@127.0.0.1:5432/myapp_dev?sslmode=disable" +TEST_DATABASE_URL="postgres://postgres@127.0.0.1:5432/myapp_test?sslmode=disable" ``` -A `socket` parameter can be specified to connect through a unix socket: +You can then specify this environment variable in your test script (Makefile or similar): ```sh -DATABASE_URL="mysql://username:password@/database_name?socket=/var/run/mysqld/mysqld.sock" +$ dbmate -e TEST_DATABASE_URL drop +Dropping: myapp_test +$ dbmate -e TEST_DATABASE_URL --no-dump-schema up +Creating: myapp_test +Applying: 20151127184807_create_users_table.sql ``` -**PostgreSQL** +Alternatively, you can specify the url directly on the command line: + +```sh +$ dbmate -u "postgres://postgres@127.0.0.1:5432/myapp_test?sslmode=disable" up +``` + +The only advantage of using `dbmate -e TEST_DATABASE_URL` over `dbmate -u $TEST_DATABASE_URL` is that the former takes advantage of dbmate's automatic `.env` file loading. + +#### PostgreSQL When connecting to Postgres, you may need to add the `sslmode=disable` option to your connection string, as dbmate by default requires a TLS connection (some other frameworks/languages allow unencrypted connections by default). @@ -150,7 +202,19 @@ DATABASE_URL="postgres://username:password@127.0.0.1:5432/database_name?search_p DATABASE_URL="postgres://username:password@127.0.0.1:5432/database_name?search_path=myschema,public" ``` -**SQLite** +#### MySQL + +```sh +DATABASE_URL="mysql://username:password@127.0.0.1:3306/database_name" +``` + +A `socket` parameter can be specified to connect through a unix socket: + +```sh +DATABASE_URL="mysql://username:password@/database_name?socket=/var/run/mysqld/mysqld.sock" +``` + +#### SQLite SQLite databases are stored on the filesystem, so you do not need to specify a host. By default, files are relative to the current directory. For example, the following will create a database at `./db/database.sqlite3`: @@ -164,7 +228,7 @@ To specify an absolute path, add a forward slash to the path. The following will DATABASE_URL="sqlite:/tmp/database.sqlite3" ``` -**ClickHouse** +#### ClickHouse ```sh DATABASE_URL="clickhouse://username:password@127.0.0.1:9000/database_name" @@ -248,7 +312,7 @@ dbmate supports options passed to a migration block in the form of `key:value` p * `transaction` -#### transaction +**transaction** `transaction` is useful if you need to run some SQL which cannot be executed from within a transaction. For example, in Postgres, you would need to disable transactions for migrations that alter an enum type to add a value: @@ -259,23 +323,6 @@ ALTER TYPE colors ADD VALUE 'orange' AFTER 'red'; `transaction` will default to `true` if your database supports it. -### Schema File - -When you run the `up`, `migrate`, or `rollback` commands, dbmate will automatically create a `./db/schema.sql` file containing a complete representation of your database schema. Dbmate keeps this file up to date for you, so you should not manually edit it. - -It is recommended to check this file into source control, so that you can easily review changes to the schema in commits or pull requests. It's also possible to use this file when you want to quickly load a database schema, without running each migration sequentially (for example in your test harness). However, if you do not wish to save this file, you could add it to `.gitignore`, or pass the `--no-dump-schema` command line option. - -To dump the `schema.sql` file without performing any other actions, run `dbmate dump`. Unlike other dbmate actions, this command relies on the respective `pg_dump`, `mysqldump`, or `sqlite3` commands being available in your PATH. If these tools are not available, dbmate will silenty skip the schema dump step during `up`, `migrate`, or `rollback` actions. You can diagnose the issue by running `dbmate dump` and looking at the output: - -```sh -$ dbmate dump -exec: "pg_dump": executable file not found in $PATH -``` - -On Ubuntu or Debian systems, you can fix this by installing `postgresql-client`, `mysql-client`, or `sqlite3` respectively. Ensure that the package version you install is greater than or equal to the version running on your database server. - -> Note: The `schema.sql` file will contain a complete schema for your database, even if some tables or columns were created outside of dbmate migrations. - ### Waiting For The Database If you use a Docker development environment for your project, you may encounter issues with the database not being immediately ready when running migrations or unit tests. This can be due to the database server having only just started. @@ -313,64 +360,61 @@ Error: unable to connect to database: dial tcp 127.0.0.1:5432: connect: connecti Please note that the `wait` command does not verify whether your specified database exists, only that the server is available and ready (so it will return success if the database server is available, but your database has not yet been created). -### Options +### Exporting Schema File -The following command line options are available with all commands. You must use command line arguments in the order `dbmate [global options] command [command options]`. Most options can also be configured via environment variables (and loaded from your `.env` file, which is helpful to share configuration between team members). +When you run the `up`, `migrate`, or `rollback` commands, dbmate will automatically create a `./db/schema.sql` file containing a complete representation of your database schema. Dbmate keeps this file up to date for you, so you should not manually edit it. -* `--url, -u "protocol://host:port/dbname"` - specify the database url directly. _(env: `$DATABASE_URL`)_ -* `--env, -e "DATABASE_URL"` - specify an environment variable to read the database connection URL from. -* `--migrations-dir, -d "./db/migrations"` - where to keep the migration files. _(env: `$DBMATE_MIGRATIONS_DIR`)_ -* `--schema-file, -s "./db/schema.sql"` - a path to keep the schema.sql file. _(env: `$DBMATE_SCHEMA_FILE`)_ -* `--no-dump-schema` - don't auto-update the schema.sql file on migrate/rollback _(env: `$DBMATE_NO_DUMP_SCHEMA`)_ -* `--wait` - wait for the db to become available before executing the subsequent command _(env: `$DBMATE_WAIT`)_ -* `--wait-timeout 60s` - timeout for --wait flag _(env: `$DBMATE_WAIT_TIMEOUT`)_ +It is recommended to check this file into source control, so that you can easily review changes to the schema in commits or pull requests. It's also possible to use this file when you want to quickly load a database schema, without running each migration sequentially (for example in your test harness). However, if you do not wish to save this file, you could add it to your `.gitignore`, or pass the `--no-dump-schema` command line option. -For example, before running your test suite, you may wish to drop and recreate the test database. One easy way to do this is to store your test database connection URL in the `TEST_DATABASE_URL` environment variable: +To dump the `schema.sql` file without performing any other actions, run `dbmate dump`. Unlike other dbmate actions, this command relies on the respective `pg_dump`, `mysqldump`, or `sqlite3` commands being available in your PATH. If these tools are not available, dbmate will silenty skip the schema dump step during `up`, `migrate`, or `rollback` actions. You can diagnose the issue by running `dbmate dump` and looking at the output: ```sh -$ cat .env -TEST_DATABASE_URL="postgres://postgres@127.0.0.1:5432/myapp_test?sslmode=disable" +$ dbmate dump +exec: "pg_dump": executable file not found in $PATH ``` -You can then specify this environment variable in your test script (Makefile or similar): +On Ubuntu or Debian systems, you can fix this by installing `postgresql-client`, `mysql-client`, or `sqlite3` respectively. Ensure that the package version you install is greater than or equal to the version running on your database server. -```sh -$ dbmate -e TEST_DATABASE_URL drop -Dropping: myapp_test -$ dbmate -e TEST_DATABASE_URL --no-dump-schema up -Creating: myapp_test -Applying: 20151127184807_create_users_table.sql +> Note: The `schema.sql` file will contain a complete schema for your database, even if some tables or columns were created outside of dbmate migrations. + +## Internals + +### schema_migrations table + +By default, dbmate stores a record of each applied migration in a `schema_migrations` table. This table will be created for you automatically if it does not already exist. The table schema is very simple: + +```sql +CREATE TABLE IF NOT EXISTS schema_migrations ( + version VARCHAR(255) PRIMARY KEY +) ``` -Alternatively, you can specify the url directly on the command line: +Dbmate records only the version number of applied migrations, so you can safely rename a migration file without affecting its applied status. -```sh -$ dbmate -u "postgres://postgres@127.0.0.1:5432/myapp_test?sslmode=disable" up -``` - -The only advantage of using `dbmate -e TEST_DATABASE_URL` over `dbmate -u $TEST_DATABASE_URL` is that the former takes advantage of dbmate's automatic `.env` file loading. +You can customize the name of this table using the `--migrations-table` flag or `$DBMATE_MIGRATIONS_TABLE` environment variable. If you already have a table with this name (possibly from a previous migration tool), you should either manually update it to conform to this schema, or configure dbmate to use a different table name. ## Alternatives Why another database schema migration tool? Dbmate was inspired by many other tools, primarily [Active Record Migrations](http://guides.rubyonrails.org/active_record_migrations.html), with the goals of being trivial to configure, and language & framework independent. Here is a comparison between dbmate and other popular migration tools. -| | [goose](https://bitbucket.org/liamstask/goose/) | [sql-migrate](https://github.com/rubenv/sql-migrate) | [golang-migrate/migrate](https://github.com/golang-migrate/migrate) | [activerecord](http://guides.rubyonrails.org/active_record_migrations.html) | [sequelize](http://docs.sequelizejs.com/manual/tutorial/migrations.html) | [dbmate](https://github.com/amacneil/dbmate) | +| | [dbmate](https://github.com/amacneil/dbmate) | [goose](https://github.com/pressly/goose) | [sql-migrate](https://github.com/rubenv/sql-migrate) | [golang-migrate](https://github.com/golang-migrate/migrate) | [activerecord](http://guides.rubyonrails.org/active_record_migrations.html) | [sequelize](http://docs.sequelizejs.com/manual/tutorial/migrations.html) | | --- |:---:|:---:|:---:|:---:|:---:|:---:| | **Features** | -|Plain SQL migration files|:white_check_mark:|:white_check_mark:|:white_check_mark:|||:white_check_mark:| -|Support for creating and dropping databases||||:white_check_mark:||:white_check_mark:| -|Support for saving schema dump files||||:white_check_mark:||:white_check_mark:| -|Timestamp-versioned migration files|:white_check_mark:|||:white_check_mark:|:white_check_mark:|:white_check_mark:| -|Ability to wait for database to become ready||||||:white_check_mark:| -|Database connection string loaded from environment variables||||||:white_check_mark:| -|Automatically load .env file||||||:white_check_mark:| -|No separate configuration file||||:white_check_mark:|:white_check_mark:|:white_check_mark:| -|Language/framework independent|:eight_pointed_black_star:|:eight_pointed_black_star:|:eight_pointed_black_star:|||:white_check_mark:| +|Plain SQL migration files|:white_check_mark:|:white_check_mark:|:white_check_mark:|:white_check_mark:||| +|Support for creating and dropping databases|:white_check_mark:||||:white_check_mark:|| +|Support for saving schema dump files|:white_check_mark:||||:white_check_mark:|| +|Timestamp-versioned migration files|:white_check_mark:|:white_check_mark:|||:white_check_mark:|:white_check_mark:| +|Custom schema migrations table|:white_check_mark:||:white_check_mark:|||:white_check_mark:| +|Ability to wait for database to become ready|:white_check_mark:|||||| +|Database connection string loaded from environment variables|:white_check_mark:|||||| +|Automatically load .env file|:white_check_mark:|||||| +|No separate configuration file|:white_check_mark:||||:white_check_mark:|:white_check_mark:| +|Language/framework independent|:white_check_mark:|:eight_pointed_black_star:|:eight_pointed_black_star:|:eight_pointed_black_star:||| | **Drivers** | |PostgreSQL|:white_check_mark:|:white_check_mark:|:white_check_mark:|:white_check_mark:|:white_check_mark:|:white_check_mark:| |MySQL|:white_check_mark:|:white_check_mark:|:white_check_mark:|:white_check_mark:|:white_check_mark:|:white_check_mark:| |SQLite|:white_check_mark:|:white_check_mark:|:white_check_mark:|:white_check_mark:|:white_check_mark:|:white_check_mark:| -|CliсkHouse|||:white_check_mark:|:white_check_mark:|:white_check_mark:|:white_check_mark:| +|CliсkHouse|:white_check_mark:|||:white_check_mark:|:white_check_mark:|:white_check_mark:| > :eight_pointed_black_star: In theory these tools could be used with other languages, but a Go development environment is required because binary builds are not provided. @@ -383,11 +427,11 @@ Dbmate is written in Go, pull requests are welcome. Tests are run against a real database using docker-compose. To build a docker image and run the tests: ```sh -$ make docker-all +$ make docker-make ``` To start a development shell: ```sh -$ make docker-bash +$ make docker-sh ``` From 2bac2c759027e899b794a0b24eaef06480c1d6ed Mon Sep 17 00:00:00 2001 From: Bouke van der Bijl Date: Thu, 18 Feb 2021 23:10:57 +0100 Subject: [PATCH 31/55] Write log lines to DB.Log output (#195) This makes it possible to redirect the logs somewhere else, useful if you embed dbmate into your application. --- pkg/dbmate/db.go | 38 ++++++++++++++++------------- pkg/dbmate/driver.go | 2 ++ pkg/driver/clickhouse/clickhouse.go | 7 ++++-- pkg/driver/mysql/mysql.go | 7 ++++-- pkg/driver/postgres/postgres.go | 9 ++++--- pkg/driver/sqlite/sqlite.go | 7 ++++-- 6 files changed, 44 insertions(+), 26 deletions(-) diff --git a/pkg/dbmate/db.go b/pkg/dbmate/db.go index 3350c02..b955686 100644 --- a/pkg/dbmate/db.go +++ b/pkg/dbmate/db.go @@ -4,6 +4,7 @@ import ( "database/sql" "errors" "fmt" + "io" "io/ioutil" "net/url" "os" @@ -41,6 +42,7 @@ type DB struct { WaitBefore bool WaitInterval time.Duration WaitTimeout time.Duration + Log io.Writer } // migrationFileRegexp pattern for valid migration files @@ -63,6 +65,7 @@ func New(databaseURL *url.URL) *DB { WaitBefore: false, WaitInterval: DefaultWaitInterval, WaitTimeout: DefaultWaitTimeout, + Log: os.Stdout, } } @@ -80,6 +83,7 @@ func (db *DB) GetDriver() (Driver, error) { config := DriverConfig{ DatabaseURL: db.DatabaseURL, MigrationsTableName: db.MigrationsTableName, + Log: db.Log, } return driverFunc(config), nil @@ -104,22 +108,22 @@ func (db *DB) wait(drv Driver) error { return nil } - fmt.Print("Waiting for database") + fmt.Fprint(db.Log, "Waiting for database") for i := 0 * time.Second; i < db.WaitTimeout; i += db.WaitInterval { - fmt.Print(".") + fmt.Fprint(db.Log, ".") time.Sleep(db.WaitInterval) // attempt connection to database server err = drv.Ping() if err == nil { // connection successful - fmt.Print("\n") + fmt.Fprint(db.Log, "\n") return nil } } // if we find outselves here, we could not connect within the timeout - fmt.Print("\n") + fmt.Fprint(db.Log, "\n") return fmt.Errorf("unable to connect to database: %s", err) } @@ -214,7 +218,7 @@ func (db *DB) dumpSchema(drv Driver) error { return err } - fmt.Printf("Writing: %s\n", db.SchemaFile) + fmt.Fprintf(db.Log, "Writing: %s\n", db.SchemaFile) // ensure schema directory exists if err = ensureDir(filepath.Dir(db.SchemaFile)); err != nil { @@ -252,7 +256,7 @@ func (db *DB) NewMigration(name string) error { // check file does not already exist path := filepath.Join(db.MigrationsDir, name) - fmt.Printf("Creating migration: %s\n", path) + fmt.Fprintf(db.Log, "Creating migration: %s\n", path) if _, err := os.Stat(path); !os.IsNotExist(err) { return fmt.Errorf("file already exists") @@ -345,7 +349,7 @@ func (db *DB) migrate(drv Driver) error { continue } - fmt.Printf("Applying: %s\n", filename) + fmt.Fprintf(db.Log, "Applying: %s\n", filename) up, _, err := parseMigration(filepath.Join(db.MigrationsDir, filename)) if err != nil { @@ -358,7 +362,7 @@ func (db *DB) migrate(drv Driver) error { if err != nil { return err } else if db.Verbose { - printVerbose(result) + db.printVerbose(result) } // record migration @@ -386,14 +390,14 @@ func (db *DB) migrate(drv Driver) error { return nil } -func printVerbose(result sql.Result) { +func (db *DB) printVerbose(result sql.Result) { lastInsertID, err := result.LastInsertId() if err == nil { - fmt.Printf("Last insert ID: %d\n", lastInsertID) + fmt.Fprintf(db.Log, "Last insert ID: %d\n", lastInsertID) } rowsAffected, err := result.RowsAffected() if err == nil { - fmt.Printf("Rows affected: %d\n", rowsAffected) + fmt.Fprintf(db.Log, "Rows affected: %d\n", rowsAffected) } } @@ -485,7 +489,7 @@ func (db *DB) Rollback() error { return err } - fmt.Printf("Rolling back: %s\n", filename) + fmt.Fprintf(db.Log, "Rolling back: %s\n", filename) _, down, err := parseMigration(filepath.Join(db.MigrationsDir, filename)) if err != nil { @@ -498,7 +502,7 @@ func (db *DB) Rollback() error { if err != nil { return err } else if db.Verbose { - printVerbose(result) + db.printVerbose(result) } // remove migration record @@ -548,15 +552,15 @@ func (db *DB) Status(quiet bool) (int, error) { line = fmt.Sprintf("[ ] %s", res.Filename) } if !quiet { - fmt.Println(line) + fmt.Fprintln(db.Log, line) } } totalPending := len(results) - totalApplied if !quiet { - fmt.Println() - fmt.Printf("Applied: %d\n", totalApplied) - fmt.Printf("Pending: %d\n", totalPending) + fmt.Fprintln(db.Log) + fmt.Fprintf(db.Log, "Applied: %d\n", totalApplied) + fmt.Fprintf(db.Log, "Pending: %d\n", totalPending) } return totalPending, nil diff --git a/pkg/dbmate/driver.go b/pkg/dbmate/driver.go index 09d2503..e79d60d 100644 --- a/pkg/dbmate/driver.go +++ b/pkg/dbmate/driver.go @@ -2,6 +2,7 @@ package dbmate import ( "database/sql" + "io" "net/url" "github.com/amacneil/dbmate/pkg/dbutil" @@ -25,6 +26,7 @@ type Driver interface { type DriverConfig struct { DatabaseURL *url.URL MigrationsTableName string + Log io.Writer } // DriverFunc represents a driver constructor diff --git a/pkg/driver/clickhouse/clickhouse.go b/pkg/driver/clickhouse/clickhouse.go index 0204a58..83571a8 100644 --- a/pkg/driver/clickhouse/clickhouse.go +++ b/pkg/driver/clickhouse/clickhouse.go @@ -4,6 +4,7 @@ import ( "bytes" "database/sql" "fmt" + "io" "net/url" "regexp" "sort" @@ -23,6 +24,7 @@ func init() { type Driver struct { migrationsTableName string databaseURL *url.URL + log io.Writer } // NewDriver initializes the driver @@ -30,6 +32,7 @@ func NewDriver(config dbmate.DriverConfig) dbmate.Driver { return &Driver{ migrationsTableName: config.MigrationsTableName, databaseURL: config.DatabaseURL, + log: config.Log, } } @@ -108,7 +111,7 @@ func (drv *Driver) quoteIdentifier(str string) string { // CreateDatabase creates the specified database func (drv *Driver) CreateDatabase() error { name := drv.databaseName() - fmt.Printf("Creating: %s\n", name) + fmt.Fprintf(drv.log, "Creating: %s\n", name) db, err := drv.openClickHouseDB() if err != nil { @@ -124,7 +127,7 @@ func (drv *Driver) CreateDatabase() error { // DropDatabase drops the specified database (if it exists) func (drv *Driver) DropDatabase() error { name := drv.databaseName() - fmt.Printf("Dropping: %s\n", name) + fmt.Fprintf(drv.log, "Dropping: %s\n", name) db, err := drv.openClickHouseDB() if err != nil { diff --git a/pkg/driver/mysql/mysql.go b/pkg/driver/mysql/mysql.go index a19b74b..ca67330 100644 --- a/pkg/driver/mysql/mysql.go +++ b/pkg/driver/mysql/mysql.go @@ -4,6 +4,7 @@ import ( "bytes" "database/sql" "fmt" + "io" "net/url" "strings" @@ -21,6 +22,7 @@ func init() { type Driver struct { migrationsTableName string databaseURL *url.URL + log io.Writer } // NewDriver initializes the driver @@ -28,6 +30,7 @@ func NewDriver(config dbmate.DriverConfig) dbmate.Driver { return &Driver{ migrationsTableName: config.MigrationsTableName, databaseURL: config.DatabaseURL, + log: config.Log, } } @@ -92,7 +95,7 @@ func (drv *Driver) quoteIdentifier(str string) string { // CreateDatabase creates the specified database func (drv *Driver) CreateDatabase() error { name := dbutil.DatabaseName(drv.databaseURL) - fmt.Printf("Creating: %s\n", name) + fmt.Fprintf(drv.log, "Creating: %s\n", name) db, err := drv.openRootDB() if err != nil { @@ -109,7 +112,7 @@ func (drv *Driver) CreateDatabase() error { // DropDatabase drops the specified database (if it exists) func (drv *Driver) DropDatabase() error { name := dbutil.DatabaseName(drv.databaseURL) - fmt.Printf("Dropping: %s\n", name) + fmt.Fprintf(drv.log, "Dropping: %s\n", name) db, err := drv.openRootDB() if err != nil { diff --git a/pkg/driver/postgres/postgres.go b/pkg/driver/postgres/postgres.go index 9624ea9..1917cb1 100644 --- a/pkg/driver/postgres/postgres.go +++ b/pkg/driver/postgres/postgres.go @@ -4,6 +4,7 @@ import ( "bytes" "database/sql" "fmt" + "io" "net/url" "strings" @@ -22,6 +23,7 @@ func init() { type Driver struct { migrationsTableName string databaseURL *url.URL + log io.Writer } // NewDriver initializes the driver @@ -29,6 +31,7 @@ func NewDriver(config dbmate.DriverConfig) dbmate.Driver { return &Driver{ migrationsTableName: config.MigrationsTableName, databaseURL: config.DatabaseURL, + log: config.Log, } } @@ -112,7 +115,7 @@ func (drv *Driver) openPostgresDB() (*sql.DB, error) { // CreateDatabase creates the specified database func (drv *Driver) CreateDatabase() error { name := dbutil.DatabaseName(drv.databaseURL) - fmt.Printf("Creating: %s\n", name) + fmt.Fprintf(drv.log, "Creating: %s\n", name) db, err := drv.openPostgresDB() if err != nil { @@ -129,7 +132,7 @@ func (drv *Driver) CreateDatabase() error { // DropDatabase drops the specified database (if it exists) func (drv *Driver) DropDatabase() error { name := dbutil.DatabaseName(drv.databaseURL) - fmt.Printf("Dropping: %s\n", name) + fmt.Fprintf(drv.log, "Dropping: %s\n", name) db, err := drv.openPostgresDB() if err != nil { @@ -233,7 +236,7 @@ func (drv *Driver) CreateMigrationsTable(db *sql.DB) error { // in theory we could attempt to create the schema every time, but we avoid that // in case the user doesn't have permissions to create schemas - fmt.Printf("Creating schema: %s\n", schema) + fmt.Fprintf(drv.log, "Creating schema: %s\n", schema) _, err = db.Exec(fmt.Sprintf("create schema if not exists %s", schema)) if err != nil { return err diff --git a/pkg/driver/sqlite/sqlite.go b/pkg/driver/sqlite/sqlite.go index 99ffe3d..a03954e 100644 --- a/pkg/driver/sqlite/sqlite.go +++ b/pkg/driver/sqlite/sqlite.go @@ -6,6 +6,7 @@ import ( "bytes" "database/sql" "fmt" + "io" "net/url" "os" "regexp" @@ -27,6 +28,7 @@ func init() { type Driver struct { migrationsTableName string databaseURL *url.URL + log io.Writer } // NewDriver initializes the driver @@ -34,6 +36,7 @@ func NewDriver(config dbmate.DriverConfig) dbmate.Driver { return &Driver{ migrationsTableName: config.MigrationsTableName, databaseURL: config.DatabaseURL, + log: config.Log, } } @@ -56,7 +59,7 @@ func (drv *Driver) Open() (*sql.DB, error) { // CreateDatabase creates the specified database func (drv *Driver) CreateDatabase() error { - fmt.Printf("Creating: %s\n", ConnectionString(drv.databaseURL)) + fmt.Fprintf(drv.log, "Creating: %s\n", ConnectionString(drv.databaseURL)) db, err := drv.Open() if err != nil { @@ -70,7 +73,7 @@ func (drv *Driver) CreateDatabase() error { // DropDatabase drops the specified database (if it exists) func (drv *Driver) DropDatabase() error { path := ConnectionString(drv.databaseURL) - fmt.Printf("Dropping: %s\n", path) + fmt.Fprintf(drv.log, "Dropping: %s\n", path) exists, err := drv.DatabaseExists() if err != nil { From 0c758a8d9fdad0359bf352bd18b1d3ec6572dd1e Mon Sep 17 00:00:00 2001 From: Adrian Macneil Date: Mon, 22 Feb 2021 11:26:35 -0800 Subject: [PATCH 32/55] Go v1.16 (#196) --- Dockerfile | 4 ++-- go.mod | 14 ++++++++------ go.sum | 36 ++++++++++++++++++------------------ 3 files changed, 28 insertions(+), 26 deletions(-) diff --git a/Dockerfile b/Dockerfile index ebdbdf9..97986af 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # development image -FROM techknowlogick/xgo:go-1.15.x as dev +FROM techknowlogick/xgo:go-1.16.x as dev WORKDIR /src ENV GOCACHE /src/.cache/go-build @@ -18,7 +18,7 @@ RUN apt-get update \ # golangci-lint RUN curl -fsSL -o /tmp/lint-install.sh https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh \ && chmod +x /tmp/lint-install.sh \ - && /tmp/lint-install.sh -b /usr/local/bin v1.32.2 \ + && /tmp/lint-install.sh -b /usr/local/bin v1.37.1 \ && rm -f /tmp/lint-install.sh # download modules diff --git a/go.mod b/go.mod index 420929a..27c0bbf 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/amacneil/dbmate -go 1.15 +go 1.16 require ( github.com/ClickHouse/clickhouse-go v1.4.3 @@ -9,11 +9,13 @@ require ( github.com/go-sql-driver/mysql v1.5.0 github.com/joho/godotenv v1.3.0 github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d - github.com/kr/pretty v0.1.0 // indirect - github.com/lib/pq v1.8.0 - github.com/mattn/go-sqlite3 v1.14.4 + github.com/kr/text v0.2.0 // indirect + github.com/lib/pq v1.9.0 + github.com/mattn/go-sqlite3 v1.14.6 + github.com/pierrec/lz4 v2.6.0+incompatible // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/stretchr/testify v1.4.0 + github.com/stretchr/testify v1.7.0 github.com/urfave/cli/v2 v2.3.0 - gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect + gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect ) diff --git a/go.sum b/go.sum index 3fd6f97..d948ac9 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,3 @@ -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/ClickHouse/clickhouse-go v1.4.3 h1:iAFMa2UrQdR5bHJ2/yaSLffZkxpcOYQMCUuKeNXGdqc= github.com/ClickHouse/clickhouse-go v1.4.3/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI= @@ -9,6 +8,7 @@ github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -20,37 +20,37 @@ github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d h1:cVtBfNW5XTHiKQe7jDaDBSh/EVM4XLPutLAGboIXuM0= github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.8.0 h1:9xohqzkUwzR4Ga4ivdTcawVS89YSDVxXMa3xJX3cGzg= -github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.9.0 h1:L8nSXQQzAYByakOFMTwpjRoHsMJklur4Gi59b6VivR8= +github.com/lib/pq v1.9.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= -github.com/mattn/go-sqlite3 v1.14.4 h1:4rQjbDxdu9fSgI/r3KN72G3c2goxknAqHHgPWWs8UlI= -github.com/mattn/go-sqlite3 v1.14.4/go.mod h1:WVKg1VTActs4Qso6iwGbiFih2UIHo0ENGwNd0Lj+XmI= -github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I= +github.com/mattn/go-sqlite3 v1.14.6 h1:dNPt6NO46WmLVt2DLNpwczCmdV5boIZ6g/tlDrlRUbg= +github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pierrec/lz4 v2.6.0+incompatible h1:Ix9yFKn1nSPBLFl/yZknTp8TU5G4Ps0JDmguYK6iH1A= +github.com/pierrec/lz4 v2.6.0+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M= github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.3 h1:fvjTMHxHEw/mxHbtzPi3JCcKXQRAnQTBRo6YCJSVHKI= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From 53c9c19e82818d2f3f2fb66e171c5593fb532995 Mon Sep 17 00:00:00 2001 From: Adrian Macneil Date: Thu, 4 Mar 2021 14:04:14 -0800 Subject: [PATCH 33/55] master -> main (#201) --- .github/workflows/build.yml | 6 +- .github/workflows/codeql-analysis.yml | 50 +++-------- README.md | 122 +++++++++++++------------- 3 files changed, 78 insertions(+), 100 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 34eb6da..0ffea4f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -2,10 +2,10 @@ name: CI on: push: - branches: [ master ] - tags: 'v*' + branches: [main] + tags: "v*" pull_request: - branches: [ master ] + branches: [main] jobs: build: diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 6c1953e..ed3cae6 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -8,12 +8,11 @@ name: "CodeQL" on: push: - branches: [ master ] + branches: [main] pull_request: - # The branches below must be a subset of the branches above - branches: [ master ] + branches: [main] schedule: - - cron: '0 0 * * 4' + - cron: "0 0 * * 4" jobs: analyze: @@ -23,40 +22,19 @@ jobs: strategy: fail-fast: false matrix: - language: [ 'go' ] - # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] - # Learn more... - # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection + language: ["go"] steps: - - name: Checkout repository - uses: actions/checkout@v2 + - name: Checkout repository + uses: actions/checkout@v2 - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@v1 - with: - languages: ${{ matrix.language }} - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. - # queries: ./path/to/local/query, your-org/your-repo/queries@main + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: ${{ matrix.language }} - # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). - # If this step fails, then you should remove it and run the build manually (see below) - - name: Autobuild - uses: github/codeql-action/autobuild@v1 + - name: Autobuild + uses: github/codeql-action/autobuild@v1 - # ℹ️ Command-line programs to run using the OS shell. - # 📚 https://git.io/JvXDl - - # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines - # and modify them (or add more) to build your code if your project - # uses a compiled language - - #- run: | - # make bootstrap - # make release - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 diff --git a/README.md b/README.md index 1064461..a0cae85 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Dbmate -[![GitHub Build](https://img.shields.io/github/workflow/status/amacneil/dbmate/CI/master)](https://github.com/amacneil/dbmate/actions?query=branch%3Amaster+event%3Apush+workflow%3ACI) +[![GitHub Build](https://img.shields.io/github/workflow/status/amacneil/dbmate/CI/main)](https://github.com/amacneil/dbmate/actions?query=branch%3Amain+event%3Apush+workflow%3ACI) [![Go Report Card](https://goreportcard.com/badge/github.com/amacneil/dbmate)](https://goreportcard.com/report/github.com/amacneil/dbmate) [![GitHub Release](https://img.shields.io/github/release/amacneil/dbmate.svg)](https://github.com/amacneil/dbmate/releases) @@ -12,38 +12,38 @@ For a comparison between dbmate and other popular database schema migration tool ## Table of Contents -* [Features](#features) -* [Installation](#installation) -* [Commands](#commands) - * [Command Line Options](#command-line-options) -* [Usage](#usage) - * [Connecting to the Database](#connecting-to-the-database) - * [PostgreSQL](#postgresql) - * [MySQL](#mysql) - * [SQLite](#sqlite) - * [ClickHouse](#clickhouse) - * [Creating Migrations](#creating-migrations) - * [Running Migrations](#running-migrations) - * [Rolling Back Migrations](#rolling-back-migrations) - * [Migration Options](#migration-options) - * [Waiting For The Database](#waiting-for-the-database) - * [Exporting Schema File](#exporting-schema-file) -* [Internals](#internals) - * [schema_migrations table](#schema_migrations-table) -* [Alternatives](#alternatives) -* [Contributing](#contributing) +- [Features](#features) +- [Installation](#installation) +- [Commands](#commands) + - [Command Line Options](#command-line-options) +- [Usage](#usage) + - [Connecting to the Database](#connecting-to-the-database) + - [PostgreSQL](#postgresql) + - [MySQL](#mysql) + - [SQLite](#sqlite) + - [ClickHouse](#clickhouse) + - [Creating Migrations](#creating-migrations) + - [Running Migrations](#running-migrations) + - [Rolling Back Migrations](#rolling-back-migrations) + - [Migration Options](#migration-options) + - [Waiting For The Database](#waiting-for-the-database) + - [Exporting Schema File](#exporting-schema-file) +- [Internals](#internals) + - [schema_migrations table](#schema_migrations-table) +- [Alternatives](#alternatives) +- [Contributing](#contributing) ## Features -* Supports MySQL, PostgreSQL, SQLite, and ClickHouse. -* Uses plain SQL for writing schema migrations. -* Migrations are timestamp-versioned, to avoid version number conflicts with multiple developers. -* Migrations are run atomically inside a transaction. -* Supports creating and dropping databases (handy in development/test). -* Supports saving a `schema.sql` file to easily diff schema changes in git. -* Database connection URL is definied using an environment variable (`DATABASE_URL` by default), or specified on the command line. -* Built-in support for reading environment variables from your `.env` file. -* Easy to distribute, single self-contained binary. +- Supports MySQL, PostgreSQL, SQLite, and ClickHouse. +- Uses plain SQL for writing schema migrations. +- Migrations are timestamp-versioned, to avoid version number conflicts with multiple developers. +- Migrations are run atomically inside a transaction. +- Supports creating and dropping databases (handy in development/test). +- Supports saving a `schema.sql` file to easily diff schema changes in git. +- Database connection URL is definied using an environment variable (`DATABASE_URL` by default), or specified on the command line. +- Built-in support for reading environment variables from your `.env` file. +- Easy to distribute, single self-contained binary. ## Installation @@ -117,14 +117,14 @@ dbmate wait # wait for the database server to become available The following options are available with all commands. You must use command line arguments in the order `dbmate [global options] command [command options]`. Most options can also be configured via environment variables (and loaded from your `.env` file, which is helpful to share configuration between team members). -* `--url, -u "protocol://host:port/dbname"` - specify the database url directly. _(env: `$DATABASE_URL`)_ -* `--env, -e "DATABASE_URL"` - specify an environment variable to read the database connection URL from. -* `--migrations-dir, -d "./db/migrations"` - where to keep the migration files. _(env: `$DBMATE_MIGRATIONS_DIR`)_ -* `--migrations-table "schema_migrations"` - database table to record migrations in. _(env: `$DBMATE_MIGRATIONS_TABLE`)_ -* `--schema-file, -s "./db/schema.sql"` - a path to keep the schema.sql file. _(env: `$DBMATE_SCHEMA_FILE`)_ -* `--no-dump-schema` - don't auto-update the schema.sql file on migrate/rollback _(env: `$DBMATE_NO_DUMP_SCHEMA`)_ -* `--wait` - wait for the db to become available before executing the subsequent command _(env: `$DBMATE_WAIT`)_ -* `--wait-timeout 60s` - timeout for --wait flag _(env: `$DBMATE_WAIT_TIMEOUT`)_ +- `--url, -u "protocol://host:port/dbname"` - specify the database url directly. _(env: `$DATABASE_URL`)_ +- `--env, -e "DATABASE_URL"` - specify an environment variable to read the database connection URL from. +- `--migrations-dir, -d "./db/migrations"` - where to keep the migration files. _(env: `$DBMATE_MIGRATIONS_DIR`)_ +- `--migrations-table "schema_migrations"` - database table to record migrations in. _(env: `$DBMATE_MIGRATIONS_TABLE`)_ +- `--schema-file, -s "./db/schema.sql"` - a path to keep the schema.sql file. _(env: `$DBMATE_SCHEMA_FILE`)_ +- `--no-dump-schema` - don't auto-update the schema.sql file on migrate/rollback _(env: `$DBMATE_NO_DUMP_SCHEMA`)_ +- `--wait` - wait for the db to become available before executing the subsequent command _(env: `$DBMATE_WAIT`)_ +- `--wait-timeout 60s` - timeout for --wait flag _(env: `$DBMATE_WAIT_TIMEOUT`)_ ## Usage @@ -147,9 +147,9 @@ DATABASE_URL="postgres://postgres@127.0.0.1:5432/myapp_development?sslmode=disab protocol://username:password@host:port/database_name?options ``` -* `protocol` must be one of `mysql`, `postgres`, `postgresql`, `sqlite`, `sqlite3`, `clickhouse` -* `host` can be either a hostname or IP address -* `options` are driver-specific (refer to the underlying Go SQL drivers if you wish to use these) +- `protocol` must be one of `mysql`, `postgres`, `postgresql`, `sqlite`, `sqlite3`, `clickhouse` +- `host` can be either a hostname or IP address +- `options` are driver-specific (refer to the underlying Go SQL drivers if you wish to use these) Dbmate can also load the connection URL from a different environment variable. For example, before running your test suite, you may wish to drop and recreate the test database. One easy way to do this is to store your test database connection URL in the `TEST_DATABASE_URL` environment variable: @@ -310,7 +310,7 @@ Writing: ./db/schema.sql dbmate supports options passed to a migration block in the form of `key:value` pairs. List of supported options: -* `transaction` +- `transaction` **transaction** @@ -397,28 +397,28 @@ You can customize the name of this table using the `--migrations-table` flag or Why another database schema migration tool? Dbmate was inspired by many other tools, primarily [Active Record Migrations](http://guides.rubyonrails.org/active_record_migrations.html), with the goals of being trivial to configure, and language & framework independent. Here is a comparison between dbmate and other popular migration tools. -| | [dbmate](https://github.com/amacneil/dbmate) | [goose](https://github.com/pressly/goose) | [sql-migrate](https://github.com/rubenv/sql-migrate) | [golang-migrate](https://github.com/golang-migrate/migrate) | [activerecord](http://guides.rubyonrails.org/active_record_migrations.html) | [sequelize](http://docs.sequelizejs.com/manual/tutorial/migrations.html) | -| --- |:---:|:---:|:---:|:---:|:---:|:---:| -| **Features** | -|Plain SQL migration files|:white_check_mark:|:white_check_mark:|:white_check_mark:|:white_check_mark:||| -|Support for creating and dropping databases|:white_check_mark:||||:white_check_mark:|| -|Support for saving schema dump files|:white_check_mark:||||:white_check_mark:|| -|Timestamp-versioned migration files|:white_check_mark:|:white_check_mark:|||:white_check_mark:|:white_check_mark:| -|Custom schema migrations table|:white_check_mark:||:white_check_mark:|||:white_check_mark:| -|Ability to wait for database to become ready|:white_check_mark:|||||| -|Database connection string loaded from environment variables|:white_check_mark:|||||| -|Automatically load .env file|:white_check_mark:|||||| -|No separate configuration file|:white_check_mark:||||:white_check_mark:|:white_check_mark:| -|Language/framework independent|:white_check_mark:|:eight_pointed_black_star:|:eight_pointed_black_star:|:eight_pointed_black_star:||| -| **Drivers** | -|PostgreSQL|:white_check_mark:|:white_check_mark:|:white_check_mark:|:white_check_mark:|:white_check_mark:|:white_check_mark:| -|MySQL|:white_check_mark:|:white_check_mark:|:white_check_mark:|:white_check_mark:|:white_check_mark:|:white_check_mark:| -|SQLite|:white_check_mark:|:white_check_mark:|:white_check_mark:|:white_check_mark:|:white_check_mark:|:white_check_mark:| -|CliсkHouse|:white_check_mark:|||:white_check_mark:|:white_check_mark:|:white_check_mark:| +| | [dbmate](https://github.com/amacneil/dbmate) | [goose](https://github.com/pressly/goose) | [sql-migrate](https://github.com/rubenv/sql-migrate) | [golang-migrate](https://github.com/golang-migrate/migrate) | [activerecord](http://guides.rubyonrails.org/active_record_migrations.html) | [sequelize](http://docs.sequelizejs.com/manual/tutorial/migrations.html) | +| ------------------------------------------------------------ | :------------------------------------------: | :---------------------------------------: | :--------------------------------------------------: | :---------------------------------------------------------: | :-------------------------------------------------------------------------: | :----------------------------------------------------------------------: | +| **Features** | +| Plain SQL migration files | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | | | +| Support for creating and dropping databases | :white_check_mark: | | | | :white_check_mark: | | +| Support for saving schema dump files | :white_check_mark: | | | | :white_check_mark: | | +| Timestamp-versioned migration files | :white_check_mark: | :white_check_mark: | | | :white_check_mark: | :white_check_mark: | +| Custom schema migrations table | :white_check_mark: | | :white_check_mark: | | | :white_check_mark: | +| Ability to wait for database to become ready | :white_check_mark: | | | | | | +| Database connection string loaded from environment variables | :white_check_mark: | | | | | | +| Automatically load .env file | :white_check_mark: | | | | | | +| No separate configuration file | :white_check_mark: | | | | :white_check_mark: | :white_check_mark: | +| Language/framework independent | :white_check_mark: | :eight_pointed_black_star: | :eight_pointed_black_star: | :eight_pointed_black_star: | | | +| **Drivers** | +| PostgreSQL | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | +| MySQL | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | +| SQLite | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | +| CliсkHouse | :white_check_mark: | | | :white_check_mark: | :white_check_mark: | :white_check_mark: | > :eight_pointed_black_star: In theory these tools could be used with other languages, but a Go development environment is required because binary builds are not provided. -*If you notice any inaccuracies in this table, please [propose a change](https://github.com/amacneil/dbmate/edit/master/README.md).* +_If you notice any inaccuracies in this table, please [propose a change](https://github.com/amacneil/dbmate/edit/main/README.md)._ ## Contributing From ece2c3c1229d6989b1c6357202c0f3a5ecca2225 Mon Sep 17 00:00:00 2001 From: Jae Bentvelzen Date: Tue, 9 Mar 2021 18:44:30 +1100 Subject: [PATCH 34/55] Improve error message when database URL is missing (#202) --- pkg/dbmate/db.go | 2 +- pkg/dbmate/db_test.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/dbmate/db.go b/pkg/dbmate/db.go index b955686..815a5cb 100644 --- a/pkg/dbmate/db.go +++ b/pkg/dbmate/db.go @@ -72,7 +72,7 @@ func New(databaseURL *url.URL) *DB { // GetDriver initializes the appropriate database driver func (db *DB) GetDriver() (Driver, error) { if db.DatabaseURL == nil || db.DatabaseURL.Scheme == "" { - return nil, errors.New("invalid url") + return nil, errors.New("invalid url, have you set your --url flag or DATABASE_URL environment variable?") } driverFunc := drivers[db.DatabaseURL.Scheme] diff --git a/pkg/dbmate/db_test.go b/pkg/dbmate/db_test.go index 0c07457..7b50daf 100644 --- a/pkg/dbmate/db_test.go +++ b/pkg/dbmate/db_test.go @@ -55,14 +55,14 @@ func TestGetDriver(t *testing.T) { db := dbmate.New(nil) drv, err := db.GetDriver() require.Nil(t, drv) - require.EqualError(t, err, "invalid url") + require.EqualError(t, err, "invalid url, have you set your --url flag or DATABASE_URL environment variable?") }) t.Run("missing schema", func(t *testing.T) { db := dbmate.New(dbutil.MustParseURL("//hi")) drv, err := db.GetDriver() require.Nil(t, drv) - require.EqualError(t, err, "invalid url") + require.EqualError(t, err, "invalid url, have you set your --url flag or DATABASE_URL environment variable?") }) t.Run("invalid driver", func(t *testing.T) { From 08022422d44bb92001aa32ba1f7c4071ae920dff Mon Sep 17 00:00:00 2001 From: Matt Snider Date: Tue, 9 Mar 2021 08:46:36 +0100 Subject: [PATCH 35/55] mysql: Fix escaping of '+' character in passwords (#200) Fixes #199 --- pkg/driver/mysql/mysql.go | 2 +- pkg/driver/mysql/mysql_test.go | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/pkg/driver/mysql/mysql.go b/pkg/driver/mysql/mysql.go index ca67330..fd25747 100644 --- a/pkg/driver/mysql/mysql.go +++ b/pkg/driver/mysql/mysql.go @@ -52,7 +52,7 @@ func connectionString(u *url.URL) string { // Get decoded user:pass userPassEncoded := u.User.String() - userPass, _ := url.QueryUnescape(userPassEncoded) + userPass, _ := url.PathUnescape(userPassEncoded) // Build DSN w/ user:pass percent-decoded normalizedString := "" diff --git a/pkg/driver/mysql/mysql_test.go b/pkg/driver/mysql/mysql_test.go index 0ff97d0..a4d2485 100644 --- a/pkg/driver/mysql/mysql_test.go +++ b/pkg/driver/mysql/mysql_test.go @@ -78,6 +78,18 @@ func TestConnectionString(t *testing.T) { require.Equal(t, "duhfsd7s:123!@123!@@tcp(host:123)/foo?flag=on&multiStatements=true", s) }) + t.Run("url encoding", func(t *testing.T) { + u, err := url.Parse("mysql://bob%2Balice:secret%5E%5B%2A%28%29@host:123/foo") + require.NoError(t, err) + require.Equal(t, "bob+alice:secret%5E%5B%2A%28%29", u.User.String()) + require.Equal(t, "123", u.Port()) + + s := connectionString(u) + // ensure that '+' is correctly encoded by url.PathUnescape as '+' + // (not whitespace as url.QueryUnescape generates) + require.Equal(t, "bob+alice:secret^[*()@tcp(host:123)/foo?multiStatements=true", s) + }) + t.Run("socket", func(t *testing.T) { // test with no user/pass u, err := url.Parse("mysql:///foo?socket=/var/run/mysqld/mysqld.sock&flag=on") From 7b92033d1bf8ccfc33b0a4e3cfe1aa19afe081a3 Mon Sep 17 00:00:00 2001 From: Ilia Ablamonov Date: Sat, 3 Apr 2021 01:34:22 +0400 Subject: [PATCH 36/55] Support non-UTC timezones (#208) Fix #207 --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 97986af..de526a3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -38,6 +38,7 @@ FROM alpine as release RUN apk add --no-cache \ mariadb-client \ postgresql-client \ - sqlite + sqlite \ + tzdata COPY --from=build /src/dist/dbmate-linux-amd64 /usr/local/bin/dbmate ENTRYPOINT ["dbmate"] From b020782b0ee9505edef504b163bd012411a0fb40 Mon Sep 17 00:00:00 2001 From: Adrian Macneil Date: Wed, 7 Apr 2021 13:51:50 -0700 Subject: [PATCH 37/55] v1.12.0 (#210) --- Dockerfile | 6 ++---- Makefile | 4 ++-- README.md | 2 +- go.mod | 5 +++-- go.sum | 14 ++++++++++---- pkg/dbmate/version.go | 2 +- 6 files changed, 19 insertions(+), 14 deletions(-) diff --git a/Dockerfile b/Dockerfile index de526a3..ab00ccb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,10 +16,8 @@ RUN apt-get update \ && rm -rf /var/lib/apt/lists/* # golangci-lint -RUN curl -fsSL -o /tmp/lint-install.sh https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh \ - && chmod +x /tmp/lint-install.sh \ - && /tmp/lint-install.sh -b /usr/local/bin v1.37.1 \ - && rm -f /tmp/lint-install.sh +RUN curl -fsSL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh \ + | sh -s -- -b /usr/local/bin v1.39.0 # download modules COPY go.* ./ diff --git a/Makefile b/Makefile index 70963e0..b543b64 100644 --- a/Makefile +++ b/Makefile @@ -49,8 +49,8 @@ build-all: clean build-linux-amd64 go build $(TAGS) $(STATICLDFLAGS) -o dist/dbmate-windows-amd64.exe . ls -lh dist -.PHONY: docker-make -docker-make: +.PHONY: docker-all +docker-all: docker-compose build docker-compose run --rm dev make diff --git a/README.md b/README.md index a0cae85..4d4edce 100644 --- a/README.md +++ b/README.md @@ -427,7 +427,7 @@ Dbmate is written in Go, pull requests are welcome. Tests are run against a real database using docker-compose. To build a docker image and run the tests: ```sh -$ make docker-make +$ make docker-all ``` To start a development shell: diff --git a/go.mod b/go.mod index 27c0bbf..652b8f1 100644 --- a/go.mod +++ b/go.mod @@ -6,11 +6,12 @@ require ( github.com/ClickHouse/clickhouse-go v1.4.3 github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/go-sql-driver/mysql v1.5.0 + github.com/frankban/quicktest v1.11.3 // indirect + github.com/go-sql-driver/mysql v1.6.0 github.com/joho/godotenv v1.3.0 github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d github.com/kr/text v0.2.0 // indirect - github.com/lib/pq v1.9.0 + github.com/lib/pq v1.10.0 github.com/mattn/go-sqlite3 v1.14.6 github.com/pierrec/lz4 v2.6.0+incompatible // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect diff --git a/go.sum b/go.sum index d948ac9..da9f515 100644 --- a/go.sum +++ b/go.sum @@ -12,9 +12,13 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/frankban/quicktest v1.11.3 h1:8sXhOn0uLys67V8EsXLc6eszDs8VXWxL3iRvebPhedY= +github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= -github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= -github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= +github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= @@ -27,8 +31,8 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.9.0 h1:L8nSXQQzAYByakOFMTwpjRoHsMJklur4Gi59b6VivR8= -github.com/lib/pq v1.9.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.10.0 h1:Zx5DJFEYQXio93kgXnQ09fXNiUKsqv4OUEu2UtGcB1E= +github.com/lib/pq v1.10.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v1.14.6 h1:dNPt6NO46WmLVt2DLNpwczCmdV5boIZ6g/tlDrlRUbg= github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= @@ -47,6 +51,8 @@ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5Cc github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M= github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/pkg/dbmate/version.go b/pkg/dbmate/version.go index a851cca..38918ee 100644 --- a/pkg/dbmate/version.go +++ b/pkg/dbmate/version.go @@ -1,4 +1,4 @@ package dbmate // Version of dbmate -const Version = "1.11.0" +const Version = "1.12.0" From abd02b7f0bec962ab44bfe4d5396f53ab9cf5610 Mon Sep 17 00:00:00 2001 From: Lucas Bremgartner Date: Wed, 7 Apr 2021 23:03:39 +0200 Subject: [PATCH 38/55] Update features for golang-migrate (#205) --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 4d4edce..5e196d9 100644 --- a/README.md +++ b/README.md @@ -403,13 +403,13 @@ Why another database schema migration tool? Dbmate was inspired by many other to | Plain SQL migration files | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | | | | Support for creating and dropping databases | :white_check_mark: | | | | :white_check_mark: | | | Support for saving schema dump files | :white_check_mark: | | | | :white_check_mark: | | -| Timestamp-versioned migration files | :white_check_mark: | :white_check_mark: | | | :white_check_mark: | :white_check_mark: | +| Timestamp-versioned migration files | :white_check_mark: | :white_check_mark: | | :white_check_mark: | :white_check_mark: | :white_check_mark: | | Custom schema migrations table | :white_check_mark: | | :white_check_mark: | | | :white_check_mark: | | Ability to wait for database to become ready | :white_check_mark: | | | | | | | Database connection string loaded from environment variables | :white_check_mark: | | | | | | | Automatically load .env file | :white_check_mark: | | | | | | -| No separate configuration file | :white_check_mark: | | | | :white_check_mark: | :white_check_mark: | -| Language/framework independent | :white_check_mark: | :eight_pointed_black_star: | :eight_pointed_black_star: | :eight_pointed_black_star: | | | +| No separate configuration file | :white_check_mark: | | | :white_check_mark: | :white_check_mark: | :white_check_mark: | +| Language/framework independent | :white_check_mark: | :eight_pointed_black_star: | :eight_pointed_black_star: | :white_check_mark: | | | | **Drivers** | | PostgreSQL | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | | MySQL | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | From cdbbdd65ea149ff9acf4342a136139adbbf33115 Mon Sep 17 00:00:00 2001 From: Adrian Macneil Date: Wed, 7 Apr 2021 14:16:18 -0700 Subject: [PATCH 39/55] Add Apple silicon (darwin/arm64) build (#211) --- Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile b/Makefile index b543b64..fe9aa16 100644 --- a/Makefile +++ b/Makefile @@ -45,6 +45,8 @@ build-all: clean build-linux-amd64 go build $(TAGS) $(STATICLDFLAGS) -o dist/dbmate-linux-arm64 . GOOS=darwin GOARCH=amd64 CC=o64-clang CXX=o64-clang++ \ go build $(TAGS) $(LDFLAGS) -o dist/dbmate-macos-amd64 . + GOOS=darwin GOARCH=arm64 CC=o64-clang CXX=o64-clang++ \ + go build $(TAGS) $(LDFLAGS) -o dist/dbmate-macos-arm64 . GOOS=windows GOARCH=amd64 CC=x86_64-w64-mingw32-gcc-posix CXX=x86_64-w64-mingw32-g++-posix \ go build $(TAGS) $(STATICLDFLAGS) -o dist/dbmate-windows-amd64.exe . ls -lh dist From 7c6f9ed747c112810f631981e1a49d178248ee2d Mon Sep 17 00:00:00 2001 From: Adrian Macneil Date: Thu, 22 Jul 2021 10:33:35 -0700 Subject: [PATCH 40/55] Add vscode config (#222) --- .vscode/extensions.json | 4 ++++ .vscode/settings.json | 9 +++++++++ 2 files changed, 13 insertions(+) create mode 100644 .vscode/extensions.json create mode 100644 .vscode/settings.json diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 0000000..dbac111 --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,4 @@ +// -*- jsonc -*- +{ + "recommendations": ["esbenp.prettier-vscode"] +} diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..1d88186 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,9 @@ +// -*- jsonc -*- +{ + "editor.defaultFormatter": "esbenp.prettier-vscode", + "editor.formatOnSave": true, + "files.eol": "\n", + "files.insertFinalNewline": true, + "files.trimFinalNewlines": true, + "go.formatTool": "goimports" +} From 4a3698c7acaf0a51ae72a3e4932948bda9aec8cc Mon Sep 17 00:00:00 2001 From: Adrian Macneil Date: Thu, 22 Jul 2021 15:19:10 -0700 Subject: [PATCH 41/55] Publish Docker image (#220) --- .github/workflows/{build.yml => ci.yml} | 17 ++++++++-- .github/workflows/publish-docker.sh | 45 +++++++++++++++++++++++++ .vscode/settings.json | 1 + docker-compose.yml | 1 + 4 files changed, 62 insertions(+), 2 deletions(-) rename .github/workflows/{build.yml => ci.yml} (68%) create mode 100755 .github/workflows/publish-docker.sh diff --git a/.github/workflows/build.yml b/.github/workflows/ci.yml similarity index 68% rename from .github/workflows/build.yml rename to .github/workflows/ci.yml index 0ffea4f..0065cb6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/ci.yml @@ -9,8 +9,9 @@ on: jobs: build: - name: Build & Test + name: Build runs-on: ubuntu-latest + steps: - name: Checkout uses: actions/checkout@v2 @@ -54,7 +55,19 @@ jobs: docker-compose run --rm dev make wait docker-compose run --rm dev make test - - name: Release + - name: Publish docker image + if: ${{ github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v') }} + env: + SRC_IMAGE: dbmate_release + DOCKERHUB_IMAGE: ${{ github.repository }} + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + GHCR_IMAGE: ghcr.io/${{ github.repository }} + GHCR_USERNAME: ${{ github.actor }} + GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: .github/workflows/publish-docker.sh + + - name: Publish release binaries uses: softprops/action-gh-release@v1 if: ${{ startsWith(github.ref, 'refs/tags/v') }} with: diff --git a/.github/workflows/publish-docker.sh b/.github/workflows/publish-docker.sh new file mode 100755 index 0000000..8a927c3 --- /dev/null +++ b/.github/workflows/publish-docker.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# Tag and publish Docker image + +set -euo pipefail + +echo "$DOCKERHUB_TOKEN" | (set -x && docker login --username "$DOCKERHUB_USERNAME" --password-stdin) +echo "$GHCR_TOKEN" | (set -x && docker login ghcr.io --username "$GHCR_USERNAME" --password-stdin) + +# Tag and push docker image +function docker_push { + src=$1 + dst=$2 + echo # newline + + ( + set -x + docker tag "$src" "$dst" + docker push "$dst" + ) +} + +# Publish image to both Docker Hub and GitHub Container Registry +function publish { + tag=$1 + docker_push "$SRC_IMAGE" "$DOCKERHUB_IMAGE:$tag" + docker_push "$SRC_IMAGE" "$GHCR_IMAGE:$tag" +} + +# Publish current branch/tag (e.g. `main` or `v1.2.3`) +ver=${GITHUB_REF##*/} +publish "$ver" + +# Publish major/minor/latest for version tags +if [[ "$GITHUB_REF" = refs/tags/v* ]]; then + major_ver=${ver%%.*} # e.g. `v1` + publish "$major_ver" + + minor_ver=${ver%.*} # e.g. `v1.2` + publish "$minor_ver" + + publish "latest" +fi + +# Clear credentials +rm -f ~/.docker/config.json diff --git a/.vscode/settings.json b/.vscode/settings.json index 1d88186..cff2122 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -5,5 +5,6 @@ "files.eol": "\n", "files.insertFinalNewline": true, "files.trimFinalNewlines": true, + "files.trimTrailingWhitespace": true, "go.formatTool": "goimports" } diff --git a/docker-compose.yml b/docker-compose.yml index f2a4754..de92e13 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,6 +20,7 @@ services: build: context: . target: release + image: dbmate_release mysql: image: mysql:5.7 From 511336d346ef1b2ea9901de220c46f3115253913 Mon Sep 17 00:00:00 2001 From: Adrian Macneil Date: Thu, 22 Jul 2021 15:40:15 -0700 Subject: [PATCH 42/55] v1.12.1 (#223) --- pkg/dbmate/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/dbmate/version.go b/pkg/dbmate/version.go index 38918ee..c0c2cdf 100644 --- a/pkg/dbmate/version.go +++ b/pkg/dbmate/version.go @@ -1,4 +1,4 @@ package dbmate // Version of dbmate -const Version = "1.12.0" +const Version = "1.12.1" From 26d5f9f30634427a59e856eae8c685df83f6a954 Mon Sep 17 00:00:00 2001 From: Adrian Macneil Date: Thu, 22 Jul 2021 16:09:57 -0700 Subject: [PATCH 43/55] Strip leading 'v' for docker image tags (#224) --- .github/workflows/publish-docker.sh | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/.github/workflows/publish-docker.sh b/.github/workflows/publish-docker.sh index 8a927c3..fc6f7c6 100755 --- a/.github/workflows/publish-docker.sh +++ b/.github/workflows/publish-docker.sh @@ -26,19 +26,17 @@ function publish { docker_push "$SRC_IMAGE" "$GHCR_IMAGE:$tag" } -# Publish current branch/tag (e.g. `main` or `v1.2.3`) -ver=${GITHUB_REF##*/} -publish "$ver" - -# Publish major/minor/latest for version tags if [[ "$GITHUB_REF" = refs/tags/v* ]]; then - major_ver=${ver%%.*} # e.g. `v1` - publish "$major_ver" - - minor_ver=${ver%.*} # e.g. `v1.2` - publish "$minor_ver" + # Publish major/minor/patch/latest version tags + ver=${GITHUB_REF#refs/tags/v} + publish "$ver" # e.g. `1.2.3` + publish "${ver%.*}" # e.g. `1.2` + publish "${ver%%.*}" # e.g. `1` publish "latest" +else + # Publish branch + publish "${GITHUB_REF##*/}" fi # Clear credentials From 6243c2b9a9e43a620d2f25895f6fcc3a0d199797 Mon Sep 17 00:00:00 2001 From: Adrian Macneil Date: Thu, 22 Jul 2021 16:57:17 -0700 Subject: [PATCH 44/55] Update readme with new docker image (#225) --- README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 5e196d9..826f2ce 100644 --- a/README.md +++ b/README.md @@ -66,16 +66,18 @@ $ sudo chmod +x /usr/local/bin/dbmate **Docker** -You can run dbmate using the official docker image (remember to set `--network=host` or see [this comment](https://github.com/amacneil/dbmate/issues/128#issuecomment-615924611) for more tips on using dbmate with docker networking): +Docker images are published to both Docker Hub ([`amacneil/dbmate`](https://hub.docker.com/r/amacneil/dbmate)) and Github Container Registry ([`ghcr.io/amacneil/dbmate`](https://ghcr.io/amacneil/dbmate)). + +Remember to set `--network=host` or see [this comment](https://github.com/amacneil/dbmate/issues/128#issuecomment-615924611) for more tips on using dbmate with docker networking): ```sh -$ docker run --rm -it --network=host amacneil/dbmate --help +$ docker run --rm -it --network=host ghcr.io/amacneil/dbmate:1 --help ``` If you wish to create or apply migrations, you will need to use Docker's [bind mount](https://docs.docker.com/storage/bind-mounts/) feature to make your local working directory (`pwd`) available inside the dbmate container: ```sh -$ docker run --rm -it --network=host -v "$(pwd)/db:/db" amacneil/dbmate new create_users_table +$ docker run --rm -it --network=host -v "$(pwd)/db:/db" ghcr.io/amacneil/dbmate:1 new create_users_table ``` **Heroku** From fb17e8eecad1b499503cc50e417290bb8c96a8dc Mon Sep 17 00:00:00 2001 From: Eng Zer Jun Date: Fri, 27 Aug 2021 12:13:37 +0800 Subject: [PATCH 45/55] refactor: move from io/ioutil to io and os package (#236) The `io/ioutil` package has been deprecated in Go 1.16. This commit replaces the existing `io/ioutil` functions with their new definitions in `io` and `os` packages. Signed-off-by: Eng Zer Jun --- pkg/dbmate/db.go | 5 ++--- pkg/dbmate/db_test.go | 11 +++++------ pkg/dbmate/migration.go | 4 ++-- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/pkg/dbmate/db.go b/pkg/dbmate/db.go index 815a5cb..f64dd88 100644 --- a/pkg/dbmate/db.go +++ b/pkg/dbmate/db.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "net/url" "os" "path/filepath" @@ -226,7 +225,7 @@ func (db *DB) dumpSchema(drv Driver) error { } // write schema to file - return ioutil.WriteFile(db.SchemaFile, schema, 0644) + return os.WriteFile(db.SchemaFile, schema, 0644) } // ensureDir creates a directory if it does not already exist @@ -402,7 +401,7 @@ func (db *DB) printVerbose(result sql.Result) { } func findMigrationFiles(dir string, re *regexp.Regexp) ([]string, error) { - files, err := ioutil.ReadDir(dir) + files, err := os.ReadDir(dir) if err != nil { return nil, fmt.Errorf("could not find migrations directory `%s`", dir) } diff --git a/pkg/dbmate/db_test.go b/pkg/dbmate/db_test.go index 7b50daf..d2e42e5 100644 --- a/pkg/dbmate/db_test.go +++ b/pkg/dbmate/db_test.go @@ -1,7 +1,6 @@ package dbmate_test import ( - "io/ioutil" "net/url" "os" "path/filepath" @@ -102,7 +101,7 @@ func TestDumpSchema(t *testing.T) { db := newTestDB(t, u) // create custom schema file directory - dir, err := ioutil.TempDir("", "dbmate") + dir, err := os.MkdirTemp("", "dbmate") require.NoError(t, err) defer func() { err := os.RemoveAll(dir) @@ -129,7 +128,7 @@ func TestDumpSchema(t *testing.T) { require.NoError(t, err) // verify schema - schema, err := ioutil.ReadFile(db.SchemaFile) + schema, err := os.ReadFile(db.SchemaFile) require.NoError(t, err) require.Contains(t, string(schema), "-- PostgreSQL database dump") } @@ -140,7 +139,7 @@ func TestAutoDumpSchema(t *testing.T) { db.AutoDumpSchema = true // create custom schema file directory - dir, err := ioutil.TempDir("", "dbmate") + dir, err := os.MkdirTemp("", "dbmate") require.NoError(t, err) defer func() { err := os.RemoveAll(dir) @@ -163,7 +162,7 @@ func TestAutoDumpSchema(t *testing.T) { require.NoError(t, err) // verify schema - schema, err := ioutil.ReadFile(db.SchemaFile) + schema, err := os.ReadFile(db.SchemaFile) require.NoError(t, err) require.Contains(t, string(schema), "-- PostgreSQL database dump") @@ -176,7 +175,7 @@ func TestAutoDumpSchema(t *testing.T) { require.NoError(t, err) // schema should be recreated - schema, err = ioutil.ReadFile(db.SchemaFile) + schema, err = os.ReadFile(db.SchemaFile) require.NoError(t, err) require.Contains(t, string(schema), "-- PostgreSQL database dump") } diff --git a/pkg/dbmate/migration.go b/pkg/dbmate/migration.go index 2c0f7b6..bb00018 100644 --- a/pkg/dbmate/migration.go +++ b/pkg/dbmate/migration.go @@ -2,7 +2,7 @@ package dbmate import ( "fmt" - "io/ioutil" + "os" "regexp" "strings" ) @@ -33,7 +33,7 @@ func NewMigration() Migration { // parseMigration reads a migration file and returns (up Migration, down Migration, error) func parseMigration(path string) (Migration, Migration, error) { - data, err := ioutil.ReadFile(path) + data, err := os.ReadFile(path) if err != nil { return NewMigration(), NewMigration(), err } From 81fe01b34fec705ec86a71e44c81c72c0443d78b Mon Sep 17 00:00:00 2001 From: Matthew Wraith Date: Fri, 17 Dec 2021 16:44:14 -0800 Subject: [PATCH 46/55] Postgres defaults to unix socket (#230) --- pkg/driver/postgres/postgres.go | 10 +++++++++- pkg/driver/postgres/postgres_test.go | 20 ++++++++++++++++---- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/pkg/driver/postgres/postgres.go b/pkg/driver/postgres/postgres.go index 1917cb1..60b1333 100644 --- a/pkg/driver/postgres/postgres.go +++ b/pkg/driver/postgres/postgres.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "net/url" + "runtime" "strings" "github.com/amacneil/dbmate/pkg/dbmate" @@ -48,7 +49,14 @@ func connectionString(u *url.URL) string { // default hostname if hostname == "" { - hostname = "localhost" + switch runtime.GOOS { + case "linux": + query.Set("host", "/var/run/postgresql") + case "darwin", "freebsd", "dragonfly", "openbsd", "netbsd": + query.Set("host", "/tmp") + default: + hostname = "localhost" + } } // host param overrides url hostname diff --git a/pkg/driver/postgres/postgres_test.go b/pkg/driver/postgres/postgres_test.go index f3f8793..1a49ce0 100644 --- a/pkg/driver/postgres/postgres_test.go +++ b/pkg/driver/postgres/postgres_test.go @@ -4,6 +4,7 @@ import ( "database/sql" "net/url" "os" + "runtime" "testing" "github.com/amacneil/dbmate/pkg/dbmate" @@ -50,13 +51,24 @@ func TestGetDriver(t *testing.T) { require.Equal(t, "schema_migrations", drv.migrationsTableName) } +func defaultConnString() string { + switch runtime.GOOS { + case "linux": + return "postgres://:5432/foo?host=%2Fvar%2Frun%2Fpostgresql" + case "darwin", "freebsd", "dragonfly", "openbsd", "netbsd": + return "postgres://:5432/foo?host=%2Ftmp" + default: + return "postgres://localhost:5432/foo" + } +} + func TestConnectionString(t *testing.T) { cases := []struct { input string expected string }{ // defaults - {"postgres:///foo", "postgres://localhost:5432/foo"}, + {"postgres:///foo", defaultConnString()}, // support custom url params {"postgres://bob:secret@myhost:1234/foo?bar=baz", "postgres://bob:secret@myhost:1234/foo?bar=baz"}, // support `host` and `port` via url params @@ -85,11 +97,11 @@ func TestConnectionArgsForDump(t *testing.T) { expected []string }{ // defaults - {"postgres:///foo", []string{"postgres://localhost:5432/foo"}}, + {"postgres:///foo", []string{defaultConnString()}}, // support single schema - {"postgres:///foo?search_path=foo", []string{"--schema", "foo", "postgres://localhost:5432/foo"}}, + {"postgres:///foo?search_path=foo", []string{"--schema", "foo", defaultConnString()}}, // support multiple schemas - {"postgres:///foo?search_path=foo,public", []string{"--schema", "foo", "--schema", "public", "postgres://localhost:5432/foo"}}, + {"postgres:///foo?search_path=foo,public", []string{"--schema", "foo", "--schema", "public", defaultConnString()}}, } for _, c := range cases { From 06d8bb7567800d2d986e550e9b083f46315d22b7 Mon Sep 17 00:00:00 2001 From: Adrian Macneil Date: Fri, 17 Dec 2021 17:22:48 -0800 Subject: [PATCH 47/55] Remove CodeQL workflow --- .github/workflows/codeql-analysis.yml | 40 --------------------------- 1 file changed, 40 deletions(-) delete mode 100644 .github/workflows/codeql-analysis.yml diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml deleted file mode 100644 index ed3cae6..0000000 --- a/.github/workflows/codeql-analysis.yml +++ /dev/null @@ -1,40 +0,0 @@ -# For most projects, this workflow file will not need changing; you simply need -# to commit it to your repository. -# -# You may wish to alter this file to override the set of languages analyzed, -# or to provide custom queries or build logic. - -name: "CodeQL" - -on: - push: - branches: [main] - pull_request: - branches: [main] - schedule: - - cron: "0 0 * * 4" - -jobs: - analyze: - name: Analyze - runs-on: ubuntu-latest - - strategy: - fail-fast: false - matrix: - language: ["go"] - - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - - name: Initialize CodeQL - uses: github/codeql-action/init@v1 - with: - languages: ${{ matrix.language }} - - - name: Autobuild - uses: github/codeql-action/autobuild@v1 - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 From f69f1dea03da0ac9329ce61e89ed7953a3005cd7 Mon Sep 17 00:00:00 2001 From: Adrian Macneil Date: Sun, 19 Dec 2021 21:08:22 -0800 Subject: [PATCH 48/55] Build using native OS workers (#231) --- .github/workflows/ci.yml | 104 ++++++++++++++------ Dockerfile | 23 ++--- Makefile | 80 ++++++++------- {.github/workflows => ci}/publish-docker.sh | 0 docker-compose.yml | 2 +- pkg/driver/mysql/mysql_test.go | 6 +- pkg/driver/postgres/postgres_test.go | 4 +- 7 files changed, 123 insertions(+), 96 deletions(-) rename {.github/workflows => ci}/publish-docker.sh (100%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0065cb6..de83371 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,44 +3,94 @@ name: CI on: push: branches: [main] - tags: "v*" + tags: "*" pull_request: - branches: [main] jobs: build: - name: Build + strategy: + fail-fast: false + matrix: + include: + - os: linux + image: ubuntu-latest + arch: amd64 + env: {} + - os: linux + image: ubuntu-latest + arch: arm64 + setup: sudo apt-get update && sudo apt-get install -qq gcc-aarch64-linux-gnu + env: + CC: aarch64-linux-gnu-gcc + CXX: aarch64-linux-gnu-g++ + - os: macos + image: macos-latest + arch: amd64 + env: {} + - os: macos + image: macos-latest + arch: arm64 + env: {} + - os: windows + image: windows-latest + arch: amd64 + env: {} + + name: Build (${{ matrix.os }}/${{ matrix.arch }}) + runs-on: ${{ matrix.image }} + env: ${{ matrix.env }} + + steps: + - uses: actions/checkout@v2 + + - uses: actions/setup-go@v2 + with: + go-version: "1.17" + + - name: Setup environment + run: ${{ matrix.setup }} + + - run: go mod download + + - run: make build ls + env: + GOARCH: ${{ matrix.arch }} + OUTPUT: dbmate-${{ matrix.os }}-${{ matrix.arch }} + + - run: dist/dbmate-${{ matrix.os }}-${{ matrix.arch }} --help + if: ${{ matrix.arch == 'amd64' }} + + - name: Publish binaries + uses: softprops/action-gh-release@v1 + if: ${{ startsWith(github.ref, 'refs/tags/v') }} + with: + files: dist/dbmate-* + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + docker: + name: Docker Test (linux/amd64) runs-on: ubuntu-latest steps: - - name: Checkout - uses: actions/checkout@v2 + - uses: actions/checkout@v2 - - name: Environment + - name: Check docker environment run: | set -x docker version docker-compose version - - name: Cache - uses: actions/cache@v2 - with: - key: cache - path: .cache - - name: Build docker image run: | set -x docker-compose build docker-compose run --rm --no-deps dbmate --version - - name: Build binaries - run: | - set -x - docker-compose run --rm --no-deps dev make build-all - dist/dbmate-linux-amd64 --version + - name: Run make build + run: docker-compose run --rm --no-deps dev make build ls - - name: Lint + - name: Run make lint run: docker-compose run --rm --no-deps dev make lint - name: Start test dependencies @@ -48,12 +98,10 @@ jobs: set -x docker-compose pull --quiet docker-compose up --detach - - - name: Run tests - run: | - set -x docker-compose run --rm dev make wait - docker-compose run --rm dev make test + + - name: Run make test + run: docker-compose run --rm dev make test - name: Publish docker image if: ${{ github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v') }} @@ -65,12 +113,4 @@ jobs: GHCR_IMAGE: ghcr.io/${{ github.repository }} GHCR_USERNAME: ${{ github.actor }} GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: .github/workflows/publish-docker.sh - - - name: Publish release binaries - uses: softprops/action-gh-release@v1 - if: ${{ startsWith(github.ref, 'refs/tags/v') }} - with: - files: dist/* - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: ci/publish-docker.sh diff --git a/Dockerfile b/Dockerfile index ab00ccb..db996f2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,16 +1,13 @@ # development image -FROM techknowlogick/xgo:go-1.16.x as dev +FROM golang:1.17 as dev WORKDIR /src -ENV GOCACHE /src/.cache/go-build - -# enable cgo to build sqlite -ENV CGO_ENABLED 1 # install database clients RUN apt-get update \ && apt-get install -qq --no-install-recommends \ curl \ - mysql-client \ + file \ + mariadb-client \ postgresql-client \ sqlite3 \ && rm -rf /var/lib/apt/lists/* @@ -20,15 +17,9 @@ RUN curl -fsSL https://raw.githubusercontent.com/golangci/golangci-lint/master/i | sh -s -- -b /usr/local/bin v1.39.0 # download modules -COPY go.* ./ +COPY go.* /src/ RUN go mod download - -ENTRYPOINT [] -CMD ["/bin/bash"] - -# build stage -FROM dev as build -COPY . ./ +COPY . /src/ RUN make build # release stage @@ -38,5 +29,5 @@ RUN apk add --no-cache \ postgresql-client \ sqlite \ tzdata -COPY --from=build /src/dist/dbmate-linux-amd64 /usr/local/bin/dbmate -ENTRYPOINT ["dbmate"] +COPY --from=dev /src/dist/dbmate /usr/local/bin/dbmate +ENTRYPOINT ["/usr/local/bin/dbmate"] diff --git a/Makefile b/Makefile index fe9aa16..7f3e418 100644 --- a/Makefile +++ b/Makefile @@ -1,60 +1,58 @@ -# no static linking for macos -LDFLAGS := -ldflags '-s' -# statically link binaries (to support alpine + scratch containers) -STATICLDFLAGS := -ldflags '-s -extldflags "-static"' -# avoid building code that is incompatible with static linking -TAGS := -tags netgo,osusergo,sqlite_omit_load_extension,sqlite_json +# enable cgo to build sqlite +export CGO_ENABLED = 1 + +# strip binaries +FLAGS := -tags sqlite_omit_load_extension,sqlite_json -ldflags '-s' + +GOOS := $(shell go env GOOS) +ifeq ($(GOOS),linux) + # statically link binaries to support alpine linux + FLAGS := -tags netgo,osusergo,sqlite_omit_load_extension,sqlite_json -ldflags '-s -extldflags "-static"' +endif +ifeq ($(GOOS),darwin) + export SDKROOT ?= $(shell xcrun --sdk macosx --show-sdk-path) +endif + +OUTPUT ?= dbmate .PHONY: all -all: build test lint +all: fix build wait test + +.PHONY: clean +clean: + rm -rf dist + +.PHONY: build +build: clean + go build -o dist/$(OUTPUT) $(FLAGS) . + +.PHONY: ls +ls: + ls -lh dist/$(OUTPUT) + file dist/$(OUTPUT) .PHONY: test test: - go test -p 1 $(TAGS) $(STATICLDFLAGS) ./... - -.PHONY: fix -fix: - golangci-lint run --fix + go test -p 1 $(FLAGS) ./... .PHONY: lint lint: golangci-lint run +.PHONY: fix +fix: + golangci-lint run --fix + .PHONY: wait wait: - dist/dbmate-linux-amd64 -e CLICKHOUSE_TEST_URL wait - dist/dbmate-linux-amd64 -e MYSQL_TEST_URL wait - dist/dbmate-linux-amd64 -e POSTGRES_TEST_URL wait - -.PHONY: clean -clean: - rm -rf dist/* - -.PHONY: build -build: clean build-linux-amd64 - ls -lh dist - -.PHONY: build-linux-amd64 -build-linux-amd64: - GOOS=linux GOARCH=amd64 \ - go build $(TAGS) $(STATICLDFLAGS) -o dist/dbmate-linux-amd64 . - -.PHONY: build-all -build-all: clean build-linux-amd64 - GOOS=linux GOARCH=arm64 CC=aarch64-linux-gnu-gcc-5 CXX=aarch64-linux-gnu-g++-5 \ - go build $(TAGS) $(STATICLDFLAGS) -o dist/dbmate-linux-arm64 . - GOOS=darwin GOARCH=amd64 CC=o64-clang CXX=o64-clang++ \ - go build $(TAGS) $(LDFLAGS) -o dist/dbmate-macos-amd64 . - GOOS=darwin GOARCH=arm64 CC=o64-clang CXX=o64-clang++ \ - go build $(TAGS) $(LDFLAGS) -o dist/dbmate-macos-arm64 . - GOOS=windows GOARCH=amd64 CC=x86_64-w64-mingw32-gcc-posix CXX=x86_64-w64-mingw32-g++-posix \ - go build $(TAGS) $(STATICLDFLAGS) -o dist/dbmate-windows-amd64.exe . - ls -lh dist + dist/dbmate -e CLICKHOUSE_TEST_URL wait + dist/dbmate -e MYSQL_TEST_URL wait + dist/dbmate -e POSTGRES_TEST_URL wait .PHONY: docker-all docker-all: docker-compose build - docker-compose run --rm dev make + docker-compose run --rm dev make all .PHONY: docker-sh docker-sh: diff --git a/.github/workflows/publish-docker.sh b/ci/publish-docker.sh similarity index 100% rename from .github/workflows/publish-docker.sh rename to ci/publish-docker.sh diff --git a/docker-compose.yml b/docker-compose.yml index de92e13..cbd7b31 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,4 @@ -version: '2.3' +version: "2.3" services: dev: build: diff --git a/pkg/driver/mysql/mysql_test.go b/pkg/driver/mysql/mysql_test.go index a4d2485..6bab279 100644 --- a/pkg/driver/mysql/mysql_test.go +++ b/pkg/driver/mysql/mysql_test.go @@ -179,10 +179,8 @@ func TestMySQLDumpSchema(t *testing.T) { drv.databaseURL.Path = "/fakedb" schema, err = drv.DumpSchema(db) require.Nil(t, schema) - require.EqualError(t, err, "mysqldump: [Warning] Using a password "+ - "on the command line interface can be insecure.\n"+ - "mysqldump: Got error: 1049: "+ - "Unknown database 'fakedb' when selecting the database") + require.Error(t, err) + require.Contains(t, err.Error(), "Unknown database 'fakedb'") } func TestMySQLDatabaseExists(t *testing.T) { diff --git a/pkg/driver/postgres/postgres_test.go b/pkg/driver/postgres/postgres_test.go index 1a49ce0..7705d6e 100644 --- a/pkg/driver/postgres/postgres_test.go +++ b/pkg/driver/postgres/postgres_test.go @@ -186,8 +186,8 @@ func TestPostgresDumpSchema(t *testing.T) { drv.databaseURL.Path = "/fakedb" schema, err = drv.DumpSchema(db) require.Nil(t, schema) - require.EqualError(t, err, "pg_dump: [archiver (db)] connection to database "+ - "\"fakedb\" failed: FATAL: database \"fakedb\" does not exist") + require.Error(t, err) + require.Contains(t, err.Error(), "database \"fakedb\" does not exist") }) t.Run("custom migrations table with schema", func(t *testing.T) { From 955c9ac653bf451b5e61ddbb3a28e9e757233073 Mon Sep 17 00:00:00 2001 From: Adrian Macneil Date: Sun, 19 Dec 2021 21:28:14 -0800 Subject: [PATCH 49/55] Update to MySQL 8.0 image for testing (#250) --- Makefile | 1 + docker-compose.yml | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 7f3e418..08beceb 100644 --- a/Makefile +++ b/Makefile @@ -51,6 +51,7 @@ wait: .PHONY: docker-all docker-all: + docker-compose pull docker-compose build docker-compose run --rm dev make all diff --git a/docker-compose.yml b/docker-compose.yml index cbd7b31..3134144 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -23,8 +23,9 @@ services: image: dbmate_release mysql: - image: mysql:5.7 + image: mysql/mysql-server:8.0 environment: + MYSQL_ROOT_HOST: "%" MYSQL_ROOT_PASSWORD: root postgres: From c99d611cb478d7529ba3646dfe71d1332fe59fe5 Mon Sep 17 00:00:00 2001 From: Adrian Macneil Date: Mon, 20 Dec 2021 10:29:33 -0800 Subject: [PATCH 50/55] Upgrade golangci-lint to v1.43.0 (#251) --- .golangci.yml | 2 +- Dockerfile | 2 +- pkg/driver/sqlite/sqlite.go | 1 + pkg/driver/sqlite/sqlite_test.go | 1 + 4 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index ac76638..e985ffd 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -5,12 +5,12 @@ linters: - depguard - errcheck - goimports - - golint - gosimple - govet - ineffassign - misspell - nakedret + - revive - rowserrcheck - staticcheck - structcheck diff --git a/Dockerfile b/Dockerfile index db996f2..b6fce52 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,7 @@ RUN apt-get update \ # golangci-lint RUN curl -fsSL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh \ - | sh -s -- -b /usr/local/bin v1.39.0 + | sh -s -- -b /usr/local/bin v1.43.0 # download modules COPY go.* /src/ diff --git a/pkg/driver/sqlite/sqlite.go b/pkg/driver/sqlite/sqlite.go index a03954e..be2c5f1 100644 --- a/pkg/driver/sqlite/sqlite.go +++ b/pkg/driver/sqlite/sqlite.go @@ -1,3 +1,4 @@ +//go:build cgo // +build cgo package sqlite diff --git a/pkg/driver/sqlite/sqlite_test.go b/pkg/driver/sqlite/sqlite_test.go index f4638f1..8473e61 100644 --- a/pkg/driver/sqlite/sqlite_test.go +++ b/pkg/driver/sqlite/sqlite_test.go @@ -1,3 +1,4 @@ +//go:build cgo // +build cgo package sqlite From 52cd75fbc1b2d4745ddecb82a57e0a2aa1d21a4e Mon Sep 17 00:00:00 2001 From: Adam Aposhian Date: Wed, 29 Dec 2021 16:58:30 -0700 Subject: [PATCH 51/55] Publish multiarch docker images (#241) --- .github/workflows/ci.yml | 59 ++++++++++++++++++++++++++++++++-------- ci/publish-docker.sh | 43 ----------------------------- 2 files changed, 47 insertions(+), 55 deletions(-) delete mode 100755 ci/publish-docker.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index de83371..f1024f0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -75,13 +75,20 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Check docker environment + - name: Configure QEMU + uses: docker/setup-qemu-action@v1 + + - name: Configure Buildx + uses: docker/setup-buildx-action@v1 + + - name: Check Docker environment run: | set -x docker version + docker buildx version docker-compose version - - name: Build docker image + - name: Build Docker image run: | set -x docker-compose build @@ -103,14 +110,42 @@ jobs: - name: Run make test run: docker-compose run --rm dev make test - - name: Publish docker image + - name: Login to Docker Hub + uses: docker/login-action@v1 if: ${{ github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v') }} - env: - SRC_IMAGE: dbmate_release - DOCKERHUB_IMAGE: ${{ github.repository }} - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - GHCR_IMAGE: ghcr.io/${{ github.repository }} - GHCR_USERNAME: ${{ github.actor }} - GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: ci/publish-docker.sh + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v1 + if: ${{ github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v') }} + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Generate Docker image tags + id: meta + uses: docker/metadata-action@v3 + if: ${{ github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v') }} + with: + images: | + ${{ github.repository }} + ghcr.io/${{ github.repository }} + tags: | + type=ref,event=branch + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + + - name: Publish Docker image + uses: docker/build-push-action@v2 + if: ${{ github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v') }} + with: + context: . + target: release + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/ci/publish-docker.sh b/ci/publish-docker.sh deleted file mode 100755 index fc6f7c6..0000000 --- a/ci/publish-docker.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash -# Tag and publish Docker image - -set -euo pipefail - -echo "$DOCKERHUB_TOKEN" | (set -x && docker login --username "$DOCKERHUB_USERNAME" --password-stdin) -echo "$GHCR_TOKEN" | (set -x && docker login ghcr.io --username "$GHCR_USERNAME" --password-stdin) - -# Tag and push docker image -function docker_push { - src=$1 - dst=$2 - echo # newline - - ( - set -x - docker tag "$src" "$dst" - docker push "$dst" - ) -} - -# Publish image to both Docker Hub and GitHub Container Registry -function publish { - tag=$1 - docker_push "$SRC_IMAGE" "$DOCKERHUB_IMAGE:$tag" - docker_push "$SRC_IMAGE" "$GHCR_IMAGE:$tag" -} - -if [[ "$GITHUB_REF" = refs/tags/v* ]]; then - # Publish major/minor/patch/latest version tags - ver=${GITHUB_REF#refs/tags/v} - - publish "$ver" # e.g. `1.2.3` - publish "${ver%.*}" # e.g. `1.2` - publish "${ver%%.*}" # e.g. `1` - publish "latest" -else - # Publish branch - publish "${GITHUB_REF##*/}" -fi - -# Clear credentials -rm -f ~/.docker/config.json From 5b60f6810769878803fb90e6303c3d56c8d0ee47 Mon Sep 17 00:00:00 2001 From: Adrian Macneil Date: Sun, 2 Jan 2022 21:56:57 -0800 Subject: [PATCH 52/55] Add Dependabot for GitHub Actions --- .github/dependabot.yml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..1230149 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,6 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "daily" From a55233c50b02fc863ac7a720febc6e4f2e7a871a Mon Sep 17 00:00:00 2001 From: Technofab Date: Fri, 7 Jan 2022 21:12:30 +0100 Subject: [PATCH 53/55] add ability to specify limits and the target version to migrate to --- main.go | 12 +++++- pkg/dbmate/db.go | 107 +++++++++++++++++++++++++++++++---------------- 2 files changed, 83 insertions(+), 36 deletions(-) diff --git a/main.go b/main.go index 2ed5875..3899cbe 100644 --- a/main.go +++ b/main.go @@ -109,6 +109,7 @@ func NewApp() *cli.App { }, }, Action: action(func(db *dbmate.DB, c *cli.Context) error { + db.TargetVersion = c.Args().First() db.Verbose = c.Bool("verbose") return db.CreateAndMigrate() }), @@ -129,7 +130,7 @@ func NewApp() *cli.App { }, { Name: "migrate", - Usage: "Migrate to the latest version", + Usage: "Migrate to the specified or latest version", Flags: []cli.Flag{ &cli.BoolFlag{ Name: "verbose", @@ -139,6 +140,7 @@ func NewApp() *cli.App { }, }, Action: action(func(db *dbmate.DB, c *cli.Context) error { + db.TargetVersion = c.Args().First() db.Verbose = c.Bool("verbose") return db.Migrate() }), @@ -154,8 +156,16 @@ func NewApp() *cli.App { EnvVars: []string{"DBMATE_VERBOSE"}, Usage: "print the result of each statement execution", }, + &cli.IntFlag{ + Name: "limit", + Aliases: []string{"l"}, + Usage: "Limits the amount of rollbacks (defaults to 1 if no target version is specified)", + Value: -1, + }, }, Action: action(func(db *dbmate.DB, c *cli.Context) error { + db.TargetVersion = c.Args().First() + db.Limit = c.Int("limit") db.Verbose = c.Bool("verbose") return db.Rollback() }), diff --git a/pkg/dbmate/db.go b/pkg/dbmate/db.go index f64dd88..8240207 100644 --- a/pkg/dbmate/db.go +++ b/pkg/dbmate/db.go @@ -41,6 +41,8 @@ type DB struct { WaitBefore bool WaitInterval time.Duration WaitTimeout time.Duration + Limit int + TargetVersion string Log io.Writer } @@ -64,6 +66,8 @@ func New(databaseURL *url.URL) *DB { WaitBefore: false, WaitInterval: DefaultWaitInterval, WaitTimeout: DefaultWaitTimeout, + Limit: -1, + TargetVersion: "", Log: os.Stdout, } } @@ -336,14 +340,14 @@ func (db *DB) migrate(drv Driver) error { } defer dbutil.MustClose(sqlDB) - applied, err := drv.SelectMigrations(sqlDB, -1) + applied, err := drv.SelectMigrations(sqlDB, db.Limit) if err != nil { return err } for _, filename := range files { ver := migrationVersion(filename) - if ok := applied[ver]; ok { + if ok := applied[ver]; ok && ver != db.TargetVersion { // migration already applied continue } @@ -379,6 +383,11 @@ func (db *DB) migrate(drv Driver) error { if err != nil { return err } + + if ver == db.TargetVersion { + fmt.Fprintf(db.Log, "Reached target version %s\n", ver) + break + } } // automatically update schema file, silence errors @@ -469,55 +478,83 @@ func (db *DB) Rollback() error { } defer dbutil.MustClose(sqlDB) - applied, err := drv.SelectMigrations(sqlDB, 1) + limit := db.Limit + // default limit is -1, if we don't specify a version it should only rollback one version, not all + if limit <= 0 && db.TargetVersion == "" { + limit = 1 + } + + applied, err := drv.SelectMigrations(sqlDB, limit) if err != nil { return err } - // grab most recent applied migration (applied has len=1) - latest := "" - for ver := range applied { - latest = ver - } - if latest == "" { - return fmt.Errorf("can't rollback: no migrations have been applied") + if len(applied) == 0 { + return fmt.Errorf("can't rollback, no migrations found") } - filename, err := findMigrationFile(db.MigrationsDir, latest) - if err != nil { - return err + var versions []string + for v := range applied { + versions = append(versions, v) } - fmt.Fprintf(db.Log, "Rolling back: %s\n", filename) + // new → old + sort.Sort(sort.Reverse(sort.StringSlice(versions))) - _, down, err := parseMigration(filepath.Join(db.MigrationsDir, filename)) - if err != nil { - return err + if db.TargetVersion != "" { + cache := map[string]bool{} + found := false + + // latest version comes first, so take every version until the version matches + for _, ver := range versions { + if ver == db.TargetVersion { + found = true + break + } + cache[ver] = true + } + if !found { + return fmt.Errorf("target version not found") + } + applied = cache } - execMigration := func(tx dbutil.Transaction) error { - // rollback migration - result, err := tx.Exec(down.Contents) + for version := range applied { + filename, err := findMigrationFile(db.MigrationsDir, version) if err != nil { return err - } else if db.Verbose { - db.printVerbose(result) } - // remove migration record - return drv.DeleteMigration(tx, latest) - } + fmt.Fprintf(db.Log, "Rolling back: %s\n", filename) + _, down, err := parseMigration(filepath.Join(db.MigrationsDir, filename)) + if err != nil { + return err + } - if down.Options.Transaction() { - // begin transaction - err = doTransaction(sqlDB, execMigration) - } else { - // run outside of transaction - err = execMigration(sqlDB) - } + execMigration := func(tx dbutil.Transaction) error { + // rollback migration + result, err := tx.Exec(down.Contents) + if err != nil { + return err + } else if db.Verbose { + db.printVerbose(result) + } - if err != nil { - return err + // remove migration record + return drv.DeleteMigration(tx, version) + } + + if down.Options.Transaction() { + // begin transaction + err = doTransaction(sqlDB, execMigration) + } else { + // run outside of transaction + err = execMigration(sqlDB) + } + + if err != nil { + return err + } } // automatically update schema file, silence errors @@ -582,7 +619,7 @@ func (db *DB) CheckMigrationsStatus(drv Driver) ([]StatusResult, error) { } defer dbutil.MustClose(sqlDB) - applied, err := drv.SelectMigrations(sqlDB, -1) + applied, err := drv.SelectMigrations(sqlDB, db.Limit) if err != nil { return nil, err } From a5b92832d3c4afff52378f9e239d6e97005fb9c5 Mon Sep 17 00:00:00 2001 From: Technofab Date: Fri, 7 Jan 2022 21:20:32 +0100 Subject: [PATCH 54/55] fix small mistake --- pkg/dbmate/db.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/dbmate/db.go b/pkg/dbmate/db.go index 8240207..7059043 100644 --- a/pkg/dbmate/db.go +++ b/pkg/dbmate/db.go @@ -340,7 +340,7 @@ func (db *DB) migrate(drv Driver) error { } defer dbutil.MustClose(sqlDB) - applied, err := drv.SelectMigrations(sqlDB, db.Limit) + applied, err := drv.SelectMigrations(sqlDB, -1) if err != nil { return err } From 60e93d5c109c680eb23a419965fcc31f5e074b21 Mon Sep 17 00:00:00 2001 From: pawndev Date: Mon, 13 Jun 2022 17:00:23 +0200 Subject: [PATCH 55/55] test: Add some to test up-to/down-to specific migrations --- README.md | 14 ++ pkg/dbmate/db_test.go | 151 +++++++++++++++++- .../20220607110405_test_category.sql | 9 ++ 3 files changed, 167 insertions(+), 7 deletions(-) create mode 100644 testdata/db/migrations/20220607110405_test_category.sql diff --git a/README.md b/README.md index 826f2ce..b2cb010 100644 --- a/README.md +++ b/README.md @@ -284,6 +284,12 @@ Writing: ./db/schema.sql Pending migrations are always applied in numerical order. However, dbmate does not prevent migrations from being applied out of order if they are committed independently (for example: if a developer has been working on a branch for a long time, and commits a migration which has a lower version number than other already-applied migrations, dbmate will simply apply the pending migration). See [#159](https://github.com/amacneil/dbmate/issues/159) for a more detailed explanation. +You can also specify a migration to up-to. + +```sh +$ dbmate up 20151127184807 +``` + ### Rolling Back Migrations By default, dbmate doesn't know how to roll back a migration. In development, it's often useful to be able to revert your database to a previous state. To accomplish this, implement the `migrate:down` section: @@ -308,6 +314,14 @@ Rolling back: 20151127184807_create_users_table.sql Writing: ./db/schema.sql ``` +You can also rollback to a specific migration. + +```sh +$ dbmate rollback 20151127184807 +# or, with a limit option +$ dbmate rollback -limit 2 # will rollback the last two migrations +``` + ### Migration Options dbmate supports options passed to a migration block in the form of `key:value` pairs. List of supported options: diff --git a/pkg/dbmate/db_test.go b/pkg/dbmate/db_test.go index d2e42e5..8582a63 100644 --- a/pkg/dbmate/db_test.go +++ b/pkg/dbmate/db_test.go @@ -47,6 +47,8 @@ func TestNew(t *testing.T) { require.False(t, db.WaitBefore) require.Equal(t, time.Second, db.WaitInterval) require.Equal(t, 60*time.Second, db.WaitTimeout) + require.Equal(t, -1, db.Limit) + require.Equal(t, "", db.TargetVersion) } func TestGetDriver(t *testing.T) { @@ -242,9 +244,11 @@ func TestWaitBeforeVerbose(t *testing.T) { `Applying: 20151129054053_test_migration.sql Rows affected: 1 Applying: 20200227231541_test_posts.sql +Rows affected: 0 +Applying: 20220607110405_test_category.sql Rows affected: 0`) require.Contains(t, output, - `Rolling back: 20200227231541_test_posts.sql + `Rolling back: 20220607110405_test_category.sql Rows affected: 0`) } @@ -291,6 +295,37 @@ func TestMigrate(t *testing.T) { } } +func TestMigrateToTarget(t *testing.T) { + for _, u := range testURLs() { + t.Run(u.Scheme, func(t *testing.T) { + db := newTestDB(t, u) + db.TargetVersion = "20151129054053" + drv, err := db.GetDriver() + require.NoError(t, err) + + // drop and recreate database + err = db.Drop() + require.NoError(t, err) + err = db.Create() + require.NoError(t, err) + + // migrate + err = db.Migrate() + require.NoError(t, err) + + // verify results + sqlDB, err := drv.Open() + require.NoError(t, err) + defer dbutil.MustClose(sqlDB) + + count := 0 + err = sqlDB.QueryRow(`select count(*) from schema_migrations`).Scan(&count) + require.NoError(t, err) + require.Equal(t, 1, count) + }) + } +} + func TestUp(t *testing.T) { for _, u := range testURLs() { t.Run(u.Scheme, func(t *testing.T) { @@ -350,13 +385,59 @@ func TestRollback(t *testing.T) { require.NoError(t, err) require.Equal(t, 1, count) - err = sqlDB.QueryRow("select count(*) from posts").Scan(&count) + err = sqlDB.QueryRow("select count(*) from categories").Scan(&count) require.Nil(t, err) // rollback err = db.Rollback() require.NoError(t, err) + // verify rollback + err = sqlDB.QueryRow("select count(*) from schema_migrations").Scan(&count) + require.NoError(t, err) + require.Equal(t, 2, count) + + err = sqlDB.QueryRow("select count(*) from categories").Scan(&count) + require.NotNil(t, err) + require.Regexp(t, "(does not exist|doesn't exist|no such table)", err.Error()) + }) + } +} + +func TestRollbackToTarget(t *testing.T) { + for _, u := range testURLs() { + t.Run(u.Scheme, func(t *testing.T) { + db := newTestDB(t, u) + drv, err := db.GetDriver() + require.NoError(t, err) + + // drop, recreate, and migrate database + err = db.Drop() + require.NoError(t, err) + err = db.Create() + require.NoError(t, err) + err = db.Migrate() + require.NoError(t, err) + + // verify migration + sqlDB, err := drv.Open() + require.NoError(t, err) + defer dbutil.MustClose(sqlDB) + + count := 0 + err = sqlDB.QueryRow(`select count(*) from schema_migrations + where version = '20151129054053'`).Scan(&count) + require.NoError(t, err) + require.Equal(t, 1, count) + + err = sqlDB.QueryRow("select count(*) from categories").Scan(&count) + require.Nil(t, err) + + // rollback + db.TargetVersion = "20151129054053" + err = db.Rollback() + require.NoError(t, err) + // verify rollback err = sqlDB.QueryRow("select count(*) from schema_migrations").Scan(&count) require.NoError(t, err) @@ -365,6 +446,60 @@ func TestRollback(t *testing.T) { err = sqlDB.QueryRow("select count(*) from posts").Scan(&count) require.NotNil(t, err) require.Regexp(t, "(does not exist|doesn't exist|no such table)", err.Error()) + + err = sqlDB.QueryRow("select count(*) from categories").Scan(&count) + require.NotNil(t, err) + require.Regexp(t, "(does not exist|doesn't exist|no such table)", err.Error()) + }) + } +} + +func TestRollbackToLimit(t *testing.T) { + for _, u := range testURLs() { + t.Run(u.Scheme, func(t *testing.T) { + db := newTestDB(t, u) + drv, err := db.GetDriver() + require.NoError(t, err) + + // drop, recreate, and migrate database + err = db.Drop() + require.NoError(t, err) + err = db.Create() + require.NoError(t, err) + err = db.Migrate() + require.NoError(t, err) + + // verify migration + sqlDB, err := drv.Open() + require.NoError(t, err) + defer dbutil.MustClose(sqlDB) + + count := 0 + err = sqlDB.QueryRow(`select count(*) from schema_migrations + where version = '20151129054053'`).Scan(&count) + require.NoError(t, err) + require.Equal(t, 1, count) + + err = sqlDB.QueryRow("select count(*) from categories").Scan(&count) + require.Nil(t, err) + + // rollback + db.Limit = 2 + err = db.Rollback() + require.NoError(t, err) + + // verify rollback + err = sqlDB.QueryRow("select count(*) from schema_migrations").Scan(&count) + require.NoError(t, err) + require.Equal(t, 1, count) + + err = sqlDB.QueryRow("select count(*) from posts").Scan(&count) + require.NotNil(t, err) + require.Regexp(t, "(does not exist|doesn't exist|no such table)", err.Error()) + + err = sqlDB.QueryRow("select count(*) from categories").Scan(&count) + require.NotNil(t, err) + require.Regexp(t, "(does not exist|doesn't exist|no such table)", err.Error()) }) } } @@ -390,7 +525,7 @@ func TestStatus(t *testing.T) { // two pending results, err := db.CheckMigrationsStatus(drv) require.NoError(t, err) - require.Len(t, results, 2) + require.Len(t, results, 3) require.False(t, results[0].Applied) require.False(t, results[1].Applied) @@ -398,12 +533,13 @@ func TestStatus(t *testing.T) { err = db.Migrate() require.NoError(t, err) - // two applied + // three applied results, err = db.CheckMigrationsStatus(drv) require.NoError(t, err) - require.Len(t, results, 2) + require.Len(t, results, 3) require.True(t, results[0].Applied) require.True(t, results[1].Applied) + require.True(t, results[2].Applied) // rollback last migration err = db.Rollback() @@ -412,9 +548,10 @@ func TestStatus(t *testing.T) { // one applied, one pending results, err = db.CheckMigrationsStatus(drv) require.NoError(t, err) - require.Len(t, results, 2) + require.Len(t, results, 3) require.True(t, results[0].Applied) - require.False(t, results[1].Applied) + require.True(t, results[1].Applied) + require.False(t, results[2].Applied) }) } } diff --git a/testdata/db/migrations/20220607110405_test_category.sql b/testdata/db/migrations/20220607110405_test_category.sql new file mode 100644 index 0000000..098ff16 --- /dev/null +++ b/testdata/db/migrations/20220607110405_test_category.sql @@ -0,0 +1,9 @@ +-- migrate:up +create table categories ( + id integer, + title varchar(50), + slug varchar(100) +); + +-- migrate:down +drop table categories;