From d855ee1adae0d2a8beb14c45fcad8696752fe84f Mon Sep 17 00:00:00 2001 From: Adrian Macneil Date: Mon, 22 Jan 2018 20:38:40 -0800 Subject: [PATCH] Add dump command (#23) Adds `dbmate dump` command to write the database schema to a file. The intent is for this file to be checked in to the codebase, similar to Rails' `schema.rb` (or `structure.sql`) file. This allows developers to share a single file documenting the database schema, and makes it considerably easier to review PRs which add (or change) migrations. The existing `up`, `migrate`, and `rollback` commands will automatically trigger a schema dump, unless `--no-dump-schema` is passed. Closes https://github.com/amacneil/dbmate/issues/5 --- .gitignore | 3 +- Dockerfile | 8 +++ cmd/dbmate/main.go | 30 ++++++++-- docker-compose.yml | 4 +- pkg/dbmate/db.go | 65 ++++++++++++++++++---- pkg/dbmate/db_test.go | 101 +++++++++++++++++++++++++++++++++- pkg/dbmate/driver.go | 1 + pkg/dbmate/mysql.go | 81 ++++++++++++++++++++++++--- pkg/dbmate/mysql_test.go | 39 +++++++++++++ pkg/dbmate/postgres.go | 45 ++++++++++++++- pkg/dbmate/postgres_test.go | 39 +++++++++++++ pkg/dbmate/sqlite.go | 44 ++++++++++++++- pkg/dbmate/sqlite_test.go | 34 ++++++++++++ pkg/dbmate/utils.go | 107 ++++++++++++++++++++++++++++++++++++ pkg/dbmate/utils_test.go | 11 ++++ 15 files changed, 578 insertions(+), 34 deletions(-) diff --git a/.gitignore b/.gitignore index 0e83ad3..df55cbf 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ -/dist .DS_Store +/dist +/testdata/db/schema.sql diff --git a/Dockerfile b/Dockerfile index 100c8df..137768c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,6 +4,14 @@ FROM golang:1.9 as build # required to force cgo (for sqlite driver) with cross compile ENV CGO_ENABLED 1 +# install database clients +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + mysql-client \ + postgresql-client \ + sqlite3 \ + && rm -rf /var/lib/apt/lists/* + # development dependencies RUN go get \ github.com/golang/lint/golint \ diff --git a/cmd/dbmate/main.go b/cmd/dbmate/main.go index 1b34e12..a5d1c05 100644 --- a/cmd/dbmate/main.go +++ b/cmd/dbmate/main.go @@ -30,15 +30,24 @@ func NewApp() *cli.App { app.Version = dbmate.Version app.Flags = []cli.Flag{ + cli.StringFlag{ + Name: "env, e", + Value: "DATABASE_URL", + Usage: "specify an environment variable containing the database URL", + }, cli.StringFlag{ Name: "migrations-dir, d", Value: dbmate.DefaultMigrationsDir, Usage: "specify the directory containing migration files", }, cli.StringFlag{ - Name: "env, e", - Value: "DATABASE_URL", - Usage: "specify an environment variable containing the database URL", + Name: "schema-file, s", + Value: dbmate.DefaultSchemaFile, + Usage: "specify the schema file location", + }, + cli.BoolFlag{ + Name: "no-dump-schema", + Usage: "don't update the schema file on migrate/rollback", }, } @@ -49,14 +58,14 @@ func NewApp() *cli.App { Usage: "Generate a new migration file", Action: action(func(db *dbmate.DB, c *cli.Context) error { name := c.Args().First() - return db.New(name) + return db.NewMigration(name) }), }, { Name: "up", Usage: "Create database (if necessary) and migrate to the latest version", Action: action(func(db *dbmate.DB, c *cli.Context) error { - return db.Up() + return db.CreateAndMigrate() }), }, { @@ -88,6 +97,13 @@ func NewApp() *cli.App { return db.Rollback() }), }, + { + Name: "dump", + Usage: "Write the database schema to disk", + Action: action(func(db *dbmate.DB, c *cli.Context) error { + return db.DumpSchema() + }), + }, } return app @@ -111,8 +127,10 @@ func action(f func(*dbmate.DB, *cli.Context) error) cli.ActionFunc { if err != nil { return err } - db := dbmate.NewDB(u) + db := dbmate.New(u) + db.AutoDumpSchema = !c.GlobalBool("no-dump-schema") db.MigrationsDir = c.GlobalString("migrations-dir") + db.SchemaFile = c.GlobalString("schema-file") return f(db, c) } diff --git a/docker-compose.yml b/docker-compose.yml index 38aa218..f9ecc70 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,13 +13,13 @@ services: condition: service_healthy mysql: - image: mysql + image: mysql:5.7 environment: MYSQL_ROOT_PASSWORD: root healthcheck: test: ["CMD", "mysqladmin", "status", "-proot"] postgres: - image: postgres + image: postgres:9.6 healthcheck: test: ["CMD", "pg_isready"] diff --git a/pkg/dbmate/db.go b/pkg/dbmate/db.go index 8c7271c..e015c08 100644 --- a/pkg/dbmate/db.go +++ b/pkg/dbmate/db.go @@ -15,17 +15,24 @@ import ( // DefaultMigrationsDir specifies default directory to find migration files var DefaultMigrationsDir = "./db/migrations" +// DefaultSchemaFile specifies default location for schema.sql +var DefaultSchemaFile = "./db/schema.sql" + // DB allows dbmate actions to be performed on a specified database type DB struct { - DatabaseURL *url.URL - MigrationsDir string + AutoDumpSchema bool + DatabaseURL *url.URL + MigrationsDir string + SchemaFile string } -// NewDB initializes a new dbmate database -func NewDB(databaseURL *url.URL) *DB { +// New initializes a new dbmate database +func New(databaseURL *url.URL) *DB { return &DB{ - DatabaseURL: databaseURL, - MigrationsDir: DefaultMigrationsDir, + AutoDumpSchema: true, + DatabaseURL: databaseURL, + MigrationsDir: DefaultMigrationsDir, + SchemaFile: DefaultSchemaFile, } } @@ -34,8 +41,8 @@ func (db *DB) GetDriver() (Driver, error) { return GetDriver(db.DatabaseURL.Scheme) } -// Up creates the database (if necessary) and runs migrations -func (db *DB) Up() error { +// CreateAndMigrate creates the database (if necessary) and runs migrations +func (db *DB) CreateAndMigrate() error { drv, err := db.GetDriver() if err != nil { return err @@ -75,10 +82,34 @@ func (db *DB) Drop() error { return drv.DropDatabase(db.DatabaseURL) } +// DumpSchema writes the current database schema to a file +func (db *DB) DumpSchema() error { + drv, sqlDB, err := db.openDatabaseForMigration() + if err != nil { + return err + } + defer mustClose(sqlDB) + + schema, err := drv.DumpSchema(db.DatabaseURL, sqlDB) + if err != nil { + return err + } + + fmt.Printf("Writing: %s\n", db.SchemaFile) + + // ensure schema directory exists + if err = ensureDir(filepath.Dir(db.SchemaFile)); err != nil { + return err + } + + // write schema to file + return ioutil.WriteFile(db.SchemaFile, schema, 0644) +} + const migrationTemplate = "-- migrate:up\n\n\n-- migrate:down\n\n" -// New creates a new migration file -func (db *DB) New(name string) error { +// NewMigration creates a new migration file +func (db *DB) NewMigration(name string) error { // new migration name timestamp := time.Now().UTC().Format("20060102150405") if name == "" { @@ -87,8 +118,8 @@ func (db *DB) New(name string) error { name = fmt.Sprintf("%s_%s.sql", timestamp, name) // create migrations dir if missing - if err := os.MkdirAll(db.MigrationsDir, 0755); err != nil { - return fmt.Errorf("unable to create directory `%s`", db.MigrationsDir) + if err := ensureDir(db.MigrationsDir); err != nil { + return err } // check file does not already exist @@ -203,6 +234,11 @@ func (db *DB) Migrate() error { } + // automatically update schema file, silence errors + if db.AutoDumpSchema { + _ = db.DumpSchema() + } + return nil } @@ -340,5 +376,10 @@ func (db *DB) Rollback() error { return err } + // automatically update schema file, silence errors + if db.AutoDumpSchema { + _ = db.DumpSchema() + } + return nil } diff --git a/pkg/dbmate/db_test.go b/pkg/dbmate/db_test.go index 2f0d195..6cc2b91 100644 --- a/pkg/dbmate/db_test.go +++ b/pkg/dbmate/db_test.go @@ -1,6 +1,7 @@ package dbmate import ( + "io/ioutil" "net/url" "os" "path/filepath" @@ -23,7 +24,103 @@ func newTestDB(t *testing.T, u *url.URL) *DB { require.Nil(t, err) } - return NewDB(u) + db := New(u) + db.AutoDumpSchema = false + + return db +} + +func TestNew(t *testing.T) { + u := postgresTestURL(t) + db := New(u) + require.True(t, db.AutoDumpSchema) + require.Equal(t, u.String(), db.DatabaseURL.String()) + require.Equal(t, "./db/migrations", db.MigrationsDir) + require.Equal(t, "./db/schema.sql", db.SchemaFile) +} + +func TestDumpSchema(t *testing.T) { + u := postgresTestURL(t) + db := newTestDB(t, u) + + // create custom schema file directory + dir, err := ioutil.TempDir("", "dbmate") + require.Nil(t, err) + defer func() { + err := os.RemoveAll(dir) + require.Nil(t, err) + }() + + // create schema.sql in subdirectory to test creating directory + db.SchemaFile = filepath.Join(dir, "/schema/schema.sql") + + // drop database + err = db.Drop() + require.Nil(t, err) + + // create and migrate + err = db.CreateAndMigrate() + require.Nil(t, err) + + // schema.sql should not exist + _, err = os.Stat(db.SchemaFile) + require.True(t, os.IsNotExist(err)) + + // dump schema + err = db.DumpSchema() + require.Nil(t, err) + + // verify schema + schema, err := ioutil.ReadFile(db.SchemaFile) + require.Nil(t, err) + require.Contains(t, string(schema), "-- PostgreSQL database dump") +} + +func TestAutoDumpSchema(t *testing.T) { + u := postgresTestURL(t) + db := newTestDB(t, u) + db.AutoDumpSchema = true + + // create custom schema file directory + dir, err := ioutil.TempDir("", "dbmate") + require.Nil(t, err) + defer func() { + err := os.RemoveAll(dir) + require.Nil(t, err) + }() + + // create schema.sql in subdirectory to test creating directory + db.SchemaFile = filepath.Join(dir, "/schema/schema.sql") + + // drop database + err = db.Drop() + require.Nil(t, err) + + // schema.sql should not exist + _, err = os.Stat(db.SchemaFile) + require.True(t, os.IsNotExist(err)) + + // create and migrate + err = db.CreateAndMigrate() + require.Nil(t, err) + + // verify schema + schema, err := ioutil.ReadFile(db.SchemaFile) + require.Nil(t, err) + require.Contains(t, string(schema), "-- PostgreSQL database dump") + + // remove schema + err = os.Remove(db.SchemaFile) + require.Nil(t, err) + + // rollback + err = db.Rollback() + require.Nil(t, err) + + // schema should be recreated + schema, err = ioutil.ReadFile(db.SchemaFile) + require.Nil(t, err) + require.Contains(t, string(schema), "-- PostgreSQL database dump") } func testURLs(t *testing.T) []*url.URL { @@ -77,7 +174,7 @@ func testUpURL(t *testing.T, u *url.URL) { require.Nil(t, err) // create and migrate - err = db.Up() + err = db.CreateAndMigrate() require.Nil(t, err) // verify results diff --git a/pkg/dbmate/driver.go b/pkg/dbmate/driver.go index f9bec1d..f1dbe09 100644 --- a/pkg/dbmate/driver.go +++ b/pkg/dbmate/driver.go @@ -12,6 +12,7 @@ type Driver interface { DatabaseExists(*url.URL) (bool, error) CreateDatabase(*url.URL) error DropDatabase(*url.URL) error + DumpSchema(*url.URL, *sql.DB) ([]byte, error) CreateMigrationsTable(*sql.DB) error SelectMigrations(*sql.DB, int) (map[string]bool, error) InsertMigration(Transaction, string) error diff --git a/pkg/dbmate/mysql.go b/pkg/dbmate/mysql.go index 12c168c..faa3f26 100644 --- a/pkg/dbmate/mysql.go +++ b/pkg/dbmate/mysql.go @@ -1,6 +1,7 @@ package dbmate import ( + "bytes" "database/sql" "fmt" "net/url" @@ -46,7 +47,7 @@ func (drv MySQLDriver) openRootDB(u *url.URL) (*sql.DB, error) { return drv.Open(&rootURL) } -func quoteIdentifier(str string) string { +func mysqlQuoteIdentifier(str string) string { str = strings.Replace(str, "`", "\\`", -1) return fmt.Sprintf("`%s`", str) @@ -64,7 +65,7 @@ func (drv MySQLDriver) CreateDatabase(u *url.URL) error { defer mustClose(db) _, err = db.Exec(fmt.Sprintf("create database %s", - quoteIdentifier(name))) + mysqlQuoteIdentifier(name))) return err } @@ -81,11 +82,77 @@ func (drv MySQLDriver) DropDatabase(u *url.URL) error { defer mustClose(db) _, err = db.Exec(fmt.Sprintf("drop database if exists %s", - quoteIdentifier(name))) + mysqlQuoteIdentifier(name))) return err } +func mysqldumpArgs(u *url.URL) []string { + // generate CLI arguments + args := []string{"--opt", "--routines", "--no-data", + "--skip-dump-date", "--skip-add-drop-table"} + + if hostname := u.Hostname(); hostname != "" { + args = append(args, "--host="+hostname) + } + if port := u.Port(); port != "" { + args = append(args, "--port="+port) + } + if username := u.User.Username(); username != "" { + args = append(args, "--user="+username) + } + // mysql recommands against using environment variables to supply password + // https://dev.mysql.com/doc/refman/5.7/en/password-security-user.html + if password, set := u.User.Password(); set { + args = append(args, "--password="+password) + } + + // add database name + args = append(args, strings.TrimLeft(u.Path, "/")) + + return args +} + +func mysqlSchemaMigrationsDump(db *sql.DB) ([]byte, error) { + // load applied migrations + migrations, err := queryColumn(db, + "select quote(version) from schema_migrations order by version asc") + if err != nil { + return nil, err + } + + // build schema_migrations table data + var buf bytes.Buffer + buf.WriteString("\n--\n-- Dbmate schema migrations\n--\n\n" + + "LOCK TABLES `schema_migrations` WRITE;\n") + + if len(migrations) > 0 { + buf.WriteString("INSERT INTO `schema_migrations` (version) VALUES\n (" + + strings.Join(migrations, "),\n (") + + ");\n") + } + + buf.WriteString("UNLOCK TABLES;\n") + + return buf.Bytes(), nil +} + +// DumpSchema returns the current database schema +func (drv MySQLDriver) DumpSchema(u *url.URL, db *sql.DB) ([]byte, error) { + schema, err := runCommand("mysqldump", mysqldumpArgs(u)...) + if err != nil { + return nil, err + } + + migrations, err := mysqlSchemaMigrationsDump(db) + if err != nil { + return nil, err + } + + schema = append(schema, migrations...) + return trimLeadingSQLComments(schema) +} + // DatabaseExists determines whether the database exists func (drv MySQLDriver) DatabaseExists(u *url.URL) (bool, error) { name := databaseName(u) @@ -97,8 +164,8 @@ func (drv MySQLDriver) DatabaseExists(u *url.URL) (bool, error) { defer mustClose(db) exists := false - err = db.QueryRow(`select true from information_schema.schemata - where schema_name = ?`, name).Scan(&exists) + err = db.QueryRow("select true from information_schema.schemata "+ + "where schema_name = ?", name).Scan(&exists) if err == sql.ErrNoRows { return false, nil } @@ -108,8 +175,8 @@ func (drv MySQLDriver) DatabaseExists(u *url.URL) (bool, error) { // CreateMigrationsTable creates the schema_migrations table func (drv MySQLDriver) CreateMigrationsTable(db *sql.DB) error { - _, err := db.Exec(`create table if not exists schema_migrations ( - version varchar(255) primary key)`) + _, err := db.Exec("create table if not exists schema_migrations " + + "(version varchar(255) primary key)") return err } diff --git a/pkg/dbmate/mysql_test.go b/pkg/dbmate/mysql_test.go index 0dbc452..f04f9fd 100644 --- a/pkg/dbmate/mysql_test.go +++ b/pkg/dbmate/mysql_test.go @@ -90,6 +90,45 @@ func TestMySQLCreateDropDatabase(t *testing.T) { }() } +func TestMySQLDumpSchema(t *testing.T) { + drv := MySQLDriver{} + u := mySQLTestURL(t) + + // prepare database + db := prepTestMySQLDB(t) + defer mustClose(db) + err := drv.CreateMigrationsTable(db) + require.Nil(t, err) + + // insert migration + err = drv.InsertMigration(db, "abc1") + require.Nil(t, err) + err = drv.InsertMigration(db, "abc2") + require.Nil(t, err) + + // DumpSchema should return schema + schema, err := drv.DumpSchema(u, db) + require.Nil(t, err) + require.Contains(t, string(schema), "CREATE TABLE `schema_migrations`") + require.Contains(t, string(schema), "\n-- Dump completed\n\n"+ + "--\n"+ + "-- Dbmate schema migrations\n"+ + "--\n\n"+ + "LOCK TABLES `schema_migrations` WRITE;\n"+ + "INSERT INTO `schema_migrations` (version) VALUES\n"+ + " ('abc1'),\n"+ + " ('abc2');\n"+ + "UNLOCK TABLES;\n") + + // DumpSchema should return error if command fails + u.Path = "/fakedb" + schema, err = drv.DumpSchema(u, db) + require.Nil(t, schema) + require.NotNil(t, err) + require.Equal(t, "mysqldump: Got error: 1049: \"Unknown database 'fakedb'\" "+ + "when selecting the database", err.Error()) +} + func TestMySQLDatabaseExists(t *testing.T) { drv := MySQLDriver{} u := mySQLTestURL(t) diff --git a/pkg/dbmate/postgres.go b/pkg/dbmate/postgres.go index 5b20e67..a21ae0a 100644 --- a/pkg/dbmate/postgres.go +++ b/pkg/dbmate/postgres.go @@ -1,9 +1,11 @@ package dbmate import ( + "bytes" "database/sql" "fmt" "net/url" + "strings" "github.com/lib/pq" ) @@ -59,6 +61,45 @@ func (drv PostgresDriver) DropDatabase(u *url.URL) error { return err } +func postgresSchemaMigrationsDump(db *sql.DB) ([]byte, error) { + // load applied migrations + migrations, err := queryColumn(db, + "select quote_literal(version) from schema_migrations order by version asc") + if err != nil { + return nil, err + } + + // build schema_migrations table data + var buf bytes.Buffer + buf.WriteString("\n--\n-- Dbmate schema migrations\n--\n\n") + + if len(migrations) > 0 { + buf.WriteString("INSERT INTO schema_migrations (version) VALUES\n (" + + strings.Join(migrations, "),\n (") + + ");\n") + } + + return buf.Bytes(), nil +} + +// DumpSchema returns the current database schema +func (drv PostgresDriver) DumpSchema(u *url.URL, db *sql.DB) ([]byte, error) { + // load schema + schema, err := runCommand("pg_dump", "--format=plain", "--encoding=UTF8", + "--schema-only", "--no-privileges", "--no-owner", u.String()) + if err != nil { + return nil, err + } + + migrations, err := postgresSchemaMigrationsDump(db) + if err != nil { + return nil, err + } + + schema = append(schema, migrations...) + return trimLeadingSQLComments(schema) +} + // DatabaseExists determines whether the database exists func (drv PostgresDriver) DatabaseExists(u *url.URL) (bool, error) { name := databaseName(u) @@ -81,8 +122,8 @@ func (drv PostgresDriver) DatabaseExists(u *url.URL) (bool, error) { // CreateMigrationsTable creates the schema_migrations table func (drv PostgresDriver) CreateMigrationsTable(db *sql.DB) error { - _, err := db.Exec(`create table if not exists schema_migrations ( - version varchar(255) primary key)`) + _, err := db.Exec("create table if not exists schema_migrations " + + "(version varchar(255) primary key)") return err } diff --git a/pkg/dbmate/postgres_test.go b/pkg/dbmate/postgres_test.go index 83a6d31..12c9d79 100644 --- a/pkg/dbmate/postgres_test.go +++ b/pkg/dbmate/postgres_test.go @@ -72,6 +72,45 @@ func TestPostgresCreateDropDatabase(t *testing.T) { }() } +func TestPostgresDumpSchema(t *testing.T) { + drv := PostgresDriver{} + u := postgresTestURL(t) + + // prepare database + db := prepTestPostgresDB(t) + defer mustClose(db) + err := drv.CreateMigrationsTable(db) + require.Nil(t, err) + + // insert migration + err = drv.InsertMigration(db, "abc1") + require.Nil(t, err) + err = drv.InsertMigration(db, "abc2") + require.Nil(t, err) + + // DumpSchema should return schema + schema, err := drv.DumpSchema(u, db) + require.Nil(t, err) + require.Contains(t, string(schema), "CREATE TABLE schema_migrations") + require.Contains(t, string(schema), "\n--\n"+ + "-- PostgreSQL database dump complete\n"+ + "--\n\n\n"+ + "--\n"+ + "-- Dbmate schema migrations\n"+ + "--\n\n"+ + "INSERT INTO schema_migrations (version) VALUES\n"+ + " ('abc1'),\n"+ + " ('abc2');\n") + + // DumpSchema should return error if command fails + u.Path = "/fakedb" + schema, err = drv.DumpSchema(u, db) + require.Nil(t, schema) + require.NotNil(t, err) + require.Equal(t, "pg_dump: [archiver (db)] connection to database \"fakedb\" failed: "+ + "FATAL: database \"fakedb\" does not exist", err.Error()) +} + func TestPostgresDatabaseExists(t *testing.T) { drv := PostgresDriver{} u := postgresTestURL(t) diff --git a/pkg/dbmate/sqlite.go b/pkg/dbmate/sqlite.go index 27694d3..616286f 100644 --- a/pkg/dbmate/sqlite.go +++ b/pkg/dbmate/sqlite.go @@ -1,11 +1,13 @@ package dbmate import ( + "bytes" "database/sql" "fmt" "net/url" "os" "regexp" + "strings" _ "github.com/mattn/go-sqlite3" // sqlite driver for database/sql ) @@ -56,6 +58,44 @@ func (drv SQLiteDriver) DropDatabase(u *url.URL) error { return os.Remove(path) } +func sqliteSchemaMigrationsDump(db *sql.DB) ([]byte, error) { + // load applied migrations + migrations, err := queryColumn(db, + "select quote(version) from schema_migrations order by version asc") + if err != nil { + return nil, err + } + + // build schema_migrations table data + var buf bytes.Buffer + buf.WriteString("-- Dbmate schema migrations\n") + + if len(migrations) > 0 { + buf.WriteString("INSERT INTO schema_migrations (version) VALUES\n (" + + strings.Join(migrations, "),\n (") + + ");\n") + } + + return buf.Bytes(), nil +} + +// DumpSchema returns the current database schema +func (drv SQLiteDriver) DumpSchema(u *url.URL, db *sql.DB) ([]byte, error) { + path := sqlitePath(u) + schema, err := runCommand("sqlite3", path, ".schema") + if err != nil { + return nil, err + } + + migrations, err := sqliteSchemaMigrationsDump(db) + if err != nil { + return nil, err + } + + schema = append(schema, migrations...) + return trimLeadingSQLComments(schema) +} + // DatabaseExists determines whether the database exists func (drv SQLiteDriver) DatabaseExists(u *url.URL) (bool, error) { _, err := os.Stat(sqlitePath(u)) @@ -71,8 +111,8 @@ func (drv SQLiteDriver) DatabaseExists(u *url.URL) (bool, error) { // CreateMigrationsTable creates the schema_migrations table func (drv SQLiteDriver) CreateMigrationsTable(db *sql.DB) error { - _, err := db.Exec(`create table if not exists schema_migrations ( - version varchar(255) primary key)`) + _, err := db.Exec("create table if not exists schema_migrations " + + "(version varchar(255) primary key)") return err } diff --git a/pkg/dbmate/sqlite_test.go b/pkg/dbmate/sqlite_test.go index 6e59283..65566d9 100644 --- a/pkg/dbmate/sqlite_test.go +++ b/pkg/dbmate/sqlite_test.go @@ -61,6 +61,40 @@ func TestSQLiteCreateDropDatabase(t *testing.T) { require.Equal(t, true, os.IsNotExist(err)) } +func TestSQLiteDumpSchema(t *testing.T) { + drv := SQLiteDriver{} + u := sqliteTestURL(t) + + // prepare database + db := prepTestSQLiteDB(t) + defer mustClose(db) + err := drv.CreateMigrationsTable(db) + require.Nil(t, err) + + // insert migration + err = drv.InsertMigration(db, "abc1") + require.Nil(t, err) + err = drv.InsertMigration(db, "abc2") + require.Nil(t, err) + + // DumpSchema should return schema + schema, err := drv.DumpSchema(u, db) + require.Nil(t, err) + require.Contains(t, string(schema), "CREATE TABLE schema_migrations") + require.Contains(t, string(schema), ");\n-- Dbmate schema migrations\n"+ + "INSERT INTO schema_migrations (version) VALUES\n"+ + " ('abc1'),\n"+ + " ('abc2');\n") + + // DumpSchema should return error if command fails + u.Path = "/." + schema, err = drv.DumpSchema(u, db) + require.Nil(t, schema) + require.NotNil(t, err) + require.Equal(t, "Error: unable to open database \".\": unable to open database file", + err.Error()) +} + func TestSQLiteDatabaseExists(t *testing.T) { drv := SQLiteDriver{} u := sqliteTestURL(t) diff --git a/pkg/dbmate/utils.go b/pkg/dbmate/utils.go index 1da4bfe..07352f3 100644 --- a/pkg/dbmate/utils.go +++ b/pkg/dbmate/utils.go @@ -1,8 +1,17 @@ package dbmate import ( + "bufio" + "bytes" + "database/sql" + "errors" + "fmt" "io" "net/url" + "os" + "os/exec" + "strings" + "unicode" ) // databaseName returns the database name from a URL @@ -15,8 +24,106 @@ func databaseName(u *url.URL) string { return name } +// mustClose ensures a stream is closed func mustClose(c io.Closer) { if err := c.Close(); err != nil { panic(err) } } + +// ensureDir creates a directory if it does not already exist +func ensureDir(dir string) error { + if err := os.MkdirAll(dir, 0755); err != nil { + return fmt.Errorf("unable to create directory `%s`", dir) + } + + return nil +} + +// runCommand runs a command and returns the stdout if successful +func runCommand(name string, args ...string) ([]byte, error) { + var stdout, stderr bytes.Buffer + cmd := exec.Command(name, args...) + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + if err := cmd.Run(); err != nil { + // return stderr if available + if s := strings.TrimSpace(stderr.String()); s != "" { + return nil, errors.New(s) + } + + // otherwise return error + return nil, err + } + + // return stdout + return stdout.Bytes(), nil +} + +// trimLeadingSQLComments removes sql comments and blank lines from the beginning of text +// generally when performing sql dumps these contain host-specific information such as +// client/server version numbers +func trimLeadingSQLComments(data []byte) ([]byte, error) { + // create decent size buffer + out := bytes.NewBuffer(make([]byte, 0, len(data))) + + // iterate over sql lines + preamble := true + scanner := bufio.NewScanner(bytes.NewReader(data)) + for scanner.Scan() { + // we read bytes directly for premature performance optimization + line := scanner.Bytes() + + if preamble && (len(line) == 0 || bytes.Equal(line[0:2], []byte("--"))) { + // header section, skip this line in output buffer + continue + } + + // header section is over + preamble = false + + // trim trailing whitespace + line = bytes.TrimRightFunc(line, unicode.IsSpace) + + // copy bytes to output buffer + if _, err := out.Write(line); err != nil { + return nil, err + } + if _, err := out.WriteString("\n"); err != nil { + return nil, err + } + } + if err := scanner.Err(); err != nil { + return nil, err + } + + return out.Bytes(), nil +} + +// queryColumn runs a SQL statement and returns a slice of strings +// it is assumed that the statement returns only one column +// e.g. schema_migrations table +func queryColumn(db *sql.DB, query string) ([]string, error) { + rows, err := db.Query(query) + if err != nil { + return nil, err + } + defer mustClose(rows) + + // read into slice + var result []string + for rows.Next() { + var v string + if err := rows.Scan(&v); err != nil { + return nil, err + } + + result = append(result, v) + } + if err = rows.Err(); err != nil { + return nil, err + } + + return result, nil +} diff --git a/pkg/dbmate/utils_test.go b/pkg/dbmate/utils_test.go index b53de0b..fc9aa60 100644 --- a/pkg/dbmate/utils_test.go +++ b/pkg/dbmate/utils_test.go @@ -22,3 +22,14 @@ func TestDatabaseName_Empty(t *testing.T) { name := databaseName(u) require.Equal(t, "", name) } + +func TestTrimLeadingSQLComments(t *testing.T) { + in := "--\n" + + "-- foo\n\n" + + "-- bar\n\n" + + "real stuff\n" + + "-- end\n" + out, err := trimLeadingSQLComments([]byte(in)) + require.Nil(t, err) + require.Equal(t, "real stuff\n-- end\n", string(out)) +}