Refactor drivers into separate packages (#179)

`dbmate` package was starting to get a bit polluted. This PR migrates each driver into a separate package, with clean separation between each.

In addition:

* Drivers are now initialized with a URL, avoiding the need to pass `*url.URL` to every method
* Sqlite supports a cleaner syntax for relative paths
* Driver tests now load their test URL from environment variables

Public API of `dbmate` package has not changed (no changes to `main` package).
This commit is contained in:
Adrian Macneil 2020-11-19 15:04:42 +13:00 committed by GitHub
parent c907c3f5c6
commit 61771e386d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
23 changed files with 1195 additions and 1078 deletions

287
pkg/driver/mysql/mysql.go Normal file
View file

@ -0,0 +1,287 @@
package mysql
import (
"bytes"
"database/sql"
"fmt"
"net/url"
"strings"
"github.com/amacneil/dbmate/pkg/dbmate"
"github.com/amacneil/dbmate/pkg/dbutil"
_ "github.com/go-sql-driver/mysql" // database/sql driver
)
func init() {
dbmate.RegisterDriver(NewDriver, "mysql")
}
// Driver provides top level database functions
type Driver struct {
migrationsTableName string
databaseURL *url.URL
}
// NewDriver initializes the driver
func NewDriver(config dbmate.DriverConfig) dbmate.Driver {
return &Driver{
migrationsTableName: config.MigrationsTableName,
databaseURL: config.DatabaseURL,
}
}
func connectionString(u *url.URL) string {
query := u.Query()
query.Set("multiStatements", "true")
host := u.Host
protocol := "tcp"
if query.Get("socket") != "" {
protocol = "unix"
host = query.Get("socket")
query.Del("socket")
} else if u.Port() == "" {
// set default port
host = fmt.Sprintf("%s:3306", host)
}
// Get decoded user:pass
userPassEncoded := u.User.String()
userPass, _ := url.QueryUnescape(userPassEncoded)
// Build DSN w/ user:pass percent-decoded
normalizedString := ""
if userPass != "" { // user:pass can be empty
normalizedString = userPass + "@"
}
// connection string format required by go-sql-driver/mysql
normalizedString = fmt.Sprintf("%s%s(%s)%s?%s", normalizedString,
protocol, host, u.Path, query.Encode())
return normalizedString
}
// Open creates a new database connection
func (drv *Driver) Open() (*sql.DB, error) {
return sql.Open("mysql", connectionString(drv.databaseURL))
}
func (drv *Driver) openRootDB() (*sql.DB, error) {
// clone databaseURL
rootURL, err := url.Parse(drv.databaseURL.String())
if err != nil {
return nil, err
}
// connect to no particular database
rootURL.Path = "/"
return sql.Open("mysql", connectionString(rootURL))
}
func (drv *Driver) quoteIdentifier(str string) string {
str = strings.Replace(str, "`", "\\`", -1)
return fmt.Sprintf("`%s`", str)
}
// CreateDatabase creates the specified database
func (drv *Driver) CreateDatabase() error {
name := dbutil.DatabaseName(drv.databaseURL)
fmt.Printf("Creating: %s\n", name)
db, err := drv.openRootDB()
if err != nil {
return err
}
defer dbutil.MustClose(db)
_, err = db.Exec(fmt.Sprintf("create database %s",
drv.quoteIdentifier(name)))
return err
}
// DropDatabase drops the specified database (if it exists)
func (drv *Driver) DropDatabase() error {
name := dbutil.DatabaseName(drv.databaseURL)
fmt.Printf("Dropping: %s\n", name)
db, err := drv.openRootDB()
if err != nil {
return err
}
defer dbutil.MustClose(db)
_, err = db.Exec(fmt.Sprintf("drop database if exists %s",
drv.quoteIdentifier(name)))
return err
}
func (drv *Driver) mysqldumpArgs() []string {
// generate CLI arguments
args := []string{"--opt", "--routines", "--no-data",
"--skip-dump-date", "--skip-add-drop-table"}
if hostname := drv.databaseURL.Hostname(); hostname != "" {
args = append(args, "--host="+hostname)
}
if port := drv.databaseURL.Port(); port != "" {
args = append(args, "--port="+port)
}
if username := drv.databaseURL.User.Username(); username != "" {
args = append(args, "--user="+username)
}
// mysql recommends against using environment variables to supply password
// https://dev.mysql.com/doc/refman/5.7/en/password-security-user.html
if password, set := drv.databaseURL.User.Password(); set {
args = append(args, "--password="+password)
}
// add database name
args = append(args, dbutil.DatabaseName(drv.databaseURL))
return args
}
func (drv *Driver) schemaMigrationsDump(db *sql.DB) ([]byte, error) {
migrationsTable := drv.quotedMigrationsTableName()
// load applied migrations
migrations, err := dbutil.QueryColumn(db,
fmt.Sprintf("select quote(version) from %s order by version asc", migrationsTable))
if err != nil {
return nil, err
}
// build schema_migrations table data
var buf bytes.Buffer
buf.WriteString("\n--\n-- Dbmate schema migrations\n--\n\n" +
fmt.Sprintf("LOCK TABLES %s WRITE;\n", migrationsTable))
if len(migrations) > 0 {
buf.WriteString(
fmt.Sprintf("INSERT INTO %s (version) VALUES\n (", migrationsTable) +
strings.Join(migrations, "),\n (") +
");\n")
}
buf.WriteString("UNLOCK TABLES;\n")
return buf.Bytes(), nil
}
// DumpSchema returns the current database schema
func (drv *Driver) DumpSchema(db *sql.DB) ([]byte, error) {
schema, err := dbutil.RunCommand("mysqldump", drv.mysqldumpArgs()...)
if err != nil {
return nil, err
}
migrations, err := drv.schemaMigrationsDump(db)
if err != nil {
return nil, err
}
schema = append(schema, migrations...)
return dbutil.TrimLeadingSQLComments(schema)
}
// DatabaseExists determines whether the database exists
func (drv *Driver) DatabaseExists() (bool, error) {
name := dbutil.DatabaseName(drv.databaseURL)
db, err := drv.openRootDB()
if err != nil {
return false, err
}
defer dbutil.MustClose(db)
exists := false
err = db.QueryRow("select true from information_schema.schemata "+
"where schema_name = ?", name).Scan(&exists)
if err == sql.ErrNoRows {
return false, nil
}
return exists, err
}
// CreateMigrationsTable creates the schema_migrations table
func (drv *Driver) CreateMigrationsTable(db *sql.DB) error {
_, err := db.Exec(fmt.Sprintf("create table if not exists %s "+
"(version varchar(255) primary key) character set latin1 collate latin1_bin",
drv.quotedMigrationsTableName()))
return err
}
// SelectMigrations returns a list of applied migrations
// with an optional limit (in descending order)
func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, error) {
query := fmt.Sprintf("select version from %s order by version desc", drv.quotedMigrationsTableName())
if limit >= 0 {
query = fmt.Sprintf("%s limit %d", query, limit)
}
rows, err := db.Query(query)
if err != nil {
return nil, err
}
defer dbutil.MustClose(rows)
migrations := map[string]bool{}
for rows.Next() {
var version string
if err := rows.Scan(&version); err != nil {
return nil, err
}
migrations[version] = true
}
if err = rows.Err(); err != nil {
return nil, err
}
return migrations, nil
}
// InsertMigration adds a new migration record
func (drv *Driver) InsertMigration(db dbutil.Transaction, version string) error {
_, err := db.Exec(
fmt.Sprintf("insert into %s (version) values (?)", drv.quotedMigrationsTableName()),
version)
return err
}
// DeleteMigration removes a migration record
func (drv *Driver) DeleteMigration(db dbutil.Transaction, version string) error {
_, err := db.Exec(
fmt.Sprintf("delete from %s where version = ?", drv.quotedMigrationsTableName()),
version)
return err
}
// Ping verifies a connection to the database server. It does not verify whether the
// specified database exists.
func (drv *Driver) Ping() error {
db, err := drv.openRootDB()
if err != nil {
return err
}
defer dbutil.MustClose(db)
return db.Ping()
}
func (drv *Driver) quotedMigrationsTableName() string {
return drv.quoteIdentifier(drv.migrationsTableName)
}

View file

@ -0,0 +1,342 @@
package mysql
import (
"database/sql"
"net/url"
"os"
"testing"
"github.com/amacneil/dbmate/pkg/dbmate"
"github.com/amacneil/dbmate/pkg/dbutil"
"github.com/stretchr/testify/require"
)
func testMySQLDriver(t *testing.T) *Driver {
u := dbutil.MustParseURL(os.Getenv("MYSQL_TEST_URL"))
drv, err := dbmate.New(u).GetDriver()
require.NoError(t, err)
return drv.(*Driver)
}
func prepTestMySQLDB(t *testing.T) *sql.DB {
drv := testMySQLDriver(t)
// drop any existing database
err := drv.DropDatabase()
require.NoError(t, err)
// create database
err = drv.CreateDatabase()
require.NoError(t, err)
// connect database
db, err := drv.Open()
require.NoError(t, err)
return db
}
func TestGetDriver(t *testing.T) {
db := dbmate.New(dbutil.MustParseURL("mysql://"))
drvInterface, err := db.GetDriver()
require.NoError(t, err)
// driver should have URL and default migrations table set
drv, ok := drvInterface.(*Driver)
require.True(t, ok)
require.Equal(t, db.DatabaseURL.String(), drv.databaseURL.String())
require.Equal(t, "schema_migrations", drv.migrationsTableName)
}
func TestConnectionString(t *testing.T) {
t.Run("defaults", func(t *testing.T) {
u, err := url.Parse("mysql://host/foo")
require.NoError(t, err)
require.Equal(t, "", u.Port())
s := connectionString(u)
require.Equal(t, "tcp(host:3306)/foo?multiStatements=true", s)
})
t.Run("custom", func(t *testing.T) {
u, err := url.Parse("mysql://bob:secret@host:123/foo?flag=on")
require.NoError(t, err)
require.Equal(t, "123", u.Port())
s := connectionString(u)
require.Equal(t, "bob:secret@tcp(host:123)/foo?flag=on&multiStatements=true", s)
})
t.Run("special chars", func(t *testing.T) {
u, err := url.Parse("mysql://duhfsd7s:123!@123!@@host:123/foo?flag=on")
require.NoError(t, err)
require.Equal(t, "123", u.Port())
s := connectionString(u)
require.Equal(t, "duhfsd7s:123!@123!@@tcp(host:123)/foo?flag=on&multiStatements=true", s)
})
t.Run("socket", func(t *testing.T) {
// test with no user/pass
u, err := url.Parse("mysql:///foo?socket=/var/run/mysqld/mysqld.sock&flag=on")
require.NoError(t, err)
require.Equal(t, "", u.Host)
s := connectionString(u)
require.Equal(t, "unix(/var/run/mysqld/mysqld.sock)/foo?flag=on&multiStatements=true", s)
// test with user/pass
u, err = url.Parse("mysql://bob:secret@fakehost/foo?socket=/var/run/mysqld/mysqld.sock&flag=on")
require.NoError(t, err)
s = connectionString(u)
require.Equal(t, "bob:secret@unix(/var/run/mysqld/mysqld.sock)/foo?flag=on&multiStatements=true", s)
})
}
func TestMySQLCreateDropDatabase(t *testing.T) {
drv := testMySQLDriver(t)
// drop any existing database
err := drv.DropDatabase()
require.NoError(t, err)
// create database
err = drv.CreateDatabase()
require.NoError(t, err)
// check that database exists and we can connect to it
func() {
db, err := drv.Open()
require.NoError(t, err)
defer dbutil.MustClose(db)
err = db.Ping()
require.NoError(t, err)
}()
// drop the database
err = drv.DropDatabase()
require.NoError(t, err)
// check that database no longer exists
func() {
db, err := drv.Open()
require.NoError(t, err)
defer dbutil.MustClose(db)
err = db.Ping()
require.Error(t, err)
require.Regexp(t, "Unknown database 'dbmate_test'", err.Error())
}()
}
func TestMySQLDumpSchema(t *testing.T) {
drv := testMySQLDriver(t)
drv.migrationsTableName = "test_migrations"
// prepare database
db := prepTestMySQLDB(t)
defer dbutil.MustClose(db)
err := drv.CreateMigrationsTable(db)
require.NoError(t, err)
// insert migration
err = drv.InsertMigration(db, "abc1")
require.NoError(t, err)
err = drv.InsertMigration(db, "abc2")
require.NoError(t, err)
// DumpSchema should return schema
schema, err := drv.DumpSchema(db)
require.NoError(t, err)
require.Contains(t, string(schema), "CREATE TABLE `test_migrations`")
require.Contains(t, string(schema), "\n-- Dump completed\n\n"+
"--\n"+
"-- Dbmate schema migrations\n"+
"--\n\n"+
"LOCK TABLES `test_migrations` WRITE;\n"+
"INSERT INTO `test_migrations` (version) VALUES\n"+
" ('abc1'),\n"+
" ('abc2');\n"+
"UNLOCK TABLES;\n")
// DumpSchema should return error if command fails
drv.databaseURL.Path = "/fakedb"
schema, err = drv.DumpSchema(db)
require.Nil(t, schema)
require.EqualError(t, err, "mysqldump: [Warning] Using a password "+
"on the command line interface can be insecure.\n"+
"mysqldump: Got error: 1049: "+
"Unknown database 'fakedb' when selecting the database")
}
func TestMySQLDatabaseExists(t *testing.T) {
drv := testMySQLDriver(t)
// drop any existing database
err := drv.DropDatabase()
require.NoError(t, err)
// DatabaseExists should return false
exists, err := drv.DatabaseExists()
require.NoError(t, err)
require.Equal(t, false, exists)
// create database
err = drv.CreateDatabase()
require.NoError(t, err)
// DatabaseExists should return true
exists, err = drv.DatabaseExists()
require.NoError(t, err)
require.Equal(t, true, exists)
}
func TestMySQLDatabaseExists_Error(t *testing.T) {
drv := testMySQLDriver(t)
drv.databaseURL.User = url.User("invalid")
exists, err := drv.DatabaseExists()
require.Error(t, err)
require.Regexp(t, "Access denied for user 'invalid'@", err.Error())
require.Equal(t, false, exists)
}
func TestMySQLCreateMigrationsTable(t *testing.T) {
drv := testMySQLDriver(t)
drv.migrationsTableName = "test_migrations"
db := prepTestMySQLDB(t)
defer dbutil.MustClose(db)
// migrations table should not exist
count := 0
err := db.QueryRow("select count(*) from test_migrations").Scan(&count)
require.Error(t, err)
require.Regexp(t, "Table 'dbmate_test.test_migrations' doesn't exist", err.Error())
// create table
err = drv.CreateMigrationsTable(db)
require.NoError(t, err)
// migrations table should exist
err = db.QueryRow("select count(*) from test_migrations").Scan(&count)
require.NoError(t, err)
// create table should be idempotent
err = drv.CreateMigrationsTable(db)
require.NoError(t, err)
}
func TestMySQLSelectMigrations(t *testing.T) {
drv := testMySQLDriver(t)
drv.migrationsTableName = "test_migrations"
db := prepTestMySQLDB(t)
defer dbutil.MustClose(db)
err := drv.CreateMigrationsTable(db)
require.NoError(t, err)
_, err = db.Exec(`insert into test_migrations (version)
values ('abc2'), ('abc1'), ('abc3')`)
require.NoError(t, err)
migrations, err := drv.SelectMigrations(db, -1)
require.NoError(t, err)
require.Equal(t, true, migrations["abc1"])
require.Equal(t, true, migrations["abc2"])
require.Equal(t, true, migrations["abc2"])
// test limit param
migrations, err = drv.SelectMigrations(db, 1)
require.NoError(t, err)
require.Equal(t, true, migrations["abc3"])
require.Equal(t, false, migrations["abc1"])
require.Equal(t, false, migrations["abc2"])
}
func TestMySQLInsertMigration(t *testing.T) {
drv := testMySQLDriver(t)
drv.migrationsTableName = "test_migrations"
db := prepTestMySQLDB(t)
defer dbutil.MustClose(db)
err := drv.CreateMigrationsTable(db)
require.NoError(t, err)
count := 0
err = db.QueryRow("select count(*) from test_migrations").Scan(&count)
require.NoError(t, err)
require.Equal(t, 0, count)
// insert migration
err = drv.InsertMigration(db, "abc1")
require.NoError(t, err)
err = db.QueryRow("select count(*) from test_migrations where version = 'abc1'").
Scan(&count)
require.NoError(t, err)
require.Equal(t, 1, count)
}
func TestMySQLDeleteMigration(t *testing.T) {
drv := testMySQLDriver(t)
drv.migrationsTableName = "test_migrations"
db := prepTestMySQLDB(t)
defer dbutil.MustClose(db)
err := drv.CreateMigrationsTable(db)
require.NoError(t, err)
_, err = db.Exec(`insert into test_migrations (version)
values ('abc1'), ('abc2')`)
require.NoError(t, err)
err = drv.DeleteMigration(db, "abc2")
require.NoError(t, err)
count := 0
err = db.QueryRow("select count(*) from test_migrations").Scan(&count)
require.NoError(t, err)
require.Equal(t, 1, count)
}
func TestMySQLPing(t *testing.T) {
drv := testMySQLDriver(t)
// drop any existing database
err := drv.DropDatabase()
require.NoError(t, err)
// ping database
err = drv.Ping()
require.NoError(t, err)
// ping invalid host should return error
drv.databaseURL.Host = "mysql:404"
err = drv.Ping()
require.Error(t, err)
require.Contains(t, err.Error(), "connect: connection refused")
}
func TestMySQLQuotedMigrationsTableName(t *testing.T) {
t.Run("default name", func(t *testing.T) {
drv := testMySQLDriver(t)
name := drv.quotedMigrationsTableName()
require.Equal(t, "`schema_migrations`", name)
})
t.Run("custom name", func(t *testing.T) {
drv := testMySQLDriver(t)
drv.migrationsTableName = "fooMigrations"
name := drv.quotedMigrationsTableName()
require.Equal(t, "`fooMigrations`", name)
})
}