Refactor drivers into separate packages (#179)

`dbmate` package was starting to get a bit polluted. This PR migrates each driver into a separate package, with clean separation between each.

In addition:

* Drivers are now initialized with a URL, avoiding the need to pass `*url.URL` to every method
* Sqlite supports a cleaner syntax for relative paths
* Driver tests now load their test URL from environment variables

Public API of `dbmate` package has not changed (no changes to `main` package).
This commit is contained in:
Adrian Macneil 2020-11-19 15:04:42 +13:00 committed by GitHub
parent c907c3f5c6
commit 61771e386d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
23 changed files with 1195 additions and 1078 deletions

View file

@ -0,0 +1,326 @@
package clickhouse
import (
"bytes"
"database/sql"
"fmt"
"net/url"
"regexp"
"sort"
"strings"
"github.com/amacneil/dbmate/pkg/dbmate"
"github.com/amacneil/dbmate/pkg/dbutil"
"github.com/ClickHouse/clickhouse-go"
)
func init() {
dbmate.RegisterDriver(NewDriver, "clickhouse")
}
// Driver provides top level database functions
type Driver struct {
migrationsTableName string
databaseURL *url.URL
}
// NewDriver initializes the driver
func NewDriver(config dbmate.DriverConfig) dbmate.Driver {
return &Driver{
migrationsTableName: config.MigrationsTableName,
databaseURL: config.DatabaseURL,
}
}
func connectionString(initialURL *url.URL) string {
u := *initialURL
u.Scheme = "tcp"
host := u.Host
if u.Port() == "" {
host = fmt.Sprintf("%s:9000", host)
}
u.Host = host
query := u.Query()
if query.Get("username") == "" && u.User.Username() != "" {
query.Set("username", u.User.Username())
}
password, passwordSet := u.User.Password()
if query.Get("password") == "" && passwordSet {
query.Set("password", password)
}
u.User = nil
if query.Get("database") == "" {
path := strings.Trim(u.Path, "/")
if path != "" {
query.Set("database", path)
u.Path = ""
}
}
u.RawQuery = query.Encode()
return u.String()
}
// Open creates a new database connection
func (drv *Driver) Open() (*sql.DB, error) {
return sql.Open("clickhouse", connectionString(drv.databaseURL))
}
func (drv *Driver) openClickHouseDB() (*sql.DB, error) {
// clone databaseURL
clickhouseURL, err := url.Parse(connectionString(drv.databaseURL))
if err != nil {
return nil, err
}
// connect to clickhouse database
values := clickhouseURL.Query()
values.Set("database", "default")
clickhouseURL.RawQuery = values.Encode()
return sql.Open("clickhouse", clickhouseURL.String())
}
func (drv *Driver) databaseName() string {
name := dbutil.MustParseURL(connectionString(drv.databaseURL)).Query().Get("database")
if name == "" {
name = "default"
}
return name
}
var clickhouseValidIdentifier = regexp.MustCompile(`^[a-zA-Z_][0-9a-zA-Z_]*$`)
func (drv *Driver) quoteIdentifier(str string) string {
if clickhouseValidIdentifier.MatchString(str) {
return str
}
str = strings.Replace(str, `"`, `""`, -1)
return fmt.Sprintf(`"%s"`, str)
}
// CreateDatabase creates the specified database
func (drv *Driver) CreateDatabase() error {
name := drv.databaseName()
fmt.Printf("Creating: %s\n", name)
db, err := drv.openClickHouseDB()
if err != nil {
return err
}
defer dbutil.MustClose(db)
_, err = db.Exec("create database " + drv.quoteIdentifier(name))
return err
}
// DropDatabase drops the specified database (if it exists)
func (drv *Driver) DropDatabase() error {
name := drv.databaseName()
fmt.Printf("Dropping: %s\n", name)
db, err := drv.openClickHouseDB()
if err != nil {
return err
}
defer dbutil.MustClose(db)
_, err = db.Exec("drop database if exists " + drv.quoteIdentifier(name))
return err
}
func (drv *Driver) schemaDump(db *sql.DB, buf *bytes.Buffer, databaseName string) error {
buf.WriteString("\n--\n-- Database schema\n--\n\n")
buf.WriteString("CREATE DATABASE " + drv.quoteIdentifier(databaseName) + " IF NOT EXISTS;\n\n")
tables, err := dbutil.QueryColumn(db, "show tables")
if err != nil {
return err
}
sort.Strings(tables)
for _, table := range tables {
var clause string
err = db.QueryRow("show create table " + drv.quoteIdentifier(table)).Scan(&clause)
if err != nil {
return err
}
buf.WriteString(clause + ";\n\n")
}
return nil
}
func (drv *Driver) schemaMigrationsDump(db *sql.DB, buf *bytes.Buffer) error {
migrationsTable := drv.quotedMigrationsTableName()
// load applied migrations
migrations, err := dbutil.QueryColumn(db,
fmt.Sprintf("select version from %s final ", migrationsTable)+
"where applied order by version asc",
)
if err != nil {
return err
}
quoter := strings.NewReplacer(`\`, `\\`, `'`, `\'`)
for i := range migrations {
migrations[i] = "'" + quoter.Replace(migrations[i]) + "'"
}
// build schema migrations table data
buf.WriteString("\n--\n-- Dbmate schema migrations\n--\n\n")
if len(migrations) > 0 {
buf.WriteString(
fmt.Sprintf("INSERT INTO %s (version) VALUES\n (", migrationsTable) +
strings.Join(migrations, "),\n (") +
");\n")
}
return nil
}
// DumpSchema returns the current database schema
func (drv *Driver) DumpSchema(db *sql.DB) ([]byte, error) {
var buf bytes.Buffer
var err error
err = drv.schemaDump(db, &buf, drv.databaseName())
if err != nil {
return nil, err
}
err = drv.schemaMigrationsDump(db, &buf)
if err != nil {
return nil, err
}
return buf.Bytes(), nil
}
// DatabaseExists determines whether the database exists
func (drv *Driver) DatabaseExists() (bool, error) {
name := drv.databaseName()
db, err := drv.openClickHouseDB()
if err != nil {
return false, err
}
defer dbutil.MustClose(db)
exists := false
err = db.QueryRow("SELECT 1 FROM system.databases where name = ?", name).
Scan(&exists)
if err == sql.ErrNoRows {
return false, nil
}
return exists, err
}
// CreateMigrationsTable creates the schema migrations table
func (drv *Driver) CreateMigrationsTable(db *sql.DB) error {
_, err := db.Exec(fmt.Sprintf(`
create table if not exists %s (
version String,
ts DateTime default now(),
applied UInt8 default 1
) engine = ReplacingMergeTree(ts)
primary key version
order by version
`, drv.quotedMigrationsTableName()))
return err
}
// SelectMigrations returns a list of applied migrations
// with an optional limit (in descending order)
func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, error) {
query := fmt.Sprintf("select version from %s final where applied order by version desc",
drv.quotedMigrationsTableName())
if limit >= 0 {
query = fmt.Sprintf("%s limit %d", query, limit)
}
rows, err := db.Query(query)
if err != nil {
return nil, err
}
defer dbutil.MustClose(rows)
migrations := map[string]bool{}
for rows.Next() {
var version string
if err := rows.Scan(&version); err != nil {
return nil, err
}
migrations[version] = true
}
if err = rows.Err(); err != nil {
return nil, err
}
return migrations, nil
}
// InsertMigration adds a new migration record
func (drv *Driver) InsertMigration(db dbutil.Transaction, version string) error {
_, err := db.Exec(
fmt.Sprintf("insert into %s (version) values (?)", drv.quotedMigrationsTableName()),
version)
return err
}
// DeleteMigration removes a migration record
func (drv *Driver) DeleteMigration(db dbutil.Transaction, version string) error {
_, err := db.Exec(
fmt.Sprintf("insert into %s (version, applied) values (?, ?)",
drv.quotedMigrationsTableName()),
version, false,
)
return err
}
// Ping verifies a connection to the database server. It does not verify whether the
// specified database exists.
func (drv *Driver) Ping() error {
// attempt connection to primary database, not "clickhouse" database
// to support servers with no "clickhouse" database
// (see https://github.com/amacneil/dbmate/issues/78)
db, err := drv.Open()
if err != nil {
return err
}
defer dbutil.MustClose(db)
err = db.Ping()
if err == nil {
return nil
}
// ignore 'Database foo doesn't exist' error
chErr, ok := err.(*clickhouse.Exception)
if ok && chErr.Code == 81 {
return nil
}
return err
}
func (drv *Driver) quotedMigrationsTableName() string {
return drv.quoteIdentifier(drv.migrationsTableName)
}

View file

@ -0,0 +1,376 @@
package clickhouse
import (
"database/sql"
"net/url"
"os"
"testing"
"github.com/amacneil/dbmate/pkg/dbmate"
"github.com/amacneil/dbmate/pkg/dbutil"
"github.com/stretchr/testify/require"
)
func testClickHouseDriver(t *testing.T) *Driver {
u := dbutil.MustParseURL(os.Getenv("CLICKHOUSE_TEST_URL"))
drv, err := dbmate.New(u).GetDriver()
require.NoError(t, err)
return drv.(*Driver)
}
func prepTestClickHouseDB(t *testing.T) *sql.DB {
drv := testClickHouseDriver(t)
// drop any existing database
err := drv.DropDatabase()
require.NoError(t, err)
// create database
err = drv.CreateDatabase()
require.NoError(t, err)
// connect database
db, err := sql.Open("clickhouse", drv.databaseURL.String())
require.NoError(t, err)
return db
}
func TestGetDriver(t *testing.T) {
db := dbmate.New(dbutil.MustParseURL("clickhouse://"))
drvInterface, err := db.GetDriver()
require.NoError(t, err)
// driver should have URL and default migrations table set
drv, ok := drvInterface.(*Driver)
require.True(t, ok)
require.Equal(t, db.DatabaseURL.String(), drv.databaseURL.String())
require.Equal(t, "schema_migrations", drv.migrationsTableName)
}
func TestConnectionString(t *testing.T) {
t.Run("simple", func(t *testing.T) {
u, err := url.Parse("clickhouse://user:pass@host/db")
require.NoError(t, err)
s := connectionString(u)
require.Equal(t, "tcp://host:9000?database=db&password=pass&username=user", s)
})
t.Run("canonical", func(t *testing.T) {
u, err := url.Parse("clickhouse://host:9000?database=db&password=pass&username=user")
require.NoError(t, err)
s := connectionString(u)
require.Equal(t, "tcp://host:9000?database=db&password=pass&username=user", s)
})
}
func TestClickHouseCreateDropDatabase(t *testing.T) {
drv := testClickHouseDriver(t)
// drop any existing database
err := drv.DropDatabase()
require.NoError(t, err)
// create database
err = drv.CreateDatabase()
require.NoError(t, err)
// check that database exists and we can connect to it
func() {
db, err := sql.Open("clickhouse", drv.databaseURL.String())
require.NoError(t, err)
defer dbutil.MustClose(db)
err = db.Ping()
require.NoError(t, err)
}()
// drop the database
err = drv.DropDatabase()
require.NoError(t, err)
// check that database no longer exists
func() {
db, err := sql.Open("clickhouse", drv.databaseURL.String())
require.NoError(t, err)
defer dbutil.MustClose(db)
err = db.Ping()
require.EqualError(t, err, "code: 81, message: Database dbmate_test doesn't exist")
}()
}
func TestClickHouseDumpSchema(t *testing.T) {
drv := testClickHouseDriver(t)
drv.migrationsTableName = "test_migrations"
// prepare database
db := prepTestClickHouseDB(t)
defer dbutil.MustClose(db)
err := drv.CreateMigrationsTable(db)
require.NoError(t, err)
// insert migration
tx, err := db.Begin()
require.NoError(t, err)
err = drv.InsertMigration(tx, "abc1")
require.NoError(t, err)
err = tx.Commit()
require.NoError(t, err)
tx, err = db.Begin()
require.NoError(t, err)
err = drv.InsertMigration(tx, "abc2")
require.NoError(t, err)
err = tx.Commit()
require.NoError(t, err)
// DumpSchema should return schema
schema, err := drv.DumpSchema(db)
require.NoError(t, err)
require.Contains(t, string(schema), "CREATE TABLE "+drv.databaseName()+".test_migrations")
require.Contains(t, string(schema), "--\n"+
"-- Dbmate schema migrations\n"+
"--\n\n"+
"INSERT INTO test_migrations (version) VALUES\n"+
" ('abc1'),\n"+
" ('abc2');\n")
// DumpSchema should return error if command fails
values := drv.databaseURL.Query()
values.Set("database", "fakedb")
drv.databaseURL.RawQuery = values.Encode()
db, err = sql.Open("clickhouse", drv.databaseURL.String())
require.NoError(t, err)
schema, err = drv.DumpSchema(db)
require.Nil(t, schema)
require.EqualError(t, err, "code: 81, message: Database fakedb doesn't exist")
}
func TestClickHouseDatabaseExists(t *testing.T) {
drv := testClickHouseDriver(t)
// drop any existing database
err := drv.DropDatabase()
require.NoError(t, err)
// DatabaseExists should return false
exists, err := drv.DatabaseExists()
require.NoError(t, err)
require.Equal(t, false, exists)
// create database
err = drv.CreateDatabase()
require.NoError(t, err)
// DatabaseExists should return true
exists, err = drv.DatabaseExists()
require.NoError(t, err)
require.Equal(t, true, exists)
}
func TestClickHouseDatabaseExists_Error(t *testing.T) {
drv := testClickHouseDriver(t)
values := drv.databaseURL.Query()
values.Set("username", "invalid")
drv.databaseURL.RawQuery = values.Encode()
exists, err := drv.DatabaseExists()
require.EqualError(t, err, "code: 192, message: Unknown user invalid")
require.Equal(t, false, exists)
}
func TestClickHouseCreateMigrationsTable(t *testing.T) {
t.Run("default table", func(t *testing.T) {
drv := testClickHouseDriver(t)
db := prepTestClickHouseDB(t)
defer dbutil.MustClose(db)
// migrations table should not exist
count := 0
err := db.QueryRow("select count(*) from schema_migrations").Scan(&count)
require.EqualError(t, err, "code: 60, message: Table dbmate_test.schema_migrations doesn't exist.")
// create table
err = drv.CreateMigrationsTable(db)
require.NoError(t, err)
// migrations table should exist
err = db.QueryRow("select count(*) from schema_migrations").Scan(&count)
require.NoError(t, err)
// create table should be idempotent
err = drv.CreateMigrationsTable(db)
require.NoError(t, err)
})
t.Run("custom table", func(t *testing.T) {
drv := testClickHouseDriver(t)
drv.migrationsTableName = "testMigrations"
db := prepTestClickHouseDB(t)
defer dbutil.MustClose(db)
// migrations table should not exist
count := 0
err := db.QueryRow("select count(*) from \"testMigrations\"").Scan(&count)
require.EqualError(t, err, "code: 60, message: Table dbmate_test.testMigrations doesn't exist.")
// create table
err = drv.CreateMigrationsTable(db)
require.NoError(t, err)
// migrations table should exist
err = db.QueryRow("select count(*) from \"testMigrations\"").Scan(&count)
require.NoError(t, err)
// create table should be idempotent
err = drv.CreateMigrationsTable(db)
require.NoError(t, err)
})
}
func TestClickHouseSelectMigrations(t *testing.T) {
drv := testClickHouseDriver(t)
drv.migrationsTableName = "test_migrations"
db := prepTestClickHouseDB(t)
defer dbutil.MustClose(db)
err := drv.CreateMigrationsTable(db)
require.NoError(t, err)
tx, err := db.Begin()
require.NoError(t, err)
stmt, err := tx.Prepare("insert into test_migrations (version) values (?)")
require.NoError(t, err)
_, err = stmt.Exec("abc2")
require.NoError(t, err)
_, err = stmt.Exec("abc1")
require.NoError(t, err)
_, err = stmt.Exec("abc3")
require.NoError(t, err)
err = tx.Commit()
require.NoError(t, err)
migrations, err := drv.SelectMigrations(db, -1)
require.NoError(t, err)
require.Equal(t, true, migrations["abc1"])
require.Equal(t, true, migrations["abc2"])
require.Equal(t, true, migrations["abc2"])
// test limit param
migrations, err = drv.SelectMigrations(db, 1)
require.NoError(t, err)
require.Equal(t, true, migrations["abc3"])
require.Equal(t, false, migrations["abc1"])
require.Equal(t, false, migrations["abc2"])
}
func TestClickHouseInsertMigration(t *testing.T) {
drv := testClickHouseDriver(t)
drv.migrationsTableName = "test_migrations"
db := prepTestClickHouseDB(t)
defer dbutil.MustClose(db)
err := drv.CreateMigrationsTable(db)
require.NoError(t, err)
count := 0
err = db.QueryRow("select count(*) from test_migrations").Scan(&count)
require.NoError(t, err)
require.Equal(t, 0, count)
// insert migration
tx, err := db.Begin()
require.NoError(t, err)
err = drv.InsertMigration(tx, "abc1")
require.NoError(t, err)
err = tx.Commit()
require.NoError(t, err)
err = db.QueryRow("select count(*) from test_migrations where version = 'abc1'").Scan(&count)
require.NoError(t, err)
require.Equal(t, 1, count)
}
func TestClickHouseDeleteMigration(t *testing.T) {
drv := testClickHouseDriver(t)
drv.migrationsTableName = "test_migrations"
db := prepTestClickHouseDB(t)
defer dbutil.MustClose(db)
err := drv.CreateMigrationsTable(db)
require.NoError(t, err)
tx, err := db.Begin()
require.NoError(t, err)
stmt, err := tx.Prepare("insert into test_migrations (version) values (?)")
require.NoError(t, err)
_, err = stmt.Exec("abc2")
require.NoError(t, err)
_, err = stmt.Exec("abc1")
require.NoError(t, err)
err = tx.Commit()
require.NoError(t, err)
tx, err = db.Begin()
require.NoError(t, err)
err = drv.DeleteMigration(tx, "abc2")
require.NoError(t, err)
err = tx.Commit()
require.NoError(t, err)
count := 0
err = db.QueryRow("select count(*) from test_migrations final where applied").Scan(&count)
require.NoError(t, err)
require.Equal(t, 1, count)
}
func TestClickHousePing(t *testing.T) {
drv := testClickHouseDriver(t)
// drop any existing database
err := drv.DropDatabase()
require.NoError(t, err)
// ping database
err = drv.Ping()
require.NoError(t, err)
// ping invalid host should return error
drv.databaseURL.Host = "clickhouse:404"
err = drv.Ping()
require.Error(t, err)
require.Contains(t, err.Error(), "connect: connection refused")
}
func TestClickHouseQuotedMigrationsTableName(t *testing.T) {
t.Run("default name", func(t *testing.T) {
drv := testClickHouseDriver(t)
name := drv.quotedMigrationsTableName()
require.Equal(t, "schema_migrations", name)
})
t.Run("custom name", func(t *testing.T) {
drv := testClickHouseDriver(t)
drv.migrationsTableName = "fooMigrations"
name := drv.quotedMigrationsTableName()
require.Equal(t, "fooMigrations", name)
})
t.Run("quoted name", func(t *testing.T) {
drv := testClickHouseDriver(t)
drv.migrationsTableName = "bizarre\"$name"
name := drv.quotedMigrationsTableName()
require.Equal(t, `"bizarre""$name"`, name)
})
}