From 9cfc758ca1d6166e07acb0647c9f0c6d48440636 Mon Sep 17 00:00:00 2001 From: Adrian Macneil Date: Wed, 25 Nov 2015 10:57:58 -0800 Subject: [PATCH] Initial commit --- .gitignore | 24 +++ Dockerfile | 20 +++ LICENSE | 21 +++ Makefile | 13 ++ README.md | 25 ++++ commands.go | 242 +++++++++++++++++++++++++++++++ commands_test.go | 19 +++ docker-compose.yml | 8 + driver/driver.go | 30 ++++ driver/postgres/postgres.go | 103 +++++++++++++ driver/postgres/postgres_test.go | 63 ++++++++ driver/shared/shared.go | 21 +++ driver/shared/shared_test.go | 24 +++ main.go | 84 +++++++++++ 14 files changed, 697 insertions(+) create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 LICENSE create mode 100644 Makefile create mode 100644 README.md create mode 100644 commands.go create mode 100644 commands_test.go create mode 100644 docker-compose.yml create mode 100644 driver/driver.go create mode 100644 driver/postgres/postgres.go create mode 100644 driver/postgres/postgres_test.go create mode 100644 driver/shared/shared.go create mode 100644 driver/shared/shared_test.go create mode 100644 main.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..daf913b --- /dev/null +++ b/.gitignore @@ -0,0 +1,24 @@ +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe +*.test +*.prof diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..dde4c2c --- /dev/null +++ b/Dockerfile @@ -0,0 +1,20 @@ +FROM alpine:3.2 + +ENV GOPATH /go +ENV PATH /go/bin:$PATH + +# install build dependencies +RUN apk add -U --no-progress go git ca-certificates +RUN go get \ + github.com/golang/lint/golint \ + golang.org/x/tools/cmd/vet + +# copy source files +COPY . /go/src/github.com/adrianmacneil/dbmate +WORKDIR /go/src/github.com/adrianmacneil/dbmate + +# build +RUN go get -d -t +RUN go install -v + +CMD dbmate diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..cb6766e --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2015 Adrian Macneil + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..3a97880 --- /dev/null +++ b/Makefile @@ -0,0 +1,13 @@ +DOCKER := docker-compose run dbmate + +all: build lint test + +build: + docker-compose build + +lint: + $(DOCKER) golint ./... + $(DOCKER) go vet ./... + +test: + $(DOCKER) go test ./... diff --git a/README.md b/README.md new file mode 100644 index 0000000..cfce9e0 --- /dev/null +++ b/README.md @@ -0,0 +1,25 @@ +# Dbmate + +## Installation + +Dbmate is currently under development. To install the latest build, run: + +```sh +$ go get -u github.com/adrianmacneil/dbmate +``` + +## Testing + +Tests are run with docker-compose. First, install the [Docker Toolbox](https://www.docker.com/docker-toolbox). + +Make sure you have docker running: + +```sh +$ docker-machine start default && eval "$(docker-machine env default)" +``` + +To build a docker image and run the tests: + +```sh +$ make +``` diff --git a/commands.go b/commands.go new file mode 100644 index 0000000..f5f98b2 --- /dev/null +++ b/commands.go @@ -0,0 +1,242 @@ +package main + +import ( + "database/sql" + "fmt" + "github.com/adrianmacneil/dbmate/driver" + "github.com/adrianmacneil/dbmate/driver/shared" + "github.com/codegangsta/cli" + "io/ioutil" + "net/url" + "os" + "path/filepath" + "regexp" + "time" +) + +// CreateCommand creates the current database +func CreateCommand(ctx *cli.Context) error { + u, err := GetDatabaseURL() + if err != nil { + return err + } + + drv, err := driver.Get(u.Scheme) + if err != nil { + return err + } + + return drv.CreateDatabase(u) +} + +// DropCommand drops the current database (if it exists) +func DropCommand(ctx *cli.Context) error { + u, err := GetDatabaseURL() + if err != nil { + return err + } + + drv, err := driver.Get(u.Scheme) + if err != nil { + return err + } + + return drv.DropDatabase(u) +} + +const migrationTemplate = "-- migrate:up\n\n\n-- migrate:down\n\n" + +// NewCommand creates a new migration file +func NewCommand(ctx *cli.Context) error { + // new migration name + timestamp := time.Now().UTC().Format("20060102150405") + name := ctx.Args().First() + if name == "" { + return fmt.Errorf("Please specify a name for the new migration.") + } + name = fmt.Sprintf("%s_%s.sql", timestamp, name) + + // create migrations dir if missing + migrationsDir := ctx.GlobalString("migrations-dir") + if err := os.MkdirAll(migrationsDir, 0755); err != nil { + return fmt.Errorf("Unable to create directory `%s`.", migrationsDir) + } + + // check file does not already exist + path := filepath.Join(migrationsDir, name) + fmt.Printf("Creating migration: %s\n", path) + + if _, err := os.Stat(path); !os.IsNotExist(err) { + return fmt.Errorf("File already exists") + } + + // write new migration + file, err := os.Create(path) + if err != nil { + return err + } + + defer file.Close() + _, err = file.WriteString(migrationTemplate) + if err != nil { + return err + } + + return nil +} + +// GetDatabaseURL returns the current environment database url +func GetDatabaseURL() (u *url.URL, err error) { + return url.Parse(os.Getenv("DATABASE_URL")) +} + +func doTransaction(db *sql.DB, txFunc func(shared.Transaction) error) error { + tx, err := db.Begin() + if err != nil { + return err + } + + if err := txFunc(tx); err != nil { + tx.Rollback() + return err + } + + return tx.Commit() +} + +// UpCommand migrates database to the latest version +func UpCommand(ctx *cli.Context) error { + migrationsDir := ctx.GlobalString("migrations-dir") + available, err := findAvailableMigrations(migrationsDir) + if err != nil { + return err + } + + if len(available) == 0 { + return fmt.Errorf("No migration files found.") + } + + u, err := GetDatabaseURL() + if err != nil { + return err + } + + drv, err := driver.Get(u.Scheme) + if err != nil { + return err + } + + db, err := drv.Open(u) + if err != nil { + return err + } + defer db.Close() + + if err := drv.CreateMigrationsTable(db); err != nil { + return err + } + + applied, err := drv.SelectMigrations(db) + if err != nil { + return err + } + + for filename := range available { + ver := migrationVersion(filename) + if _, ok := applied[ver]; ok { + // migration already applied + continue + } + + fmt.Printf("Applying: %s\n", filename) + + migration, err := parseMigration(filepath.Join(migrationsDir, filename)) + if err != nil { + return err + } + + // begin transaction + doTransaction(db, func(tx shared.Transaction) error { + // run actual migration + if _, err := tx.Exec(migration["up"]); err != nil { + return err + } + + // record migration + if err := drv.InsertMigration(tx, ver); err != nil { + return err + } + + return nil + }) + + } + + return nil +} + +func findAvailableMigrations(dir string) (map[string]struct{}, error) { + files, err := ioutil.ReadDir(dir) + if err != nil { + return nil, fmt.Errorf("Could not find migrations directory `%s`.", dir) + } + + nameRegexp := regexp.MustCompile(`^\d.*\.sql$`) + migrations := map[string]struct{}{} + + for _, file := range files { + if file.IsDir() { + continue + } + + name := file.Name() + if !nameRegexp.MatchString(name) { + continue + } + + migrations[name] = struct{}{} + } + + return migrations, nil +} + +func migrationVersion(filename string) string { + return regexp.MustCompile(`^\d+`).FindString(filename) +} + +// parseMigration reads a migration file into a map with up/down keys +// implementation is similar to regexp.Split() +func parseMigration(path string) (map[string]string, error) { + // read migration file into string + data, err := ioutil.ReadFile(path) + if err != nil { + return nil, err + } + contents := string(data) + + // split string on our trigger comment + separatorRegexp := regexp.MustCompile(`(?m)^-- migrate:(.*)$`) + matches := separatorRegexp.FindAllStringSubmatchIndex(contents, -1) + + migrations := map[string]string{} + direction := "" + beg := 0 + end := 0 + + for _, match := range matches { + end = match[0] + if direction != "" { + // write previous direction to output map + migrations[direction] = contents[beg:end] + } + + // each match records the start of a new direction + direction = contents[match[2]:match[3]] + beg = match[1] + } + + // write final direction to output map + migrations[direction] = contents[beg:] + + return migrations, nil +} diff --git a/commands_test.go b/commands_test.go new file mode 100644 index 0000000..daab091 --- /dev/null +++ b/commands_test.go @@ -0,0 +1,19 @@ +package main_test + +import ( + "github.com/adrianmacneil/dbmate" + "github.com/stretchr/testify/require" + "os" + "testing" +) + +func TestGetDatabaseUrl(t *testing.T) { + os.Setenv("DATABASE_URL", "postgres://example.org/db") + + u, err := main.GetDatabaseURL() + require.Nil(t, err) + + require.Equal(t, "postgres", u.Scheme) + require.Equal(t, "example.org", u.Host) + require.Equal(t, "/db", u.Path) +} diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..c5e065b --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,8 @@ +postgres: + image: postgres:9.4 +dbmate: + build: . + volumes: + - .:/go/src/github.com/adrianmacneil/dbmate + links: + - postgres diff --git a/driver/driver.go b/driver/driver.go new file mode 100644 index 0000000..a79af97 --- /dev/null +++ b/driver/driver.go @@ -0,0 +1,30 @@ +package driver + +import ( + "database/sql" + "fmt" + "github.com/adrianmacneil/dbmate/driver/postgres" + "github.com/adrianmacneil/dbmate/driver/shared" + "net/url" +) + +// Driver provides top level database functions +type Driver interface { + Open(*url.URL) (*sql.DB, error) + CreateDatabase(*url.URL) error + DropDatabase(*url.URL) error + CreateMigrationsTable(*sql.DB) error + SelectMigrations(*sql.DB) (map[string]struct{}, error) + InsertMigration(shared.Transaction, string) error + DeleteMigration(shared.Transaction, string) error +} + +// Get loads a database driver by name +func Get(name string) (Driver, error) { + switch name { + case "postgres": + return postgres.Driver{}, nil + default: + return nil, fmt.Errorf("Unknown driver: %s", name) + } +} diff --git a/driver/postgres/postgres.go b/driver/postgres/postgres.go new file mode 100644 index 0000000..90ff1a1 --- /dev/null +++ b/driver/postgres/postgres.go @@ -0,0 +1,103 @@ +package postgres + +import ( + "database/sql" + "fmt" + "github.com/adrianmacneil/dbmate/driver/shared" + pq "github.com/lib/pq" + "net/url" +) + +// Driver provides top level database functions +type Driver struct { +} + +// Open creates a new database connection +func (postgres Driver) Open(u *url.URL) (*sql.DB, error) { + return sql.Open("postgres", u.String()) +} + +// postgresExec runs a sql statement on the "postgres" database +func (postgres Driver) postgresExec(u *url.URL, statement string) error { + // connect to postgres database + postgresURL := *u + postgresURL.Path = "postgres" + + db, err := postgres.Open(&postgresURL) + if err != nil { + return err + } + defer db.Close() + + // run statement + _, err = db.Exec(statement) + + return err +} + +// CreateDatabase creates the specified database +func (postgres Driver) CreateDatabase(u *url.URL) error { + database := shared.DatabaseName(u) + fmt.Printf("Creating: %s\n", database) + + return postgres.postgresExec(u, fmt.Sprintf("CREATE DATABASE %s", + pq.QuoteIdentifier(database))) +} + +// DropDatabase drops the specified database (if it exists) +func (postgres Driver) DropDatabase(u *url.URL) error { + database := shared.DatabaseName(u) + fmt.Printf("Dropping: %s\n", database) + + return postgres.postgresExec(u, fmt.Sprintf("DROP DATABASE IF EXISTS %s", + pq.QuoteIdentifier(database))) +} + +// HasMigrationsTable returns true if the schema_migrations table exists +func (postgres Driver) HasMigrationsTable(db *sql.DB) (bool, error) { + return false, fmt.Errorf("not implemented") +} + +// CreateMigrationsTable creates the schema_migrations table +func (postgres Driver) CreateMigrationsTable(db *sql.DB) error { + _, err := db.Exec(`CREATE TABLE IF NOT EXISTS schema_migrations ( + version varchar(255) PRIMARY KEY)`) + + return err +} + +// SelectMigrations returns a list of applied migrations +func (postgres Driver) SelectMigrations(db *sql.DB) (map[string]struct{}, error) { + rows, err := db.Query("SELECT version FROM schema_migrations") + if err != nil { + return nil, err + } + + defer rows.Close() + + migrations := map[string]struct{}{} + for rows.Next() { + var version string + if err := rows.Scan(&version); err != nil { + return nil, err + } + + migrations[version] = struct{}{} + } + + return migrations, nil +} + +// InsertMigration adds a new migration record +func (postgres Driver) InsertMigration(db shared.Transaction, version string) error { + _, err := db.Exec("INSERT INTO schema_migrations (version) VALUES ($1)", version) + + return err +} + +// DeleteMigration removes a migration record +func (postgres Driver) DeleteMigration(db shared.Transaction, version string) error { + _, err := db.Exec("DELETE FROM schema_migrations WHERE version = $1", version) + + return err +} diff --git a/driver/postgres/postgres_test.go b/driver/postgres/postgres_test.go new file mode 100644 index 0000000..0efa9b7 --- /dev/null +++ b/driver/postgres/postgres_test.go @@ -0,0 +1,63 @@ +package postgres_test + +import ( + "database/sql" + "github.com/adrianmacneil/dbmate/driver/postgres" + "github.com/stretchr/testify/require" + "net/url" + "os" + "testing" +) + +func testURL(t *testing.T) *url.URL { + str := os.Getenv("POSTGRES_PORT") + require.NotEmpty(t, str) + + u, err := url.Parse(str) + require.Nil(t, err) + + u.Scheme = "postgres" + u.User = url.User("postgres") + u.Path = "/dbmate" + u.RawQuery = "sslmode=disable" + + return u +} + +func TestCreateDropDatabase(t *testing.T) { + d := postgres.Driver{} + u := testURL(t) + + // drop any existing database + err := d.DropDatabase(u) + require.Nil(t, err) + + // create database + err = d.CreateDatabase(u) + require.Nil(t, err) + + // check that database exists and we can connect to it + func() { + db, err := sql.Open("postgres", u.String()) + require.Nil(t, err) + defer db.Close() + + err = db.Ping() + require.Nil(t, err) + }() + + // drop the database + err = d.DropDatabase(u) + require.Nil(t, err) + + // check that database no longer exists + func() { + db, err := sql.Open("postgres", u.String()) + require.Nil(t, err) + defer db.Close() + + err = db.Ping() + require.NotNil(t, err) + require.Equal(t, "pq: database \"dbmate\" does not exist", err.Error()) + }() +} diff --git a/driver/shared/shared.go b/driver/shared/shared.go new file mode 100644 index 0000000..d0a9c40 --- /dev/null +++ b/driver/shared/shared.go @@ -0,0 +1,21 @@ +package shared + +import ( + "database/sql" + "net/url" +) + +// DatabaseName returns the database name from a URL +func DatabaseName(u *url.URL) string { + name := u.Path + if len(name) > 0 && name[:1] == "/" { + name = name[1:len(name)] + } + + return name +} + +// Transaction can represent a database or open transaction +type Transaction interface { + Exec(query string, args ...interface{}) (sql.Result, error) +} diff --git a/driver/shared/shared_test.go b/driver/shared/shared_test.go new file mode 100644 index 0000000..1585459 --- /dev/null +++ b/driver/shared/shared_test.go @@ -0,0 +1,24 @@ +package shared_test + +import ( + "github.com/adrianmacneil/dbmate/driver/shared" + "github.com/stretchr/testify/require" + "net/url" + "testing" +) + +func TestDatabaseName(t *testing.T) { + u, err := url.Parse("ignore://localhost/foo?query") + require.Nil(t, err) + + name := shared.DatabaseName(u) + require.Equal(t, "foo", name) +} + +func TestDatabaseName_Empty(t *testing.T) { + u, err := url.Parse("ignore://localhost") + require.Nil(t, err) + + name := shared.DatabaseName(u) + require.Equal(t, "", name) +} diff --git a/main.go b/main.go new file mode 100644 index 0000000..a324f71 --- /dev/null +++ b/main.go @@ -0,0 +1,84 @@ +package main + +import ( + "fmt" + "github.com/codegangsta/cli" + "github.com/joho/godotenv" + "log" + "os" +) + +func main() { + loadDotEnv() + + app := cli.NewApp() + app.Name = "dbmate" + app.Usage = "A lightweight, framework-independent database migration tool." + + app.Flags = []cli.Flag{ + cli.StringFlag{ + Name: "migrations-dir, d", + Value: "./db/migrations", + Usage: "specify the directory containing migration files", + }, + } + + app.Commands = []cli.Command{ + { + Name: "up", + Usage: "Migrate to the latest version", + Action: func(ctx *cli.Context) { + runCommand(UpCommand, ctx) + }, + Flags: []cli.Flag{ + cli.BoolFlag{ + Name: "pretend, p", + Usage: "don't do anything; print migrations to apply", + }, + }, + }, + { + Name: "new", + Usage: "Generate a new migration file", + Action: func(ctx *cli.Context) { + runCommand(NewCommand, ctx) + }, + }, + { + Name: "create", + Usage: "Create database", + Action: func(ctx *cli.Context) { + runCommand(CreateCommand, ctx) + }, + }, + { + Name: "drop", + Usage: "Drop database (if it exists)", + Action: func(ctx *cli.Context) { + runCommand(DropCommand, ctx) + }, + }, + } + + app.Run(os.Args) +} + +type command func(*cli.Context) error + +func runCommand(cmd command, ctx *cli.Context) { + err := cmd(ctx) + if err != nil { + fmt.Fprintf(os.Stderr, "Error: %s\n", err) + os.Exit(1) + } +} + +func loadDotEnv() { + if _, err := os.Stat(".env"); err != nil { + return + } + + if err := godotenv.Load(); err != nil { + log.Fatal("Error loading .env file") + } +}