Compare commits

..

No commits in common. "main" and "v1.10.0" have entirely different histories.

44 changed files with 2804 additions and 4101 deletions

View file

@ -1 +0,0 @@
.git

View file

@ -1,6 +0,0 @@
version: 2
updates:
- package-ecosystem: "github-actions"
directory: "/"
schedule:
interval: "daily"

View file

@ -1,151 +0,0 @@
name: CI
on:
push:
branches: [main]
tags: "*"
pull_request:
jobs:
build:
strategy:
fail-fast: false
matrix:
include:
- os: linux
image: ubuntu-latest
arch: amd64
env: {}
- os: linux
image: ubuntu-latest
arch: arm64
setup: sudo apt-get update && sudo apt-get install -qq gcc-aarch64-linux-gnu
env:
CC: aarch64-linux-gnu-gcc
CXX: aarch64-linux-gnu-g++
- os: macos
image: macos-latest
arch: amd64
env: {}
- os: macos
image: macos-latest
arch: arm64
env: {}
- os: windows
image: windows-latest
arch: amd64
env: {}
name: Build (${{ matrix.os }}/${{ matrix.arch }})
runs-on: ${{ matrix.image }}
env: ${{ matrix.env }}
steps:
- uses: actions/checkout@v2
- uses: actions/setup-go@v2
with:
go-version: "1.17"
- name: Setup environment
run: ${{ matrix.setup }}
- run: go mod download
- run: make build ls
env:
GOARCH: ${{ matrix.arch }}
OUTPUT: dbmate-${{ matrix.os }}-${{ matrix.arch }}
- run: dist/dbmate-${{ matrix.os }}-${{ matrix.arch }} --help
if: ${{ matrix.arch == 'amd64' }}
- name: Publish binaries
uses: softprops/action-gh-release@v1
if: ${{ startsWith(github.ref, 'refs/tags/v') }}
with:
files: dist/dbmate-*
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
docker:
name: Docker Test (linux/amd64)
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Configure QEMU
uses: docker/setup-qemu-action@v1
- name: Configure Buildx
uses: docker/setup-buildx-action@v1
- name: Check Docker environment
run: |
set -x
docker version
docker buildx version
docker-compose version
- name: Build Docker image
run: |
set -x
docker-compose build
docker-compose run --rm --no-deps dbmate --version
- name: Run make build
run: docker-compose run --rm --no-deps dev make build ls
- name: Run make lint
run: docker-compose run --rm --no-deps dev make lint
- name: Start test dependencies
run: |
set -x
docker-compose pull --quiet
docker-compose up --detach
docker-compose run --rm dev make wait
- name: Run make test
run: docker-compose run --rm dev make test
- name: Login to Docker Hub
uses: docker/login-action@v1
if: ${{ github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v') }}
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Login to GitHub Container Registry
uses: docker/login-action@v1
if: ${{ github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v') }}
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Generate Docker image tags
id: meta
uses: docker/metadata-action@v3
if: ${{ github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v') }}
with:
images: |
${{ github.repository }}
ghcr.io/${{ github.repository }}
tags: |
type=ref,event=branch
type=semver,pattern={{version}}
type=semver,pattern={{major}}.{{minor}}
type=semver,pattern={{major}}
- name: Publish Docker image
uses: docker/build-push-action@v2
if: ${{ github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v') }}
with:
context: .
target: release
platforms: linux/amd64,linux/arm64
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}

1
.gitignore vendored
View file

@ -1,6 +1,5 @@
.DS_Store
.env
/.cache
/db
/dbmate
/dist

View file

@ -10,8 +10,6 @@ linters:
- ineffassign
- misspell
- nakedret
- revive
- rowserrcheck
- staticcheck
- structcheck
- typecheck
@ -26,7 +24,3 @@ linters-settings:
local-prefixes: github.com/amacneil/dbmate
misspell:
locale: US
issues:
include:
- EXC0002

27
.travis.yml Normal file
View file

@ -0,0 +1,27 @@
sudo: required
services:
- docker
install:
- docker version
- docker-compose version
- rm docker-compose.override.yml
- docker-compose pull
- docker-compose build
- docker-compose up -d
script:
- docker-compose run --rm --volume "$PWD/dist:/src/dist" dbmate make build-all
- docker-compose run --rm dbmate make lint
- docker-compose run --rm dbmate make wait
- docker-compose run --rm dbmate make test
- docker build -t dbmate .
- docker run --rm dbmate --help
deploy:
provider: releases
api_key:
secure: LuDKEwGYaJWqYe0Ym6qoxHAbZx1kDiTigYcycribnmugGVDvRpZp5MJuQivTD1eZ4sl58UO3NX6jyf8pfx814m6G+3gjWaQ56OtJIKF2OwtxnwvMZNaVz63hSi8n1jCdbGTHlOqDAUQbjGFGrmsI5wAGsUM16yRktCAEn5APHxNHMnQcGTIe3Wcp+G4Fp+iRQ80Ro6BLPo2ys+WWDxz6Wahv3U6CJWtkQMAuZyZTSXL1Pl6kqlZyGKhUbPHvq1KU0wWccvwT5P6KVo314aF5Skw0LJ3qciwUTnc7dsHCkvJKF5/Nev3/KWWVKR3DBh98gS2hDNjpSozYAO/e9QiIjaidqYYifoEFIY7Jx0DArJwaw3PLnRMKGKMyww2CaFopxr5HT1s18EGMytRbduASUieeF+7pFs29Bouc8xC0OnKZdlXRewAYFjWzWdCiXQVU18q3DggFK6fb1HWLmy6NX2RmxDODSv3B8P3DzmsdwR0vc64IxmnS+zTdjUwE0+FuxOEmWl/iqYi+nXKXOj0domFudfaBxGT2f5ThBw5Ns9FXKBGxyRSD8wf8+sDbUIUxUdZw1kCttNM/JSbbz9ErLV/Ik23BWBPkjDxo4DpLgqVMg8LHPbmhCuKHvckhoCBpORuvX3PTzzdCsJfiYJCr6nMt/deAp/B/O2O/3/2nFYI=
file_glob: true
file: dist/*
skip_cleanup: true
on:
tags: true
repo: amacneil/dbmate

View file

@ -1,4 +0,0 @@
// -*- jsonc -*-
{
"recommendations": ["esbenp.prettier-vscode"]
}

10
.vscode/settings.json vendored
View file

@ -1,10 +0,0 @@
// -*- jsonc -*-
{
"editor.defaultFormatter": "esbenp.prettier-vscode",
"editor.formatOnSave": true,
"files.eol": "\n",
"files.insertFinalNewline": true,
"files.trimFinalNewlines": true,
"files.trimTrailingWhitespace": true,
"go.formatTool": "goimports"
}

View file

@ -1,33 +1,41 @@
# development image
FROM golang:1.17 as dev
# build image
FROM techknowlogick/xgo:go-1.14.x as build
WORKDIR /src
# enable cgo to build sqlite
ENV CGO_ENABLED 1
# install database clients
RUN apt-get update \
&& apt-get install -qq --no-install-recommends \
curl \
file \
mariadb-client \
mysql-client \
postgresql-client \
sqlite3 \
&& rm -rf /var/lib/apt/lists/*
# golangci-lint
RUN curl -fsSL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh \
| sh -s -- -b /usr/local/bin v1.43.0
RUN curl -fsSL -o /tmp/lint-install.sh https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh \
&& chmod +x /tmp/lint-install.sh \
&& /tmp/lint-install.sh -b /usr/local/bin v1.30.0 \
&& rm -f /tmp/lint-install.sh
# download modules
COPY go.* /src/
COPY go.* ./
RUN go mod download
COPY . /src/
# build
COPY . ./
RUN make build
# release stage
FROM alpine as release
ENTRYPOINT []
CMD ["/bin/bash"]
# runtime image
FROM alpine
RUN apk add --no-cache \
mariadb-client \
postgresql-client \
sqlite \
tzdata
COPY --from=dev /src/dist/dbmate /usr/local/bin/dbmate
ENTRYPOINT ["/usr/local/bin/dbmate"]
sqlite
COPY --from=build /src/dist/dbmate-linux-amd64 /usr/local/bin/dbmate
ENTRYPOINT ["dbmate"]

View file

@ -1,60 +1,59 @@
# enable cgo to build sqlite
export CGO_ENABLED = 1
# strip binaries
FLAGS := -tags sqlite_omit_load_extension,sqlite_json -ldflags '-s'
GOOS := $(shell go env GOOS)
ifeq ($(GOOS),linux)
# statically link binaries to support alpine linux
FLAGS := -tags netgo,osusergo,sqlite_omit_load_extension,sqlite_json -ldflags '-s -extldflags "-static"'
endif
ifeq ($(GOOS),darwin)
export SDKROOT ?= $(shell xcrun --sdk macosx --show-sdk-path)
endif
OUTPUT ?= dbmate
# no static linking for macos
LDFLAGS := -ldflags '-s'
# statically link binaries (to support alpine + scratch containers)
STATICLDFLAGS := -ldflags '-s -extldflags "-static"'
# avoid building code that is incompatible with static linking
TAGS := -tags netgo,osusergo,sqlite_omit_load_extension
.PHONY: all
all: fix build wait test
.PHONY: clean
clean:
rm -rf dist
.PHONY: build
build: clean
go build -o dist/$(OUTPUT) $(FLAGS) .
.PHONY: ls
ls:
ls -lh dist/$(OUTPUT)
file dist/$(OUTPUT)
all: build lint test
.PHONY: test
test:
go test -p 1 $(FLAGS) ./...
.PHONY: lint
lint:
golangci-lint run
go test -v $(TAGS) $(STATICLDFLAGS) ./...
.PHONY: fix
fix:
golangci-lint run --fix
.PHONY: lint
lint:
golangci-lint run
.PHONY: wait
wait:
dist/dbmate -e CLICKHOUSE_TEST_URL wait
dist/dbmate -e MYSQL_TEST_URL wait
dist/dbmate -e POSTGRES_TEST_URL wait
dist/dbmate-linux-amd64 -e MYSQL_URL wait
dist/dbmate-linux-amd64 -e POSTGRESQL_URL wait
dist/dbmate-linux-amd64 -e CLICKHOUSE_URL wait
.PHONY: docker-all
docker-all:
docker-compose pull
.PHONY: clean
clean:
rm -rf dist/*
.PHONY: build
build: clean build-linux-amd64
ls -lh dist
.PHONY: build-linux-amd64
build-linux-amd64:
GOOS=linux GOARCH=amd64 \
go build $(TAGS) $(STATICLDFLAGS) -o dist/dbmate-linux-amd64 .
.PHONY: build-all
build-all: clean build-linux-amd64
GOOS=linux GOARCH=arm64 CC=aarch64-linux-gnu-gcc-5 CXX=aarch64-linux-gnu-g++-5 \
go build $(TAGS) $(STATICLDFLAGS) -o dist/dbmate-linux-arm64 .
GOOS=darwin GOARCH=amd64 CC=o64-clang CXX=o64-clang++ \
go build $(TAGS) $(LDFLAGS) -o dist/dbmate-macos-amd64 .
GOOS=windows GOARCH=amd64 CC=x86_64-w64-mingw32-gcc-posix CXX=x86_64-w64-mingw32-g++-posix \
go build $(TAGS) $(STATICLDFLAGS) -o dist/dbmate-windows-amd64.exe .
ls -lh dist
.PHONY: docker-make
docker-make:
docker-compose build
docker-compose run --rm dev make all
docker-compose run --rm dbmate make
.PHONY: docker-sh
docker-sh:
-docker-compose run --rm dev
.PHONY: docker-bash
docker-bash:
-docker-compose run --rm dbmate bash

257
README.md
View file

@ -1,6 +1,6 @@
# Dbmate
[![GitHub Build](https://img.shields.io/github/workflow/status/amacneil/dbmate/CI/main)](https://github.com/amacneil/dbmate/actions?query=branch%3Amain+event%3Apush+workflow%3ACI)
[![Build Status](https://travis-ci.org/amacneil/dbmate.svg?branch=master)](https://travis-ci.org/amacneil/dbmate)
[![Go Report Card](https://goreportcard.com/badge/github.com/amacneil/dbmate)](https://goreportcard.com/report/github.com/amacneil/dbmate)
[![GitHub Release](https://img.shields.io/github/release/amacneil/dbmate.svg)](https://github.com/amacneil/dbmate/releases)
@ -10,40 +10,17 @@ It is a standalone command line tool, which can be used with Go, Node.js, Python
For a comparison between dbmate and other popular database schema migration tools, please see the [Alternatives](#alternatives) table.
## Table of Contents
- [Features](#features)
- [Installation](#installation)
- [Commands](#commands)
- [Command Line Options](#command-line-options)
- [Usage](#usage)
- [Connecting to the Database](#connecting-to-the-database)
- [PostgreSQL](#postgresql)
- [MySQL](#mysql)
- [SQLite](#sqlite)
- [ClickHouse](#clickhouse)
- [Creating Migrations](#creating-migrations)
- [Running Migrations](#running-migrations)
- [Rolling Back Migrations](#rolling-back-migrations)
- [Migration Options](#migration-options)
- [Waiting For The Database](#waiting-for-the-database)
- [Exporting Schema File](#exporting-schema-file)
- [Internals](#internals)
- [schema_migrations table](#schema_migrations-table)
- [Alternatives](#alternatives)
- [Contributing](#contributing)
## Features
- Supports MySQL, PostgreSQL, SQLite, and ClickHouse.
- Uses plain SQL for writing schema migrations.
- Migrations are timestamp-versioned, to avoid version number conflicts with multiple developers.
- Migrations are run atomically inside a transaction.
- Supports creating and dropping databases (handy in development/test).
- Supports saving a `schema.sql` file to easily diff schema changes in git.
- Database connection URL is definied using an environment variable (`DATABASE_URL` by default), or specified on the command line.
- Built-in support for reading environment variables from your `.env` file.
- Easy to distribute, single self-contained binary.
* Supports MySQL, PostgreSQL, SQLite, and ClickHouse.
* Uses plain SQL for writing schema migrations.
* Migrations are timestamp-versioned, to avoid version number conflicts with multiple developers.
* Migrations are run atomically inside a transaction.
* Supports creating and dropping databases (handy in development/test).
* Supports saving a `schema.sql` file to easily diff schema changes in git.
* Database connection URL is definied using an environment variable (`DATABASE_URL` by default), or specified on the command line.
* Built-in support for reading environment variables from your `.env` file.
* Easy to distribute, single self-contained binary.
## Installation
@ -66,18 +43,16 @@ $ sudo chmod +x /usr/local/bin/dbmate
**Docker**
Docker images are published to both Docker Hub ([`amacneil/dbmate`](https://hub.docker.com/r/amacneil/dbmate)) and Github Container Registry ([`ghcr.io/amacneil/dbmate`](https://ghcr.io/amacneil/dbmate)).
Remember to set `--network=host` or see [this comment](https://github.com/amacneil/dbmate/issues/128#issuecomment-615924611) for more tips on using dbmate with docker networking):
You can run dbmate using the official docker image (remember to set `--network=host` or see [this comment](https://github.com/amacneil/dbmate/issues/128#issuecomment-615924611) for more tips on using dbmate with docker networking):
```sh
$ docker run --rm -it --network=host ghcr.io/amacneil/dbmate:1 --help
$ docker run --rm --network=host -it amacneil/dbmate --help
```
If you wish to create or apply migrations, you will need to use Docker's [bind mount](https://docs.docker.com/storage/bind-mounts/) feature to make your local working directory (`pwd`) available inside the dbmate container:
If you wish to create or apply migrations, you will need to use Docker's [bind mount](https://docs.docker.com/storage/bind-mounts/) feature to make your local working directory available inside the dbmate container:
```sh
$ docker run --rm -it --network=host -v "$(pwd)/db:/db" ghcr.io/amacneil/dbmate:1 new create_users_table
$ docker run --rm -it -v "$(pwd)"/db:/db amacneil/dbmate new create_users_table
```
**Heroku**
@ -102,7 +77,7 @@ $ heroku run bin/dbmate up
## Commands
```sh
dbmate --help # print usage help
dbmate # print help
dbmate new # generate a new migration file
dbmate up # create the database (if it does not already exist) and run any pending migrations
dbmate create # create the database
@ -115,23 +90,8 @@ dbmate dump # write the database schema.sql file
dbmate wait # wait for the database server to become available
```
### Command Line Options
The following options are available with all commands. You must use command line arguments in the order `dbmate [global options] command [command options]`. Most options can also be configured via environment variables (and loaded from your `.env` file, which is helpful to share configuration between team members).
- `--url, -u "protocol://host:port/dbname"` - specify the database url directly. _(env: `$DATABASE_URL`)_
- `--env, -e "DATABASE_URL"` - specify an environment variable to read the database connection URL from.
- `--migrations-dir, -d "./db/migrations"` - where to keep the migration files. _(env: `$DBMATE_MIGRATIONS_DIR`)_
- `--migrations-table "schema_migrations"` - database table to record migrations in. _(env: `$DBMATE_MIGRATIONS_TABLE`)_
- `--schema-file, -s "./db/schema.sql"` - a path to keep the schema.sql file. _(env: `$DBMATE_SCHEMA_FILE`)_
- `--no-dump-schema` - don't auto-update the schema.sql file on migrate/rollback _(env: `$DBMATE_NO_DUMP_SCHEMA`)_
- `--wait` - wait for the db to become available before executing the subsequent command _(env: `$DBMATE_WAIT`)_
- `--wait-timeout 60s` - timeout for --wait flag _(env: `$DBMATE_WAIT_TIMEOUT`)_
## Usage
### Connecting to the Database
Dbmate locates your database using the `DATABASE_URL` environment variable by default. If you are writing a [twelve-factor app](http://12factor.net/), you should be storing all connection strings in environment variables.
To make this easy in development, dbmate looks for a `.env` file in the current directory, and treats any variables listed there as if they were specified in the current environment (existing environment variables take preference, however).
@ -149,37 +109,23 @@ DATABASE_URL="postgres://postgres@127.0.0.1:5432/myapp_development?sslmode=disab
protocol://username:password@host:port/database_name?options
```
- `protocol` must be one of `mysql`, `postgres`, `postgresql`, `sqlite`, `sqlite3`, `clickhouse`
- `host` can be either a hostname or IP address
- `options` are driver-specific (refer to the underlying Go SQL drivers if you wish to use these)
* `protocol` must be one of `mysql`, `postgres`, `postgresql`, `sqlite`, `sqlite3`, `clickhouse`
* `host` can be either a hostname or IP address
* `options` are driver-specific (refer to the underlying Go SQL drivers if you wish to use these)
Dbmate can also load the connection URL from a different environment variable. For example, before running your test suite, you may wish to drop and recreate the test database. One easy way to do this is to store your test database connection URL in the `TEST_DATABASE_URL` environment variable:
**MySQL**
```sh
$ cat .env
DATABASE_URL="postgres://postgres@127.0.0.1:5432/myapp_dev?sslmode=disable"
TEST_DATABASE_URL="postgres://postgres@127.0.0.1:5432/myapp_test?sslmode=disable"
DATABASE_URL="mysql://username:password@127.0.0.1:3306/database_name"
```
You can then specify this environment variable in your test script (Makefile or similar):
A `socket` parameter can be specified to connect through a unix socket:
```sh
$ dbmate -e TEST_DATABASE_URL drop
Dropping: myapp_test
$ dbmate -e TEST_DATABASE_URL --no-dump-schema up
Creating: myapp_test
Applying: 20151127184807_create_users_table.sql
DATABASE_URL="mysql://username:password@/database_name?socket=/var/run/mysqld/mysqld.sock"
```
Alternatively, you can specify the url directly on the command line:
```sh
$ dbmate -u "postgres://postgres@127.0.0.1:5432/myapp_test?sslmode=disable" up
```
The only advantage of using `dbmate -e TEST_DATABASE_URL` over `dbmate -u $TEST_DATABASE_URL` is that the former takes advantage of dbmate's automatic `.env` file loading.
#### PostgreSQL
**PostgreSQL**
When connecting to Postgres, you may need to add the `sslmode=disable` option to your connection string, as dbmate by default requires a TLS connection (some other frameworks/languages allow unencrypted connections by default).
@ -193,44 +139,21 @@ A `socket` or `host` parameter can be specified to connect through a unix socket
DATABASE_URL="postgres://username:password@/database_name?socket=/var/run/postgresql"
```
A `search_path` parameter can be used to specify the [current schema](https://www.postgresql.org/docs/13/ddl-schemas.html#DDL-SCHEMAS-PATH) while applying migrations, as well as for dbmate's `schema_migrations` table.
If the schema does not exist, it will be created automatically. If multiple comma-separated schemas are passed, the first will be used for the `schema_migrations` table.
**SQLite**
SQLite databases are stored on the filesystem, so you do not need to specify a host. By default, files are relative to the current directory. For example, the following will create a database at `./db/database_name.sqlite3`:
```sh
DATABASE_URL="postgres://username:password@127.0.0.1:5432/database_name?search_path=myschema"
DATABASE_URL="sqlite:///db/database_name.sqlite3"
```
To specify an absolute path, add an additional forward slash to the path. The following will create a database at `/tmp/database_name.sqlite3`:
```sh
DATABASE_URL="postgres://username:password@127.0.0.1:5432/database_name?search_path=myschema,public"
DATABASE_URL="sqlite:////tmp/database_name.sqlite3"
```
#### MySQL
```sh
DATABASE_URL="mysql://username:password@127.0.0.1:3306/database_name"
```
A `socket` parameter can be specified to connect through a unix socket:
```sh
DATABASE_URL="mysql://username:password@/database_name?socket=/var/run/mysqld/mysqld.sock"
```
#### SQLite
SQLite databases are stored on the filesystem, so you do not need to specify a host. By default, files are relative to the current directory. For example, the following will create a database at `./db/database.sqlite3`:
```sh
DATABASE_URL="sqlite:db/database.sqlite3"
```
To specify an absolute path, add a forward slash to the path. The following will create a database at `/tmp/database.sqlite3`:
```sh
DATABASE_URL="sqlite:/tmp/database.sqlite3"
```
#### ClickHouse
**ClickHouse**
```sh
DATABASE_URL="clickhouse://username:password@127.0.0.1:9000/database_name"
@ -282,14 +205,6 @@ Writing: ./db/schema.sql
> Note: `dbmate up` will create the database if it does not already exist (assuming the current user has permission to create databases). If you want to run migrations without creating the database, run `dbmate migrate`.
Pending migrations are always applied in numerical order. However, dbmate does not prevent migrations from being applied out of order if they are committed independently (for example: if a developer has been working on a branch for a long time, and commits a migration which has a lower version number than other already-applied migrations, dbmate will simply apply the pending migration). See [#159](https://github.com/amacneil/dbmate/issues/159) for a more detailed explanation.
You can also specify a migration to up-to.
```sh
$ dbmate up 20151127184807
```
### Rolling Back Migrations
By default, dbmate doesn't know how to roll back a migration. In development, it's often useful to be able to revert your database to a previous state. To accomplish this, implement the `migrate:down` section:
@ -314,21 +229,13 @@ Rolling back: 20151127184807_create_users_table.sql
Writing: ./db/schema.sql
```
You can also rollback to a specific migration.
```sh
$ dbmate rollback 20151127184807
# or, with a limit option
$ dbmate rollback -limit 2 # will rollback the last two migrations
```
### Migration Options
dbmate supports options passed to a migration block in the form of `key:value` pairs. List of supported options:
- `transaction`
* `transaction`
**transaction**
#### transaction
`transaction` is useful if you need to run some SQL which cannot be executed from within a transaction. For example, in Postgres, you would need to disable transactions for migrations that alter an enum type to add a value:
@ -339,6 +246,23 @@ ALTER TYPE colors ADD VALUE 'orange' AFTER 'red';
`transaction` will default to `true` if your database supports it.
### Schema File
When you run the `up`, `migrate`, or `rollback` commands, dbmate will automatically create a `./db/schema.sql` file containing a complete representation of your database schema. Dbmate keeps this file up to date for you, so you should not manually edit it.
It is recommended to check this file into source control, so that you can easily review changes to the schema in commits or pull requests. It's also possible to use this file when you want to quickly load a database schema, without running each migration sequentially (for example in your test harness). However, if you do not wish to save this file, you could add it to `.gitignore`, or pass the `--no-dump-schema` command line option.
To dump the `schema.sql` file without performing any other actions, run `dbmate dump`. Unlike other dbmate actions, this command relies on the respective `pg_dump`, `mysqldump`, or `sqlite3` commands being available in your PATH. If these tools are not available, dbmate will silenty skip the schema dump step during `up`, `migrate`, or `rollback` actions. You can diagnose the issue by running `dbmate dump` and looking at the output:
```sh
$ dbmate dump
exec: "pg_dump": executable file not found in $PATH
```
On Ubuntu or Debian systems, you can fix this by installing `postgresql-client`, `mysql-client`, or `sqlite3` respectively. Ensure that the package version you install is greater than or equal to the version running on your database server.
> Note: The `schema.sql` file will contain a complete schema for your database, even if some tables or columns were created outside of dbmate migrations.
### Waiting For The Database
If you use a Docker development environment for your project, you may encounter issues with the database not being immediately ready when running migrations or unit tests. This can be due to the database server having only just started.
@ -376,65 +300,68 @@ Error: unable to connect to database: dial tcp 127.0.0.1:5432: connect: connecti
Please note that the `wait` command does not verify whether your specified database exists, only that the server is available and ready (so it will return success if the database server is available, but your database has not yet been created).
### Exporting Schema File
### Options
When you run the `up`, `migrate`, or `rollback` commands, dbmate will automatically create a `./db/schema.sql` file containing a complete representation of your database schema. Dbmate keeps this file up to date for you, so you should not manually edit it.
The following command line options are available with all commands. You must use command line arguments in the order `dbmate [global options] command [command options]`. Most options can also be configured via environment variables (and loaded from your `.env` file, which is helpful to share configuration between team members).
It is recommended to check this file into source control, so that you can easily review changes to the schema in commits or pull requests. It's also possible to use this file when you want to quickly load a database schema, without running each migration sequentially (for example in your test harness). However, if you do not wish to save this file, you could add it to your `.gitignore`, or pass the `--no-dump-schema` command line option.
* `--url, -u "protocol://host:port/dbname"` - specify the database url directly. _(env: `$DATABASE_URL`)_
* `--env, -e "DATABASE_URL"` - specify an environment variable to read the database connection URL from.
* `--migrations-dir, -d "./db/migrations"` - where to keep the migration files. _(env: `$DBMATE_MIGRATIONS_DIR`)_
* `--schema-file, -s "./db/schema.sql"` - a path to keep the schema.sql file. _(env: `$DBMATE_SCHEMA_FILE`)_
* `--no-dump-schema` - don't auto-update the schema.sql file on migrate/rollback _(env: `$DBMATE_NO_DUMP_SCHEMA`)_
* `--wait` - wait for the db to become available before executing the subsequent command _(env: `$DBMATE_WAIT`)_
* `--wait-timeout 60s` - timeout for --wait flag _(env: `$DBMATE_WAIT_TIMEOUT`)_
To dump the `schema.sql` file without performing any other actions, run `dbmate dump`. Unlike other dbmate actions, this command relies on the respective `pg_dump`, `mysqldump`, or `sqlite3` commands being available in your PATH. If these tools are not available, dbmate will silenty skip the schema dump step during `up`, `migrate`, or `rollback` actions. You can diagnose the issue by running `dbmate dump` and looking at the output:
For example, before running your test suite, you may wish to drop and recreate the test database. One easy way to do this is to store your test database connection URL in the `TEST_DATABASE_URL` environment variable:
```sh
$ dbmate dump
exec: "pg_dump": executable file not found in $PATH
$ cat .env
TEST_DATABASE_URL="postgres://postgres@127.0.0.1:5432/myapp_test?sslmode=disable"
```
On Ubuntu or Debian systems, you can fix this by installing `postgresql-client`, `mysql-client`, or `sqlite3` respectively. Ensure that the package version you install is greater than or equal to the version running on your database server.
You can then specify this environment variable in your test script (Makefile or similar):
> Note: The `schema.sql` file will contain a complete schema for your database, even if some tables or columns were created outside of dbmate migrations.
## Internals
### schema_migrations table
By default, dbmate stores a record of each applied migration in a `schema_migrations` table. This table will be created for you automatically if it does not already exist. The table schema is very simple:
```sql
CREATE TABLE IF NOT EXISTS schema_migrations (
version VARCHAR(255) PRIMARY KEY
)
```sh
$ dbmate -e TEST_DATABASE_URL drop
Dropping: myapp_test
$ dbmate -e TEST_DATABASE_URL --no-dump-schema up
Creating: myapp_test
Applying: 20151127184807_create_users_table.sql
```
Dbmate records only the version number of applied migrations, so you can safely rename a migration file without affecting its applied status.
Alternatively, you can specify the url directly on the command line:
You can customize the name of this table using the `--migrations-table` flag or `$DBMATE_MIGRATIONS_TABLE` environment variable. If you already have a table with this name (possibly from a previous migration tool), you should either manually update it to conform to this schema, or configure dbmate to use a different table name.
```sh
$ dbmate -u "postgres://postgres@127.0.0.1:5432/myapp_test?sslmode=disable" up
```
The only advantage of using `dbmate -e TEST_DATABASE_URL` over `dbmate -u $TEST_DATABASE_URL` is that the former takes advantage of dbmate's automatic `.env` file loading.
## Alternatives
Why another database schema migration tool? Dbmate was inspired by many other tools, primarily [Active Record Migrations](http://guides.rubyonrails.org/active_record_migrations.html), with the goals of being trivial to configure, and language & framework independent. Here is a comparison between dbmate and other popular migration tools.
| | [dbmate](https://github.com/amacneil/dbmate) | [goose](https://github.com/pressly/goose) | [sql-migrate](https://github.com/rubenv/sql-migrate) | [golang-migrate](https://github.com/golang-migrate/migrate) | [activerecord](http://guides.rubyonrails.org/active_record_migrations.html) | [sequelize](http://docs.sequelizejs.com/manual/tutorial/migrations.html) |
| ------------------------------------------------------------ | :------------------------------------------: | :---------------------------------------: | :--------------------------------------------------: | :---------------------------------------------------------: | :-------------------------------------------------------------------------: | :----------------------------------------------------------------------: |
| | [goose](https://bitbucket.org/liamstask/goose/) | [sql-migrate](https://github.com/rubenv/sql-migrate) | [golang-migrate/migrate](https://github.com/golang-migrate/migrate) | [activerecord](http://guides.rubyonrails.org/active_record_migrations.html) | [sequelize](http://docs.sequelizejs.com/manual/tutorial/migrations.html) | [dbmate](https://github.com/amacneil/dbmate) |
| --- |:---:|:---:|:---:|:---:|:---:|:---:|
| **Features** |
| Plain SQL migration files | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | | |
| Support for creating and dropping databases | :white_check_mark: | | | | :white_check_mark: | |
| Support for saving schema dump files | :white_check_mark: | | | | :white_check_mark: | |
| Timestamp-versioned migration files | :white_check_mark: | :white_check_mark: | | :white_check_mark: | :white_check_mark: | :white_check_mark: |
| Custom schema migrations table | :white_check_mark: | | :white_check_mark: | | | :white_check_mark: |
| Ability to wait for database to become ready | :white_check_mark: | | | | | |
| Database connection string loaded from environment variables | :white_check_mark: | | | | | |
| Automatically load .env file | :white_check_mark: | | | | | |
| No separate configuration file | :white_check_mark: | | | :white_check_mark: | :white_check_mark: | :white_check_mark: |
| Language/framework independent | :white_check_mark: | :eight_pointed_black_star: | :eight_pointed_black_star: | :white_check_mark: | | |
|Plain SQL migration files|:white_check_mark:|:white_check_mark:|:white_check_mark:|||:white_check_mark:|
|Support for creating and dropping databases||||:white_check_mark:||:white_check_mark:|
|Support for saving schema dump files||||:white_check_mark:||:white_check_mark:|
|Timestamp-versioned migration files|:white_check_mark:|||:white_check_mark:|:white_check_mark:|:white_check_mark:|
|Ability to wait for database to become ready||||||:white_check_mark:|
|Database connection string loaded from environment variables||||||:white_check_mark:|
|Automatically load .env file||||||:white_check_mark:|
|No separate configuration file||||:white_check_mark:|:white_check_mark:|:white_check_mark:|
|Language/framework independent|:eight_pointed_black_star:|:eight_pointed_black_star:|:eight_pointed_black_star:|||:white_check_mark:|
| **Drivers** |
| PostgreSQL | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: |
| MySQL | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: |
| SQLite | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: |
| CliсkHouse | :white_check_mark: | | | :white_check_mark: | :white_check_mark: | :white_check_mark: |
|PostgreSQL|:white_check_mark:|:white_check_mark:|:white_check_mark:|:white_check_mark:|:white_check_mark:|:white_check_mark:|
|MySQL|:white_check_mark:|:white_check_mark:|:white_check_mark:|:white_check_mark:|:white_check_mark:|:white_check_mark:|
|SQLite|:white_check_mark:|:white_check_mark:|:white_check_mark:|:white_check_mark:|:white_check_mark:|:white_check_mark:|
|CliсkHouse|||:white_check_mark:|:white_check_mark:|:white_check_mark:|:white_check_mark:|
> :eight_pointed_black_star: In theory these tools could be used with other languages, but a Go development environment is required because binary builds are not provided.
_If you notice any inaccuracies in this table, please [propose a change](https://github.com/amacneil/dbmate/edit/main/README.md)._
*If you notice any inaccuracies in this table, please [propose a change](https://github.com/amacneil/dbmate/edit/master/README.md).*
## Contributing
@ -449,5 +376,5 @@ $ make docker-all
To start a development shell:
```sh
$ make docker-sh
$ make docker-bash
```

View file

@ -3,5 +3,6 @@
The following steps should be followed to publish a new version of dbmate (requires write access to this repository).
1. Update [version.go](/pkg/dbmate/version.go) with new version number ([example PR](https://github.com/amacneil/dbmate/pull/146/files))
2. Create new release on [releases page](https://github.com/amacneil/dbmate/releases) and write release notes
3. GitHub Actions will automatically publish release binaries and submit Homebrew PR
2. Create new release on GitHub project [releases page](https://github.com/amacneil/dbmate/releases)
3. Travis CI will automatically publish release binaries to GitHub
4. GitHub Actions will automatically create PR to update Homebrew package

View file

@ -0,0 +1,7 @@
# this file is used to mount the current directory as a volume
# we remove it in CI to use files inside the container only
version: '2.3'
services:
dbmate:
volumes:
- .:/src

View file

@ -1,31 +1,21 @@
version: "2.3"
version: '2.3'
services:
dev:
dbmate:
build:
context: .
target: dev
volumes:
- .:/src
target: build
depends_on:
- mysql
- postgres
- clickhouse
environment:
CLICKHOUSE_TEST_URL: clickhouse://clickhouse:9000?database=dbmate_test
MYSQL_TEST_URL: mysql://root:root@mysql/dbmate_test
POSTGRES_TEST_URL: postgres://postgres:postgres@postgres/dbmate_test?sslmode=disable
SQLITE_TEST_URL: sqlite3:/tmp/dbmate_test.sqlite3
dbmate:
build:
context: .
target: release
image: dbmate_release
MYSQL_URL: mysql://root:root@mysql/dbmate
POSTGRESQL_URL: postgres://postgres:postgres@postgres/dbmate?sslmode=disable
CLICKHOUSE_URL: clickhouse://clickhouse:9000?database=dbmate
mysql:
image: mysql/mysql-server:8.0
image: mysql:5.7
environment:
MYSQL_ROOT_HOST: "%"
MYSQL_ROOT_PASSWORD: root
postgres:

22
go.mod
View file

@ -1,22 +1,18 @@
module github.com/amacneil/dbmate
go 1.16
go 1.14
require (
github.com/ClickHouse/clickhouse-go v1.4.3
github.com/ClickHouse/clickhouse-go v1.4.1
github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/frankban/quicktest v1.11.3 // indirect
github.com/go-sql-driver/mysql v1.6.0
github.com/go-sql-driver/mysql v1.5.0
github.com/joho/godotenv v1.3.0
github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d
github.com/kr/text v0.2.0 // indirect
github.com/lib/pq v1.10.0
github.com/mattn/go-sqlite3 v1.14.6
github.com/pierrec/lz4 v2.6.0+incompatible // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/stretchr/testify v1.7.0
github.com/urfave/cli/v2 v2.3.0
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
github.com/kr/pretty v0.1.0 // indirect
github.com/lib/pq v1.8.0
github.com/mattn/go-sqlite3 v1.14.0
github.com/stretchr/testify v1.4.0
github.com/urfave/cli/v2 v2.2.0
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
)

64
go.sum
View file

@ -1,6 +1,8 @@
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/ClickHouse/clickhouse-go v1.4.3 h1:iAFMa2UrQdR5bHJ2/yaSLffZkxpcOYQMCUuKeNXGdqc=
github.com/ClickHouse/clickhouse-go v1.4.3/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI=
github.com/ClickHouse/clickhouse-go v1.4.1 h1:D9cihLg76O1ZyILLaXq1eksYzEuV010NdvucgKGGK14=
github.com/ClickHouse/clickhouse-go v1.4.1/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI=
github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
github.com/bkaradzic/go-lz4 v1.0.0 h1:RXc4wYsyz985CkXXeX04y4VnZFGG8Rd43pRaHsOXAKk=
github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4=
github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 h1:F1EaeKL/ta07PY/k9Os/UFtwERei2/XzGemhpGnBKNg=
@ -8,55 +10,51 @@ github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM=
github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/frankban/quicktest v1.11.3 h1:8sXhOn0uLys67V8EsXLc6eszDs8VXWxL3iRvebPhedY=
github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M=
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc=
github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d h1:cVtBfNW5XTHiKQe7jDaDBSh/EVM4XLPutLAGboIXuM0=
github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0=
github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/lib/pq v1.10.0 h1:Zx5DJFEYQXio93kgXnQ09fXNiUKsqv4OUEu2UtGcB1E=
github.com/lib/pq v1.10.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/lib/pq v1.8.0 h1:9xohqzkUwzR4Ga4ivdTcawVS89YSDVxXMa3xJX3cGzg=
github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
github.com/mattn/go-sqlite3 v1.14.6 h1:dNPt6NO46WmLVt2DLNpwczCmdV5boIZ6g/tlDrlRUbg=
github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
github.com/mattn/go-sqlite3 v1.14.0 h1:mLyGNKR8+Vv9CAU7PphKa2hkEqxxhn8i32J6FPj1/QA=
github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus=
github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I=
github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
github.com/pierrec/lz4 v2.6.0+incompatible h1:Ix9yFKn1nSPBLFl/yZknTp8TU5G4Ps0JDmguYK6iH1A=
github.com/pierrec/lz4 v2.6.0+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M=
github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/urfave/cli/v2 v2.2.0 h1:JTTnM6wKzdA0Jqodd966MVj4vWbbquZykeX1sKbe2C4=
github.com/urfave/cli/v2 v2.2.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

25
main.go
View file

@ -11,10 +11,6 @@ import (
"github.com/urfave/cli/v2"
"github.com/amacneil/dbmate/pkg/dbmate"
_ "github.com/amacneil/dbmate/pkg/driver/clickhouse"
_ "github.com/amacneil/dbmate/pkg/driver/mysql"
_ "github.com/amacneil/dbmate/pkg/driver/postgres"
_ "github.com/amacneil/dbmate/pkg/driver/sqlite"
)
func main() {
@ -56,12 +52,6 @@ func NewApp() *cli.App {
Value: dbmate.DefaultMigrationsDir,
Usage: "specify the directory containing migration files",
},
&cli.StringFlag{
Name: "migrations-table",
EnvVars: []string{"DBMATE_MIGRATIONS_TABLE"},
Value: dbmate.DefaultMigrationsTableName,
Usage: "specify the database table to record migrations in",
},
&cli.StringFlag{
Name: "schema-file",
Aliases: []string{"s"},
@ -109,7 +99,6 @@ func NewApp() *cli.App {
},
},
Action: action(func(db *dbmate.DB, c *cli.Context) error {
db.TargetVersion = c.Args().First()
db.Verbose = c.Bool("verbose")
return db.CreateAndMigrate()
}),
@ -130,7 +119,7 @@ func NewApp() *cli.App {
},
{
Name: "migrate",
Usage: "Migrate to the specified or latest version",
Usage: "Migrate to the latest version",
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "verbose",
@ -140,7 +129,6 @@ func NewApp() *cli.App {
},
},
Action: action(func(db *dbmate.DB, c *cli.Context) error {
db.TargetVersion = c.Args().First()
db.Verbose = c.Bool("verbose")
return db.Migrate()
}),
@ -156,16 +144,8 @@ func NewApp() *cli.App {
EnvVars: []string{"DBMATE_VERBOSE"},
Usage: "print the result of each statement execution",
},
&cli.IntFlag{
Name: "limit",
Aliases: []string{"l"},
Usage: "Limits the amount of rollbacks (defaults to 1 if no target version is specified)",
Value: -1,
},
},
Action: action(func(db *dbmate.DB, c *cli.Context) error {
db.TargetVersion = c.Args().First()
db.Limit = c.Int("limit")
db.Verbose = c.Bool("verbose")
return db.Rollback()
}),
@ -196,7 +176,7 @@ func NewApp() *cli.App {
}
if pending > 0 && setExitCode {
return cli.Exit("", 1)
return cli.NewExitError("", 1)
}
return nil
@ -242,7 +222,6 @@ func action(f func(*dbmate.DB, *cli.Context) error) cli.ActionFunc {
db := dbmate.New(u)
db.AutoDumpSchema = !c.Bool("no-dump-schema")
db.MigrationsDir = c.String("migrations-dir")
db.MigrationsTableName = c.String("migrations-table")
db.SchemaFile = c.String("schema-file")
db.WaitBefore = c.Bool("wait")
overrideTimeout := c.Duration("wait-timeout")

289
pkg/dbmate/clickhouse.go Normal file
View file

@ -0,0 +1,289 @@
package dbmate
import (
"bytes"
"database/sql"
"fmt"
"net/url"
"regexp"
"sort"
"strings"
"github.com/ClickHouse/clickhouse-go"
)
func init() {
RegisterDriver(ClickHouseDriver{}, "clickhouse")
}
// ClickHouseDriver provides top level database functions
type ClickHouseDriver struct {
}
func normalizeClickHouseURL(initialURL *url.URL) *url.URL {
u := *initialURL
u.Scheme = "tcp"
host := u.Host
if u.Port() == "" {
host = fmt.Sprintf("%s:9000", host)
}
u.Host = host
query := u.Query()
if query.Get("username") == "" && u.User.Username() != "" {
query.Set("username", u.User.Username())
}
password, passwordSet := u.User.Password()
if query.Get("password") == "" && passwordSet {
query.Set("password", password)
}
u.User = nil
if query.Get("database") == "" {
path := strings.Trim(u.Path, "/")
if path != "" {
query.Set("database", path)
u.Path = ""
}
}
u.RawQuery = query.Encode()
return &u
}
// Open creates a new database connection
func (drv ClickHouseDriver) Open(u *url.URL) (*sql.DB, error) {
return sql.Open("clickhouse", normalizeClickHouseURL(u).String())
}
func (drv ClickHouseDriver) openClickHouseDB(u *url.URL) (*sql.DB, error) {
// connect to clickhouse database
clickhouseURL := normalizeClickHouseURL(u)
values := clickhouseURL.Query()
values.Set("database", "default")
clickhouseURL.RawQuery = values.Encode()
return drv.Open(clickhouseURL)
}
func (drv ClickHouseDriver) databaseName(u *url.URL) string {
name := normalizeClickHouseURL(u).Query().Get("database")
if name == "" {
name = "default"
}
return name
}
var clickhouseValidIdentifier = regexp.MustCompile(`^[a-zA-Z_][0-9a-zA-Z_]*$`)
func clickhouseQuoteIdentifier(str string) string {
if clickhouseValidIdentifier.MatchString(str) {
return str
}
str = strings.Replace(str, `"`, `""`, -1)
return fmt.Sprintf(`"%s"`, str)
}
// CreateDatabase creates the specified database
func (drv ClickHouseDriver) CreateDatabase(u *url.URL) error {
name := drv.databaseName(u)
fmt.Printf("Creating: %s\n", name)
db, err := drv.openClickHouseDB(u)
if err != nil {
return err
}
defer mustClose(db)
_, err = db.Exec("create database " + clickhouseQuoteIdentifier(name))
return err
}
// DropDatabase drops the specified database (if it exists)
func (drv ClickHouseDriver) DropDatabase(u *url.URL) error {
name := drv.databaseName(u)
fmt.Printf("Dropping: %s\n", name)
db, err := drv.openClickHouseDB(u)
if err != nil {
return err
}
defer mustClose(db)
_, err = db.Exec("drop database if exists " + clickhouseQuoteIdentifier(name))
return err
}
func clickhouseSchemaDump(db *sql.DB, buf *bytes.Buffer, databaseName string) error {
buf.WriteString("\n--\n-- Database schema\n--\n\n")
buf.WriteString("CREATE DATABASE " + clickhouseQuoteIdentifier(databaseName) + " IF NOT EXISTS;\n\n")
tables, err := queryColumn(db, "show tables")
if err != nil {
return err
}
sort.Strings(tables)
for _, table := range tables {
var clause string
err = db.QueryRow("show create table " + clickhouseQuoteIdentifier(table)).Scan(&clause)
if err != nil {
return err
}
buf.WriteString(clause + ";\n\n")
}
return nil
}
func clickhouseSchemaMigrationsDump(db *sql.DB, buf *bytes.Buffer) error {
// load applied migrations
migrations, err := queryColumn(db,
"select version from schema_migrations final where applied order by version asc",
)
if err != nil {
return err
}
quoter := strings.NewReplacer(`\`, `\\`, `'`, `\'`)
for i := range migrations {
migrations[i] = "'" + quoter.Replace(migrations[i]) + "'"
}
// build schema_migrations table data
buf.WriteString("\n--\n-- Dbmate schema migrations\n--\n\n")
if len(migrations) > 0 {
buf.WriteString("INSERT INTO schema_migrations (version) VALUES\n (" +
strings.Join(migrations, "),\n (") +
");\n")
}
return nil
}
// DumpSchema returns the current database schema
func (drv ClickHouseDriver) DumpSchema(u *url.URL, db *sql.DB) ([]byte, error) {
var buf bytes.Buffer
var err error
err = clickhouseSchemaDump(db, &buf, drv.databaseName(u))
if err != nil {
return nil, err
}
err = clickhouseSchemaMigrationsDump(db, &buf)
if err != nil {
return nil, err
}
return buf.Bytes(), nil
}
// DatabaseExists determines whether the database exists
func (drv ClickHouseDriver) DatabaseExists(u *url.URL) (bool, error) {
name := drv.databaseName(u)
db, err := drv.openClickHouseDB(u)
if err != nil {
return false, err
}
defer mustClose(db)
exists := false
err = db.QueryRow("SELECT 1 FROM system.databases where name = ?", name).
Scan(&exists)
if err == sql.ErrNoRows {
return false, nil
}
return exists, err
}
// CreateMigrationsTable creates the schema_migrations table
func (drv ClickHouseDriver) CreateMigrationsTable(db *sql.DB) error {
_, err := db.Exec(`
create table if not exists schema_migrations (
version String,
ts DateTime default now(),
applied UInt8 default 1
) engine = ReplacingMergeTree(ts)
primary key version
order by version
`)
return err
}
// SelectMigrations returns a list of applied migrations
// with an optional limit (in descending order)
func (drv ClickHouseDriver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, error) {
query := "select version from schema_migrations final where applied order by version desc"
if limit >= 0 {
query = fmt.Sprintf("%s limit %d", query, limit)
}
rows, err := db.Query(query)
if err != nil {
return nil, err
}
defer mustClose(rows)
migrations := map[string]bool{}
for rows.Next() {
var version string
if err := rows.Scan(&version); err != nil {
return nil, err
}
migrations[version] = true
}
return migrations, nil
}
// InsertMigration adds a new migration record
func (drv ClickHouseDriver) InsertMigration(db Transaction, version string) error {
_, err := db.Exec("insert into schema_migrations (version) values (?)", version)
return err
}
// DeleteMigration removes a migration record
func (drv ClickHouseDriver) DeleteMigration(db Transaction, version string) error {
_, err := db.Exec(
"insert into schema_migrations (version, applied) values (?, ?)",
version, false,
)
return err
}
// Ping verifies a connection to the database server. It does not verify whether the
// specified database exists.
func (drv ClickHouseDriver) Ping(u *url.URL) error {
// attempt connection to primary database, not "clickhouse" database
// to support servers with no "clickhouse" database
// (see https://github.com/amacneil/dbmate/issues/78)
db, err := drv.Open(u)
if err != nil {
return err
}
defer mustClose(db)
err = db.Ping()
if err == nil {
return nil
}
// ignore 'Database foo doesn't exist' error
chErr, ok := err.(*clickhouse.Exception)
if ok && chErr.Code == 81 {
return nil
}
return err
}

View file

@ -0,0 +1,305 @@
package dbmate
import (
"database/sql"
"net/url"
"testing"
"github.com/stretchr/testify/require"
)
func clickhouseTestURL(t *testing.T) *url.URL {
u, err := url.Parse("clickhouse://clickhouse:9000?database=dbmate")
require.NoError(t, err)
return u
}
func prepTestClickHouseDB(t *testing.T) *sql.DB {
drv := ClickHouseDriver{}
u := clickhouseTestURL(t)
// drop any existing database
err := drv.DropDatabase(u)
require.NoError(t, err)
// create database
err = drv.CreateDatabase(u)
require.NoError(t, err)
// connect database
db, err := sql.Open("clickhouse", u.String())
require.NoError(t, err)
return db
}
func TestNormalizeClickHouseURLSimplified(t *testing.T) {
u, err := url.Parse("clickhouse://user:pass@host/db")
require.NoError(t, err)
s := normalizeClickHouseURL(u).String()
require.Equal(t, "tcp://host:9000?database=db&password=pass&username=user", s)
}
func TestNormalizeClickHouseURLCanonical(t *testing.T) {
u, err := url.Parse("clickhouse://host:9000?database=db&password=pass&username=user")
require.NoError(t, err)
s := normalizeClickHouseURL(u).String()
require.Equal(t, "tcp://host:9000?database=db&password=pass&username=user", s)
}
func TestClickHouseCreateDropDatabase(t *testing.T) {
drv := ClickHouseDriver{}
u := clickhouseTestURL(t)
// drop any existing database
err := drv.DropDatabase(u)
require.NoError(t, err)
// create database
err = drv.CreateDatabase(u)
require.NoError(t, err)
// check that database exists and we can connect to it
func() {
db, err := sql.Open("clickhouse", u.String())
require.NoError(t, err)
defer mustClose(db)
err = db.Ping()
require.NoError(t, err)
}()
// drop the database
err = drv.DropDatabase(u)
require.NoError(t, err)
// check that database no longer exists
func() {
db, err := sql.Open("clickhouse", u.String())
require.NoError(t, err)
defer mustClose(db)
err = db.Ping()
require.EqualError(t, err, "code: 81, message: Database dbmate doesn't exist")
}()
}
func TestClickHouseDumpSchema(t *testing.T) {
drv := ClickHouseDriver{}
u := clickhouseTestURL(t)
// prepare database
db := prepTestClickHouseDB(t)
defer mustClose(db)
err := drv.CreateMigrationsTable(db)
require.NoError(t, err)
// insert migration
tx, err := db.Begin()
require.NoError(t, err)
err = drv.InsertMigration(tx, "abc1")
require.NoError(t, err)
err = tx.Commit()
require.NoError(t, err)
tx, err = db.Begin()
require.NoError(t, err)
err = drv.InsertMigration(tx, "abc2")
require.NoError(t, err)
err = tx.Commit()
require.NoError(t, err)
// DumpSchema should return schema
schema, err := drv.DumpSchema(u, db)
require.NoError(t, err)
require.Contains(t, string(schema), "CREATE TABLE "+drv.databaseName(u)+".schema_migrations")
require.Contains(t, string(schema), "--\n"+
"-- Dbmate schema migrations\n"+
"--\n\n"+
"INSERT INTO schema_migrations (version) VALUES\n"+
" ('abc1'),\n"+
" ('abc2');\n")
// DumpSchema should return error if command fails
values := u.Query()
values.Set("database", "fakedb")
u.RawQuery = values.Encode()
db, err = sql.Open("clickhouse", u.String())
require.NoError(t, err)
schema, err = drv.DumpSchema(u, db)
require.Nil(t, schema)
require.EqualError(t, err, "code: 81, message: Database fakedb doesn't exist")
}
func TestClickHouseDatabaseExists(t *testing.T) {
drv := ClickHouseDriver{}
u := clickhouseTestURL(t)
// drop any existing database
err := drv.DropDatabase(u)
require.NoError(t, err)
// DatabaseExists should return false
exists, err := drv.DatabaseExists(u)
require.NoError(t, err)
require.Equal(t, false, exists)
// create database
err = drv.CreateDatabase(u)
require.NoError(t, err)
// DatabaseExists should return true
exists, err = drv.DatabaseExists(u)
require.NoError(t, err)
require.Equal(t, true, exists)
}
func TestClickHouseDatabaseExists_Error(t *testing.T) {
drv := ClickHouseDriver{}
u := clickhouseTestURL(t)
values := u.Query()
values.Set("username", "invalid")
u.RawQuery = values.Encode()
exists, err := drv.DatabaseExists(u)
require.EqualError(t, err, "code: 192, message: Unknown user invalid")
require.Equal(t, false, exists)
}
func TestClickHouseCreateMigrationsTable(t *testing.T) {
drv := ClickHouseDriver{}
db := prepTestClickHouseDB(t)
defer mustClose(db)
// migrations table should not exist
count := 0
err := db.QueryRow("select count(*) from schema_migrations").Scan(&count)
require.EqualError(t, err, "code: 60, message: Table dbmate.schema_migrations doesn't exist.")
// create table
err = drv.CreateMigrationsTable(db)
require.NoError(t, err)
// migrations table should exist
err = db.QueryRow("select count(*) from schema_migrations").Scan(&count)
require.NoError(t, err)
// create table should be idempotent
err = drv.CreateMigrationsTable(db)
require.NoError(t, err)
}
func TestClickHouseSelectMigrations(t *testing.T) {
drv := ClickHouseDriver{}
db := prepTestClickHouseDB(t)
defer mustClose(db)
err := drv.CreateMigrationsTable(db)
require.NoError(t, err)
tx, err := db.Begin()
require.NoError(t, err)
stmt, err := tx.Prepare("insert into schema_migrations (version) values (?)")
require.NoError(t, err)
_, err = stmt.Exec("abc2")
require.NoError(t, err)
_, err = stmt.Exec("abc1")
require.NoError(t, err)
_, err = stmt.Exec("abc3")
require.NoError(t, err)
err = tx.Commit()
require.NoError(t, err)
migrations, err := drv.SelectMigrations(db, -1)
require.NoError(t, err)
require.Equal(t, true, migrations["abc1"])
require.Equal(t, true, migrations["abc2"])
require.Equal(t, true, migrations["abc2"])
// test limit param
migrations, err = drv.SelectMigrations(db, 1)
require.NoError(t, err)
require.Equal(t, true, migrations["abc3"])
require.Equal(t, false, migrations["abc1"])
require.Equal(t, false, migrations["abc2"])
}
func TestClickHouseInsertMigration(t *testing.T) {
drv := ClickHouseDriver{}
db := prepTestClickHouseDB(t)
defer mustClose(db)
err := drv.CreateMigrationsTable(db)
require.NoError(t, err)
count := 0
err = db.QueryRow("select count(*) from schema_migrations").Scan(&count)
require.NoError(t, err)
require.Equal(t, 0, count)
// insert migration
tx, err := db.Begin()
require.NoError(t, err)
err = drv.InsertMigration(tx, "abc1")
require.NoError(t, err)
err = tx.Commit()
require.NoError(t, err)
err = db.QueryRow("select count(*) from schema_migrations where version = 'abc1'").Scan(&count)
require.NoError(t, err)
require.Equal(t, 1, count)
}
func TestClickHouseDeleteMigration(t *testing.T) {
drv := ClickHouseDriver{}
db := prepTestClickHouseDB(t)
defer mustClose(db)
err := drv.CreateMigrationsTable(db)
require.NoError(t, err)
tx, err := db.Begin()
require.NoError(t, err)
stmt, err := tx.Prepare("insert into schema_migrations (version) values (?)")
require.NoError(t, err)
_, err = stmt.Exec("abc2")
require.NoError(t, err)
_, err = stmt.Exec("abc1")
require.NoError(t, err)
err = tx.Commit()
require.NoError(t, err)
tx, err = db.Begin()
require.NoError(t, err)
err = drv.DeleteMigration(tx, "abc2")
require.NoError(t, err)
err = tx.Commit()
require.NoError(t, err)
count := 0
err = db.QueryRow("select count(*) from schema_migrations final where applied").Scan(&count)
require.NoError(t, err)
require.Equal(t, 1, count)
}
func TestClickHousePing(t *testing.T) {
drv := ClickHouseDriver{}
u := clickhouseTestURL(t)
// drop any existing database
err := drv.DropDatabase(u)
require.NoError(t, err)
// ping database
err = drv.Ping(u)
require.NoError(t, err)
// ping invalid host should return error
u.Host = "clickhouse:404"
err = drv.Ping(u)
require.Error(t, err)
require.Contains(t, err.Error(), "connect: connection refused")
}

View file

@ -2,25 +2,19 @@ package dbmate
import (
"database/sql"
"errors"
"fmt"
"io"
"io/ioutil"
"net/url"
"os"
"path/filepath"
"regexp"
"sort"
"time"
"github.com/amacneil/dbmate/pkg/dbutil"
)
// DefaultMigrationsDir specifies default directory to find migration files
const DefaultMigrationsDir = "./db/migrations"
// DefaultMigrationsTableName specifies default database tables to record migraitons in
const DefaultMigrationsTableName = "schema_migrations"
// DefaultSchemaFile specifies default location for schema.sql
const DefaultSchemaFile = "./db/schema.sql"
@ -35,24 +29,19 @@ type DB struct {
AutoDumpSchema bool
DatabaseURL *url.URL
MigrationsDir string
MigrationsTableName string
SchemaFile string
Verbose bool
WaitBefore bool
WaitInterval time.Duration
WaitTimeout time.Duration
Limit int
TargetVersion string
Log io.Writer
}
// migrationFileRegexp pattern for valid migration files
var migrationFileRegexp = regexp.MustCompile(`^\d.*\.sql$`)
// StatusResult represents an available migration status
type StatusResult struct {
Filename string
Applied bool
type statusResult struct {
filename string
applied bool
}
// New initializes a new dbmate database
@ -61,35 +50,16 @@ func New(databaseURL *url.URL) *DB {
AutoDumpSchema: true,
DatabaseURL: databaseURL,
MigrationsDir: DefaultMigrationsDir,
MigrationsTableName: DefaultMigrationsTableName,
SchemaFile: DefaultSchemaFile,
WaitBefore: false,
WaitInterval: DefaultWaitInterval,
WaitTimeout: DefaultWaitTimeout,
Limit: -1,
TargetVersion: "",
Log: os.Stdout,
}
}
// GetDriver initializes the appropriate database driver
// GetDriver loads the required database driver
func (db *DB) GetDriver() (Driver, error) {
if db.DatabaseURL == nil || db.DatabaseURL.Scheme == "" {
return nil, errors.New("invalid url, have you set your --url flag or DATABASE_URL environment variable?")
}
driverFunc := drivers[db.DatabaseURL.Scheme]
if driverFunc == nil {
return nil, fmt.Errorf("unsupported driver: %s", db.DatabaseURL.Scheme)
}
config := DriverConfig{
DatabaseURL: db.DatabaseURL,
MigrationsTableName: db.MigrationsTableName,
Log: db.Log,
}
return driverFunc(config), nil
return GetDriver(db.DatabaseURL.Scheme)
}
// Wait blocks until the database server is available. It does not verify that
@ -100,128 +70,115 @@ func (db *DB) Wait() error {
return err
}
return db.wait(drv)
}
func (db *DB) wait(drv Driver) error {
// attempt connection to database server
err := drv.Ping()
err = drv.Ping(db.DatabaseURL)
if err == nil {
// connection successful
return nil
}
fmt.Fprint(db.Log, "Waiting for database")
fmt.Print("Waiting for database")
for i := 0 * time.Second; i < db.WaitTimeout; i += db.WaitInterval {
fmt.Fprint(db.Log, ".")
fmt.Print(".")
time.Sleep(db.WaitInterval)
// attempt connection to database server
err = drv.Ping()
err = drv.Ping(db.DatabaseURL)
if err == nil {
// connection successful
fmt.Fprint(db.Log, "\n")
fmt.Print("\n")
return nil
}
}
// if we find outselves here, we could not connect within the timeout
fmt.Fprint(db.Log, "\n")
fmt.Print("\n")
return fmt.Errorf("unable to connect to database: %s", err)
}
// CreateAndMigrate creates the database (if necessary) and runs migrations
func (db *DB) CreateAndMigrate() error {
if db.WaitBefore {
err := db.Wait()
if err != nil {
return err
}
}
drv, err := db.GetDriver()
if err != nil {
return err
}
if db.WaitBefore {
err := db.wait(drv)
if err != nil {
return err
}
}
// create database if it does not already exist
// skip this step if we cannot determine status
// (e.g. user does not have list database permission)
exists, err := drv.DatabaseExists()
exists, err := drv.DatabaseExists(db.DatabaseURL)
if err == nil && !exists {
if err := drv.CreateDatabase(); err != nil {
if err := drv.CreateDatabase(db.DatabaseURL); err != nil {
return err
}
}
// migrate
return db.migrate(drv)
return db.Migrate()
}
// Create creates the current database
func (db *DB) Create() error {
if db.WaitBefore {
err := db.Wait()
if err != nil {
return err
}
}
drv, err := db.GetDriver()
if err != nil {
return err
}
if db.WaitBefore {
err := db.wait(drv)
if err != nil {
return err
}
}
return drv.CreateDatabase()
return drv.CreateDatabase(db.DatabaseURL)
}
// Drop drops the current database (if it exists)
func (db *DB) Drop() error {
if db.WaitBefore {
err := db.Wait()
if err != nil {
return err
}
}
drv, err := db.GetDriver()
if err != nil {
return err
}
if db.WaitBefore {
err := db.wait(drv)
if err != nil {
return err
}
}
return drv.DropDatabase()
return drv.DropDatabase(db.DatabaseURL)
}
// DumpSchema writes the current database schema to a file
func (db *DB) DumpSchema() error {
drv, err := db.GetDriver()
if err != nil {
return err
}
return db.dumpSchema(drv)
}
func (db *DB) dumpSchema(drv Driver) error {
if db.WaitBefore {
err := db.wait(drv)
err := db.Wait()
if err != nil {
return err
}
}
sqlDB, err := db.openDatabaseForMigration(drv)
drv, sqlDB, err := db.openDatabaseForMigration()
if err != nil {
return err
}
defer dbutil.MustClose(sqlDB)
defer mustClose(sqlDB)
schema, err := drv.DumpSchema(sqlDB)
schema, err := drv.DumpSchema(db.DatabaseURL, sqlDB)
if err != nil {
return err
}
fmt.Fprintf(db.Log, "Writing: %s\n", db.SchemaFile)
fmt.Printf("Writing: %s\n", db.SchemaFile)
// ensure schema directory exists
if err = ensureDir(filepath.Dir(db.SchemaFile)); err != nil {
@ -229,16 +186,7 @@ func (db *DB) dumpSchema(drv Driver) error {
}
// write schema to file
return os.WriteFile(db.SchemaFile, schema, 0644)
}
// ensureDir creates a directory if it does not already exist
func ensureDir(dir string) error {
if err := os.MkdirAll(dir, 0755); err != nil {
return fmt.Errorf("unable to create directory `%s`", dir)
}
return nil
return ioutil.WriteFile(db.SchemaFile, schema, 0644)
}
const migrationTemplate = "-- migrate:up\n\n\n-- migrate:down\n\n"
@ -259,7 +207,7 @@ func (db *DB) NewMigration(name string) error {
// check file does not already exist
path := filepath.Join(db.MigrationsDir, name)
fmt.Fprintf(db.Log, "Creating migration: %s\n", path)
fmt.Printf("Creating migration: %s\n", path)
if _, err := os.Stat(path); !os.IsNotExist(err) {
return fmt.Errorf("file already exists")
@ -271,13 +219,13 @@ func (db *DB) NewMigration(name string) error {
return err
}
defer dbutil.MustClose(file)
defer mustClose(file)
_, err = file.WriteString(migrationTemplate)
return err
}
func doTransaction(sqlDB *sql.DB, txFunc func(dbutil.Transaction) error) error {
tx, err := sqlDB.Begin()
func doTransaction(db *sql.DB, txFunc func(Transaction) error) error {
tx, err := db.Begin()
if err != nil {
return err
}
@ -293,31 +241,27 @@ func doTransaction(sqlDB *sql.DB, txFunc func(dbutil.Transaction) error) error {
return tx.Commit()
}
func (db *DB) openDatabaseForMigration(drv Driver) (*sql.DB, error) {
sqlDB, err := drv.Open()
func (db *DB) openDatabaseForMigration() (Driver, *sql.DB, error) {
drv, err := db.GetDriver()
if err != nil {
return nil, err
return nil, nil, err
}
sqlDB, err := drv.Open(db.DatabaseURL)
if err != nil {
return nil, nil, err
}
if err := drv.CreateMigrationsTable(sqlDB); err != nil {
dbutil.MustClose(sqlDB)
return nil, err
mustClose(sqlDB)
return nil, nil, err
}
return sqlDB, nil
return drv, sqlDB, nil
}
// Migrate migrates database to the latest version
func (db *DB) Migrate() error {
drv, err := db.GetDriver()
if err != nil {
return err
}
return db.migrate(drv)
}
func (db *DB) migrate(drv Driver) error {
files, err := findMigrationFiles(db.MigrationsDir, migrationFileRegexp)
if err != nil {
return err
@ -328,17 +272,17 @@ func (db *DB) migrate(drv Driver) error {
}
if db.WaitBefore {
err := db.wait(drv)
err := db.Wait()
if err != nil {
return err
}
}
sqlDB, err := db.openDatabaseForMigration(drv)
drv, sqlDB, err := db.openDatabaseForMigration()
if err != nil {
return err
}
defer dbutil.MustClose(sqlDB)
defer mustClose(sqlDB)
applied, err := drv.SelectMigrations(sqlDB, -1)
if err != nil {
@ -347,25 +291,25 @@ func (db *DB) migrate(drv Driver) error {
for _, filename := range files {
ver := migrationVersion(filename)
if ok := applied[ver]; ok && ver != db.TargetVersion {
if ok := applied[ver]; ok {
// migration already applied
continue
}
fmt.Fprintf(db.Log, "Applying: %s\n", filename)
fmt.Printf("Applying: %s\n", filename)
up, _, err := parseMigration(filepath.Join(db.MigrationsDir, filename))
if err != nil {
return err
}
execMigration := func(tx dbutil.Transaction) error {
execMigration := func(tx Transaction) error {
// run actual migration
result, err := tx.Exec(up.Contents)
if err != nil {
return err
} else if db.Verbose {
db.printVerbose(result)
printVerbose(result)
}
// record migration
@ -383,34 +327,18 @@ func (db *DB) migrate(drv Driver) error {
if err != nil {
return err
}
if ver == db.TargetVersion {
fmt.Fprintf(db.Log, "Reached target version %s\n", ver)
break
}
}
// automatically update schema file, silence errors
if db.AutoDumpSchema {
_ = db.dumpSchema(drv)
_ = db.DumpSchema()
}
return nil
}
func (db *DB) printVerbose(result sql.Result) {
lastInsertID, err := result.LastInsertId()
if err == nil {
fmt.Fprintf(db.Log, "Last insert ID: %d\n", lastInsertID)
}
rowsAffected, err := result.RowsAffected()
if err == nil {
fmt.Fprintf(db.Log, "Rows affected: %d\n", rowsAffected)
}
}
func findMigrationFiles(dir string, re *regexp.Regexp) ([]string, error) {
files, err := os.ReadDir(dir)
files, err := ioutil.ReadDir(dir)
if err != nil {
return nil, fmt.Errorf("could not find migrations directory `%s`", dir)
}
@ -460,88 +388,56 @@ func migrationVersion(filename string) string {
// Rollback rolls back the most recent migration
func (db *DB) Rollback() error {
drv, err := db.GetDriver()
if err != nil {
return err
}
if db.WaitBefore {
err := db.wait(drv)
err := db.Wait()
if err != nil {
return err
}
}
sqlDB, err := db.openDatabaseForMigration(drv)
drv, sqlDB, err := db.openDatabaseForMigration()
if err != nil {
return err
}
defer dbutil.MustClose(sqlDB)
defer mustClose(sqlDB)
limit := db.Limit
// default limit is -1, if we don't specify a version it should only rollback one version, not all
if limit <= 0 && db.TargetVersion == "" {
limit = 1
}
applied, err := drv.SelectMigrations(sqlDB, limit)
applied, err := drv.SelectMigrations(sqlDB, 1)
if err != nil {
return err
}
if len(applied) == 0 {
return fmt.Errorf("can't rollback, no migrations found")
// grab most recent applied migration (applied has len=1)
latest := ""
for ver := range applied {
latest = ver
}
if latest == "" {
return fmt.Errorf("can't rollback: no migrations have been applied")
}
var versions []string
for v := range applied {
versions = append(versions, v)
}
// new → old
sort.Sort(sort.Reverse(sort.StringSlice(versions)))
if db.TargetVersion != "" {
cache := map[string]bool{}
found := false
// latest version comes first, so take every version until the version matches
for _, ver := range versions {
if ver == db.TargetVersion {
found = true
break
}
cache[ver] = true
}
if !found {
return fmt.Errorf("target version not found")
}
applied = cache
}
for version := range applied {
filename, err := findMigrationFile(db.MigrationsDir, version)
filename, err := findMigrationFile(db.MigrationsDir, latest)
if err != nil {
return err
}
fmt.Fprintf(db.Log, "Rolling back: %s\n", filename)
fmt.Printf("Rolling back: %s\n", filename)
_, down, err := parseMigration(filepath.Join(db.MigrationsDir, filename))
if err != nil {
return err
}
execMigration := func(tx dbutil.Transaction) error {
execMigration := func(tx Transaction) error {
// rollback migration
result, err := tx.Exec(down.Contents)
if err != nil {
return err
} else if db.Verbose {
db.printVerbose(result)
printVerbose(result)
}
// remove migration record
return drv.DeleteMigration(tx, version)
return drv.DeleteMigration(tx, latest)
}
if down.Options.Transaction() {
@ -555,55 +451,16 @@ func (db *DB) Rollback() error {
if err != nil {
return err
}
}
// automatically update schema file, silence errors
if db.AutoDumpSchema {
_ = db.dumpSchema(drv)
_ = db.DumpSchema()
}
return nil
}
// Status shows the status of all migrations
func (db *DB) Status(quiet bool) (int, error) {
drv, err := db.GetDriver()
if err != nil {
return -1, err
}
results, err := db.CheckMigrationsStatus(drv)
if err != nil {
return -1, err
}
var totalApplied int
var line string
for _, res := range results {
if res.Applied {
line = fmt.Sprintf("[X] %s", res.Filename)
totalApplied++
} else {
line = fmt.Sprintf("[ ] %s", res.Filename)
}
if !quiet {
fmt.Fprintln(db.Log, line)
}
}
totalPending := len(results) - totalApplied
if !quiet {
fmt.Fprintln(db.Log)
fmt.Fprintf(db.Log, "Applied: %d\n", totalApplied)
fmt.Fprintf(db.Log, "Pending: %d\n", totalPending)
}
return totalPending, nil
}
// CheckMigrationsStatus returns the status of all available mgirations
func (db *DB) CheckMigrationsStatus(drv Driver) ([]StatusResult, error) {
func checkMigrationsStatus(db *DB) ([]statusResult, error) {
files, err := findMigrationFiles(db.MigrationsDir, migrationFileRegexp)
if err != nil {
return nil, err
@ -613,26 +470,26 @@ func (db *DB) CheckMigrationsStatus(drv Driver) ([]StatusResult, error) {
return nil, fmt.Errorf("no migration files found")
}
sqlDB, err := db.openDatabaseForMigration(drv)
drv, sqlDB, err := db.openDatabaseForMigration()
if err != nil {
return nil, err
}
defer dbutil.MustClose(sqlDB)
defer mustClose(sqlDB)
applied, err := drv.SelectMigrations(sqlDB, db.Limit)
applied, err := drv.SelectMigrations(sqlDB, -1)
if err != nil {
return nil, err
}
var results []StatusResult
var results []statusResult
for _, filename := range files {
ver := migrationVersion(filename)
res := StatusResult{Filename: filename}
res := statusResult{filename: filename}
if ok := applied[ver]; ok {
res.Applied = true
res.applied = true
} else {
res.Applied = false
res.applied = false
}
results = append(results, res)
@ -640,3 +497,35 @@ func (db *DB) CheckMigrationsStatus(drv Driver) ([]StatusResult, error) {
return results, nil
}
// Status shows the status of all migrations
func (db *DB) Status(quiet bool) (int, error) {
results, err := checkMigrationsStatus(db)
if err != nil {
return -1, err
}
var totalApplied int
var line string
for _, res := range results {
if res.applied {
line = fmt.Sprintf("[X] %s", res.filename)
totalApplied++
} else {
line = fmt.Sprintf("[ ] %s", res.filename)
}
if !quiet {
fmt.Println(line)
}
}
totalPending := len(results) - totalApplied
if !quiet {
fmt.Println()
fmt.Printf("Applied: %d\n", totalApplied)
fmt.Printf("Pending: %d\n", totalPending)
}
return totalPending, nil
}

View file

@ -1,25 +1,20 @@
package dbmate_test
package dbmate
import (
"io/ioutil"
"net/url"
"os"
"path/filepath"
"testing"
"time"
"github.com/amacneil/dbmate/pkg/dbmate"
"github.com/amacneil/dbmate/pkg/dbutil"
_ "github.com/amacneil/dbmate/pkg/driver/mysql"
_ "github.com/amacneil/dbmate/pkg/driver/postgres"
_ "github.com/amacneil/dbmate/pkg/driver/sqlite"
"github.com/kami-zh/go-capturer"
"github.com/stretchr/testify/require"
)
var testdataDir string
func newTestDB(t *testing.T, u *url.URL) *dbmate.DB {
func newTestDB(t *testing.T, u *url.URL) *DB {
var err error
// only chdir once, because testdata is relative to current directory
@ -31,51 +26,26 @@ func newTestDB(t *testing.T, u *url.URL) *dbmate.DB {
require.NoError(t, err)
}
db := dbmate.New(u)
db := New(u)
db.AutoDumpSchema = false
return db
}
func TestNew(t *testing.T) {
db := dbmate.New(dbutil.MustParseURL("foo:test"))
u := postgresTestURL(t)
db := New(u)
require.True(t, db.AutoDumpSchema)
require.Equal(t, "foo:test", db.DatabaseURL.String())
require.Equal(t, u.String(), db.DatabaseURL.String())
require.Equal(t, "./db/migrations", db.MigrationsDir)
require.Equal(t, "schema_migrations", db.MigrationsTableName)
require.Equal(t, "./db/schema.sql", db.SchemaFile)
require.False(t, db.WaitBefore)
require.Equal(t, time.Second, db.WaitInterval)
require.Equal(t, 60*time.Second, db.WaitTimeout)
require.Equal(t, -1, db.Limit)
require.Equal(t, "", db.TargetVersion)
}
func TestGetDriver(t *testing.T) {
t.Run("missing URL", func(t *testing.T) {
db := dbmate.New(nil)
drv, err := db.GetDriver()
require.Nil(t, drv)
require.EqualError(t, err, "invalid url, have you set your --url flag or DATABASE_URL environment variable?")
})
t.Run("missing schema", func(t *testing.T) {
db := dbmate.New(dbutil.MustParseURL("//hi"))
drv, err := db.GetDriver()
require.Nil(t, drv)
require.EqualError(t, err, "invalid url, have you set your --url flag or DATABASE_URL environment variable?")
})
t.Run("invalid driver", func(t *testing.T) {
db := dbmate.New(dbutil.MustParseURL("foo://bar"))
drv, err := db.GetDriver()
require.EqualError(t, err, "unsupported driver: foo")
require.Nil(t, drv)
})
}
func TestWait(t *testing.T) {
u := dbutil.MustParseURL(os.Getenv("POSTGRES_TEST_URL"))
u := postgresTestURL(t)
db := newTestDB(t, u)
// speed up our retry loop for testing
@ -99,11 +69,11 @@ func TestWait(t *testing.T) {
}
func TestDumpSchema(t *testing.T) {
u := dbutil.MustParseURL(os.Getenv("POSTGRES_TEST_URL"))
u := postgresTestURL(t)
db := newTestDB(t, u)
// create custom schema file directory
dir, err := os.MkdirTemp("", "dbmate")
dir, err := ioutil.TempDir("", "dbmate")
require.NoError(t, err)
defer func() {
err := os.RemoveAll(dir)
@ -130,18 +100,18 @@ func TestDumpSchema(t *testing.T) {
require.NoError(t, err)
// verify schema
schema, err := os.ReadFile(db.SchemaFile)
schema, err := ioutil.ReadFile(db.SchemaFile)
require.NoError(t, err)
require.Contains(t, string(schema), "-- PostgreSQL database dump")
}
func TestAutoDumpSchema(t *testing.T) {
u := dbutil.MustParseURL(os.Getenv("POSTGRES_TEST_URL"))
u := postgresTestURL(t)
db := newTestDB(t, u)
db.AutoDumpSchema = true
// create custom schema file directory
dir, err := os.MkdirTemp("", "dbmate")
dir, err := ioutil.TempDir("", "dbmate")
require.NoError(t, err)
defer func() {
err := os.RemoveAll(dir)
@ -164,7 +134,7 @@ func TestAutoDumpSchema(t *testing.T) {
require.NoError(t, err)
// verify schema
schema, err := os.ReadFile(db.SchemaFile)
schema, err := ioutil.ReadFile(db.SchemaFile)
require.NoError(t, err)
require.Contains(t, string(schema), "-- PostgreSQL database dump")
@ -177,7 +147,7 @@ func TestAutoDumpSchema(t *testing.T) {
require.NoError(t, err)
// schema should be recreated
schema, err = os.ReadFile(db.SchemaFile)
schema, err = ioutil.ReadFile(db.SchemaFile)
require.NoError(t, err)
require.Contains(t, string(schema), "-- PostgreSQL database dump")
}
@ -193,7 +163,7 @@ func checkWaitCalled(t *testing.T, u *url.URL, command func() error) {
}
func testWaitBefore(t *testing.T, verbose bool) {
u := dbutil.MustParseURL(os.Getenv("POSTGRES_TEST_URL"))
u := postgresTestURL(t)
db := newTestDB(t, u)
db.Verbose = verbose
db.WaitBefore = true
@ -244,31 +214,25 @@ func TestWaitBeforeVerbose(t *testing.T) {
`Applying: 20151129054053_test_migration.sql
Rows affected: 1
Applying: 20200227231541_test_posts.sql
Rows affected: 0
Applying: 20220607110405_test_category.sql
Rows affected: 0`)
require.Contains(t, output,
`Rolling back: 20220607110405_test_category.sql
`Rolling back: 20200227231541_test_posts.sql
Rows affected: 0`)
}
func testURLs() []*url.URL {
func testURLs(t *testing.T) []*url.URL {
return []*url.URL{
dbutil.MustParseURL(os.Getenv("MYSQL_TEST_URL")),
dbutil.MustParseURL(os.Getenv("POSTGRES_TEST_URL")),
dbutil.MustParseURL(os.Getenv("SQLITE_TEST_URL")),
postgresTestURL(t),
mySQLTestURL(t),
sqliteTestURL(t),
}
}
func TestMigrate(t *testing.T) {
for _, u := range testURLs() {
t.Run(u.Scheme, func(t *testing.T) {
func testMigrateURL(t *testing.T, u *url.URL) {
db := newTestDB(t, u)
drv, err := db.GetDriver()
require.NoError(t, err)
// drop and recreate database
err = db.Drop()
err := db.Drop()
require.NoError(t, err)
err = db.Create()
require.NoError(t, err)
@ -278,9 +242,9 @@ func TestMigrate(t *testing.T) {
require.NoError(t, err)
// verify results
sqlDB, err := drv.Open()
sqlDB, err := GetDriverOpen(u)
require.NoError(t, err)
defer dbutil.MustClose(sqlDB)
defer mustClose(sqlDB)
count := 0
err = sqlDB.QueryRow(`select count(*) from schema_migrations
@ -291,50 +255,19 @@ func TestMigrate(t *testing.T) {
err = sqlDB.QueryRow("select count(*) from users").Scan(&count)
require.NoError(t, err)
require.Equal(t, 1, count)
})
}
func TestMigrate(t *testing.T) {
for _, u := range testURLs(t) {
testMigrateURL(t, u)
}
}
func TestMigrateToTarget(t *testing.T) {
for _, u := range testURLs() {
t.Run(u.Scheme, func(t *testing.T) {
func testUpURL(t *testing.T, u *url.URL) {
db := newTestDB(t, u)
db.TargetVersion = "20151129054053"
drv, err := db.GetDriver()
require.NoError(t, err)
// drop and recreate database
err = db.Drop()
require.NoError(t, err)
err = db.Create()
require.NoError(t, err)
// migrate
err = db.Migrate()
require.NoError(t, err)
// verify results
sqlDB, err := drv.Open()
require.NoError(t, err)
defer dbutil.MustClose(sqlDB)
count := 0
err = sqlDB.QueryRow(`select count(*) from schema_migrations`).Scan(&count)
require.NoError(t, err)
require.Equal(t, 1, count)
})
}
}
func TestUp(t *testing.T) {
for _, u := range testURLs() {
t.Run(u.Scheme, func(t *testing.T) {
db := newTestDB(t, u)
drv, err := db.GetDriver()
require.NoError(t, err)
// drop database
err = db.Drop()
err := db.Drop()
require.NoError(t, err)
// create and migrate
@ -342,9 +275,9 @@ func TestUp(t *testing.T) {
require.NoError(t, err)
// verify results
sqlDB, err := drv.Open()
sqlDB, err := GetDriverOpen(u)
require.NoError(t, err)
defer dbutil.MustClose(sqlDB)
defer mustClose(sqlDB)
count := 0
err = sqlDB.QueryRow(`select count(*) from schema_migrations
@ -355,203 +288,105 @@ func TestUp(t *testing.T) {
err = sqlDB.QueryRow("select count(*) from users").Scan(&count)
require.NoError(t, err)
require.Equal(t, 1, count)
})
}
func TestUp(t *testing.T) {
for _, u := range testURLs(t) {
testUpURL(t, u)
}
}
func testRollbackURL(t *testing.T, u *url.URL) {
db := newTestDB(t, u)
// drop, recreate, and migrate database
err := db.Drop()
require.NoError(t, err)
err = db.Create()
require.NoError(t, err)
err = db.Migrate()
require.NoError(t, err)
// verify migration
sqlDB, err := GetDriverOpen(u)
require.NoError(t, err)
defer mustClose(sqlDB)
count := 0
err = sqlDB.QueryRow(`select count(*) from schema_migrations
where version = '20151129054053'`).Scan(&count)
require.NoError(t, err)
require.Equal(t, 1, count)
err = sqlDB.QueryRow("select count(*) from posts").Scan(&count)
require.Nil(t, err)
// rollback
err = db.Rollback()
require.NoError(t, err)
// verify rollback
err = sqlDB.QueryRow("select count(*) from schema_migrations").Scan(&count)
require.NoError(t, err)
require.Equal(t, 1, count)
err = sqlDB.QueryRow("select count(*) from posts").Scan(&count)
require.NotNil(t, err)
require.Regexp(t, "(does not exist|doesn't exist|no such table)", err.Error())
}
func TestRollback(t *testing.T) {
for _, u := range testURLs() {
t.Run(u.Scheme, func(t *testing.T) {
db := newTestDB(t, u)
drv, err := db.GetDriver()
require.NoError(t, err)
// drop, recreate, and migrate database
err = db.Drop()
require.NoError(t, err)
err = db.Create()
require.NoError(t, err)
err = db.Migrate()
require.NoError(t, err)
// verify migration
sqlDB, err := drv.Open()
require.NoError(t, err)
defer dbutil.MustClose(sqlDB)
count := 0
err = sqlDB.QueryRow(`select count(*) from schema_migrations
where version = '20151129054053'`).Scan(&count)
require.NoError(t, err)
require.Equal(t, 1, count)
err = sqlDB.QueryRow("select count(*) from categories").Scan(&count)
require.Nil(t, err)
// rollback
err = db.Rollback()
require.NoError(t, err)
// verify rollback
err = sqlDB.QueryRow("select count(*) from schema_migrations").Scan(&count)
require.NoError(t, err)
require.Equal(t, 2, count)
err = sqlDB.QueryRow("select count(*) from categories").Scan(&count)
require.NotNil(t, err)
require.Regexp(t, "(does not exist|doesn't exist|no such table)", err.Error())
})
for _, u := range testURLs(t) {
testRollbackURL(t, u)
}
}
func TestRollbackToTarget(t *testing.T) {
for _, u := range testURLs() {
t.Run(u.Scheme, func(t *testing.T) {
func testStatusUrl(t *testing.T, u *url.URL) {
db := newTestDB(t, u)
drv, err := db.GetDriver()
require.NoError(t, err)
// drop, recreate, and migrate database
err = db.Drop()
require.NoError(t, err)
err = db.Create()
require.NoError(t, err)
err = db.Migrate()
require.NoError(t, err)
// verify migration
sqlDB, err := drv.Open()
require.NoError(t, err)
defer dbutil.MustClose(sqlDB)
count := 0
err = sqlDB.QueryRow(`select count(*) from schema_migrations
where version = '20151129054053'`).Scan(&count)
require.NoError(t, err)
require.Equal(t, 1, count)
err = sqlDB.QueryRow("select count(*) from categories").Scan(&count)
require.Nil(t, err)
// rollback
db.TargetVersion = "20151129054053"
err = db.Rollback()
require.NoError(t, err)
// verify rollback
err = sqlDB.QueryRow("select count(*) from schema_migrations").Scan(&count)
require.NoError(t, err)
require.Equal(t, 1, count)
err = sqlDB.QueryRow("select count(*) from posts").Scan(&count)
require.NotNil(t, err)
require.Regexp(t, "(does not exist|doesn't exist|no such table)", err.Error())
err = sqlDB.QueryRow("select count(*) from categories").Scan(&count)
require.NotNil(t, err)
require.Regexp(t, "(does not exist|doesn't exist|no such table)", err.Error())
})
}
}
func TestRollbackToLimit(t *testing.T) {
for _, u := range testURLs() {
t.Run(u.Scheme, func(t *testing.T) {
db := newTestDB(t, u)
drv, err := db.GetDriver()
require.NoError(t, err)
// drop, recreate, and migrate database
err = db.Drop()
require.NoError(t, err)
err = db.Create()
require.NoError(t, err)
err = db.Migrate()
require.NoError(t, err)
// verify migration
sqlDB, err := drv.Open()
require.NoError(t, err)
defer dbutil.MustClose(sqlDB)
count := 0
err = sqlDB.QueryRow(`select count(*) from schema_migrations
where version = '20151129054053'`).Scan(&count)
require.NoError(t, err)
require.Equal(t, 1, count)
err = sqlDB.QueryRow("select count(*) from categories").Scan(&count)
require.Nil(t, err)
// rollback
db.Limit = 2
err = db.Rollback()
require.NoError(t, err)
// verify rollback
err = sqlDB.QueryRow("select count(*) from schema_migrations").Scan(&count)
require.NoError(t, err)
require.Equal(t, 1, count)
err = sqlDB.QueryRow("select count(*) from posts").Scan(&count)
require.NotNil(t, err)
require.Regexp(t, "(does not exist|doesn't exist|no such table)", err.Error())
err = sqlDB.QueryRow("select count(*) from categories").Scan(&count)
require.NotNil(t, err)
require.Regexp(t, "(does not exist|doesn't exist|no such table)", err.Error())
})
}
}
func TestStatus(t *testing.T) {
for _, u := range testURLs() {
t.Run(u.Scheme, func(t *testing.T) {
db := newTestDB(t, u)
drv, err := db.GetDriver()
require.NoError(t, err)
// drop, recreate, and migrate database
err = db.Drop()
err := db.Drop()
require.NoError(t, err)
err = db.Create()
require.NoError(t, err)
// verify migration
sqlDB, err := drv.Open()
sqlDB, err := GetDriverOpen(u)
require.NoError(t, err)
defer dbutil.MustClose(sqlDB)
defer mustClose(sqlDB)
// two pending
results, err := db.CheckMigrationsStatus(drv)
results, err := checkMigrationsStatus(db)
require.NoError(t, err)
require.Len(t, results, 3)
require.False(t, results[0].Applied)
require.False(t, results[1].Applied)
require.Len(t, results, 2)
require.False(t, results[0].applied)
require.False(t, results[1].applied)
// run migrations
err = db.Migrate()
require.NoError(t, err)
// three applied
results, err = db.CheckMigrationsStatus(drv)
// two applied
results, err = checkMigrationsStatus(db)
require.NoError(t, err)
require.Len(t, results, 3)
require.True(t, results[0].Applied)
require.True(t, results[1].Applied)
require.True(t, results[2].Applied)
require.Len(t, results, 2)
require.True(t, results[0].applied)
require.True(t, results[1].applied)
// rollback last migration
err = db.Rollback()
require.NoError(t, err)
// one applied, one pending
results, err = db.CheckMigrationsStatus(drv)
results, err = checkMigrationsStatus(db)
require.NoError(t, err)
require.Len(t, results, 3)
require.True(t, results[0].Applied)
require.True(t, results[1].Applied)
require.False(t, results[2].Applied)
})
require.Len(t, results, 2)
require.True(t, results[0].applied)
require.False(t, results[1].applied)
}
func TestStatus(t *testing.T) {
for _, u := range testURLs(t) {
testStatusUrl(t, u)
}
}

View file

@ -2,39 +2,51 @@ package dbmate
import (
"database/sql"
"io"
"fmt"
"net/url"
"github.com/amacneil/dbmate/pkg/dbutil"
)
// Driver provides top level database functions
type Driver interface {
Open() (*sql.DB, error)
DatabaseExists() (bool, error)
CreateDatabase() error
DropDatabase() error
DumpSchema(*sql.DB) ([]byte, error)
Open(*url.URL) (*sql.DB, error)
DatabaseExists(*url.URL) (bool, error)
CreateDatabase(*url.URL) error
DropDatabase(*url.URL) error
DumpSchema(*url.URL, *sql.DB) ([]byte, error)
CreateMigrationsTable(*sql.DB) error
SelectMigrations(*sql.DB, int) (map[string]bool, error)
InsertMigration(dbutil.Transaction, string) error
DeleteMigration(dbutil.Transaction, string) error
Ping() error
InsertMigration(Transaction, string) error
DeleteMigration(Transaction, string) error
Ping(*url.URL) error
}
// DriverConfig holds configuration passed to driver constructors
type DriverConfig struct {
DatabaseURL *url.URL
MigrationsTableName string
Log io.Writer
var drivers = map[string]Driver{}
// RegisterDriver registers a driver for a URL scheme
func RegisterDriver(drv Driver, scheme string) {
drivers[scheme] = drv
}
// DriverFunc represents a driver constructor
type DriverFunc func(DriverConfig) Driver
var drivers = map[string]DriverFunc{}
// RegisterDriver registers a driver constructor for a given URL scheme
func RegisterDriver(f DriverFunc, scheme string) {
drivers[scheme] = f
// Transaction can represent a database or open transaction
type Transaction interface {
Exec(query string, args ...interface{}) (sql.Result, error)
}
// GetDriver loads a database driver by name
func GetDriver(name string) (Driver, error) {
if val, ok := drivers[name]; ok {
return val, nil
}
return nil, fmt.Errorf("unsupported driver: %s", name)
}
// GetDriverOpen is a shortcut for GetDriver(u.Scheme).Open(u)
func GetDriverOpen(u *url.URL) (*sql.DB, error) {
drv, err := GetDriver(u.Scheme)
if err != nil {
return nil, err
}
return drv.Open(u)
}

27
pkg/dbmate/driver_test.go Normal file
View file

@ -0,0 +1,27 @@
package dbmate
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestGetDriver_Postgres(t *testing.T) {
drv, err := GetDriver("postgres")
require.NoError(t, err)
_, ok := drv.(PostgresDriver)
require.Equal(t, true, ok)
}
func TestGetDriver_MySQL(t *testing.T) {
drv, err := GetDriver("mysql")
require.NoError(t, err)
_, ok := drv.(MySQLDriver)
require.Equal(t, true, ok)
}
func TestGetDriver_Error(t *testing.T) {
drv, err := GetDriver("foo")
require.EqualError(t, err, "unsupported driver: foo")
require.Nil(t, drv)
}

View file

@ -2,7 +2,7 @@ package dbmate
import (
"fmt"
"os"
"io/ioutil"
"regexp"
"strings"
)
@ -33,7 +33,7 @@ func NewMigration() Migration {
// parseMigration reads a migration file and returns (up Migration, down Migration, error)
func parseMigration(path string) (Migration, Migration, error) {
data, err := os.ReadFile(path)
data, err := ioutil.ReadFile(path)
if err != nil {
return NewMigration(), NewMigration(), err
}

253
pkg/dbmate/mysql.go Normal file
View file

@ -0,0 +1,253 @@
package dbmate
import (
"bytes"
"database/sql"
"fmt"
"net/url"
"strings"
_ "github.com/go-sql-driver/mysql" // mysql driver for database/sql
)
func init() {
RegisterDriver(MySQLDriver{}, "mysql")
}
// MySQLDriver provides top level database functions
type MySQLDriver struct {
}
func normalizeMySQLURL(u *url.URL) string {
query := u.Query()
query.Set("multiStatements", "true")
host := u.Host
protocol := "tcp"
if query.Get("socket") != "" {
protocol = "unix"
host = query.Get("socket")
query.Del("socket")
} else if u.Port() == "" {
// set default port
host = fmt.Sprintf("%s:3306", host)
}
// Get decoded user:pass
userPassEncoded := u.User.String()
userPass, _ := url.QueryUnescape(userPassEncoded)
// Build DSN w/ user:pass percent-decoded
normalizedString := ""
if userPass != "" { // user:pass can be empty
normalizedString = userPass + "@"
}
// connection string format required by go-sql-driver/mysql
normalizedString = fmt.Sprintf("%s%s(%s)%s?%s", normalizedString,
protocol, host, u.Path, query.Encode())
return normalizedString
}
// Open creates a new database connection
func (drv MySQLDriver) Open(u *url.URL) (*sql.DB, error) {
return sql.Open("mysql", normalizeMySQLURL(u))
}
func (drv MySQLDriver) openRootDB(u *url.URL) (*sql.DB, error) {
// connect to no particular database
rootURL := *u
rootURL.Path = "/"
return drv.Open(&rootURL)
}
func mysqlQuoteIdentifier(str string) string {
str = strings.Replace(str, "`", "\\`", -1)
return fmt.Sprintf("`%s`", str)
}
// CreateDatabase creates the specified database
func (drv MySQLDriver) CreateDatabase(u *url.URL) error {
name := databaseName(u)
fmt.Printf("Creating: %s\n", name)
db, err := drv.openRootDB(u)
if err != nil {
return err
}
defer mustClose(db)
_, err = db.Exec(fmt.Sprintf("create database %s",
mysqlQuoteIdentifier(name)))
return err
}
// DropDatabase drops the specified database (if it exists)
func (drv MySQLDriver) DropDatabase(u *url.URL) error {
name := databaseName(u)
fmt.Printf("Dropping: %s\n", name)
db, err := drv.openRootDB(u)
if err != nil {
return err
}
defer mustClose(db)
_, err = db.Exec(fmt.Sprintf("drop database if exists %s",
mysqlQuoteIdentifier(name)))
return err
}
func mysqldumpArgs(u *url.URL) []string {
// generate CLI arguments
args := []string{"--opt", "--routines", "--no-data",
"--skip-dump-date", "--skip-add-drop-table"}
if hostname := u.Hostname(); hostname != "" {
args = append(args, "--host="+hostname)
}
if port := u.Port(); port != "" {
args = append(args, "--port="+port)
}
if username := u.User.Username(); username != "" {
args = append(args, "--user="+username)
}
// mysql recommends against using environment variables to supply password
// https://dev.mysql.com/doc/refman/5.7/en/password-security-user.html
if password, set := u.User.Password(); set {
args = append(args, "--password="+password)
}
// add database name
args = append(args, strings.TrimLeft(u.Path, "/"))
return args
}
func mysqlSchemaMigrationsDump(db *sql.DB) ([]byte, error) {
// load applied migrations
migrations, err := queryColumn(db,
"select quote(version) from schema_migrations order by version asc")
if err != nil {
return nil, err
}
// build schema_migrations table data
var buf bytes.Buffer
buf.WriteString("\n--\n-- Dbmate schema migrations\n--\n\n" +
"LOCK TABLES `schema_migrations` WRITE;\n")
if len(migrations) > 0 {
buf.WriteString("INSERT INTO `schema_migrations` (version) VALUES\n (" +
strings.Join(migrations, "),\n (") +
");\n")
}
buf.WriteString("UNLOCK TABLES;\n")
return buf.Bytes(), nil
}
// DumpSchema returns the current database schema
func (drv MySQLDriver) DumpSchema(u *url.URL, db *sql.DB) ([]byte, error) {
schema, err := runCommand("mysqldump", mysqldumpArgs(u)...)
if err != nil {
return nil, err
}
migrations, err := mysqlSchemaMigrationsDump(db)
if err != nil {
return nil, err
}
schema = append(schema, migrations...)
return trimLeadingSQLComments(schema)
}
// DatabaseExists determines whether the database exists
func (drv MySQLDriver) DatabaseExists(u *url.URL) (bool, error) {
name := databaseName(u)
db, err := drv.openRootDB(u)
if err != nil {
return false, err
}
defer mustClose(db)
exists := false
err = db.QueryRow("select true from information_schema.schemata "+
"where schema_name = ?", name).Scan(&exists)
if err == sql.ErrNoRows {
return false, nil
}
return exists, err
}
// CreateMigrationsTable creates the schema_migrations table
func (drv MySQLDriver) CreateMigrationsTable(db *sql.DB) error {
_, err := db.Exec("create table if not exists schema_migrations " +
"(version varchar(255) primary key)")
return err
}
// SelectMigrations returns a list of applied migrations
// with an optional limit (in descending order)
func (drv MySQLDriver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, error) {
query := "select version from schema_migrations order by version desc"
if limit >= 0 {
query = fmt.Sprintf("%s limit %d", query, limit)
}
rows, err := db.Query(query)
if err != nil {
return nil, err
}
defer mustClose(rows)
migrations := map[string]bool{}
for rows.Next() {
var version string
if err := rows.Scan(&version); err != nil {
return nil, err
}
migrations[version] = true
}
return migrations, nil
}
// InsertMigration adds a new migration record
func (drv MySQLDriver) InsertMigration(db Transaction, version string) error {
_, err := db.Exec("insert into schema_migrations (version) values (?)", version)
return err
}
// DeleteMigration removes a migration record
func (drv MySQLDriver) DeleteMigration(db Transaction, version string) error {
_, err := db.Exec("delete from schema_migrations where version = ?", version)
return err
}
// Ping verifies a connection to the database server. It does not verify whether the
// specified database exists.
func (drv MySQLDriver) Ping(u *url.URL) error {
db, err := drv.openRootDB(u)
if err != nil {
return err
}
defer mustClose(db)
return db.Ping()
}

302
pkg/dbmate/mysql_test.go Normal file
View file

@ -0,0 +1,302 @@
package dbmate
import (
"database/sql"
"net/url"
"testing"
"github.com/stretchr/testify/require"
)
func mySQLTestURL(t *testing.T) *url.URL {
u, err := url.Parse("mysql://root:root@mysql/dbmate")
require.NoError(t, err)
return u
}
func prepTestMySQLDB(t *testing.T) *sql.DB {
drv := MySQLDriver{}
u := mySQLTestURL(t)
// drop any existing database
err := drv.DropDatabase(u)
require.NoError(t, err)
// create database
err = drv.CreateDatabase(u)
require.NoError(t, err)
// connect database
db, err := drv.Open(u)
require.NoError(t, err)
return db
}
func TestNormalizeMySQLURLDefaults(t *testing.T) {
u, err := url.Parse("mysql://host/foo")
require.NoError(t, err)
require.Equal(t, "", u.Port())
s := normalizeMySQLURL(u)
require.Equal(t, "tcp(host:3306)/foo?multiStatements=true", s)
}
func TestNormalizeMySQLURLCustom(t *testing.T) {
u, err := url.Parse("mysql://bob:secret@host:123/foo?flag=on")
require.NoError(t, err)
require.Equal(t, "123", u.Port())
s := normalizeMySQLURL(u)
require.Equal(t, "bob:secret@tcp(host:123)/foo?flag=on&multiStatements=true", s)
}
func TestNormalizeMySQLURLCustomSpecialChars(t *testing.T) {
u, err := url.Parse("mysql://duhfsd7s:123!@123!@@host:123/foo?flag=on")
require.NoError(t, err)
require.Equal(t, "123", u.Port())
s := normalizeMySQLURL(u)
require.Equal(t, "duhfsd7s:123!@123!@@tcp(host:123)/foo?flag=on&multiStatements=true", s)
}
func TestNormalizeMySQLURLSocket(t *testing.T) {
// test with no user/pass
u, err := url.Parse("mysql:///foo?socket=/var/run/mysqld/mysqld.sock&flag=on")
require.NoError(t, err)
require.Equal(t, "", u.Host)
s := normalizeMySQLURL(u)
require.Equal(t, "unix(/var/run/mysqld/mysqld.sock)/foo?flag=on&multiStatements=true", s)
// test with user/pass
u, err = url.Parse("mysql://bob:secret@fakehost/foo?socket=/var/run/mysqld/mysqld.sock&flag=on")
require.NoError(t, err)
s = normalizeMySQLURL(u)
require.Equal(t, "bob:secret@unix(/var/run/mysqld/mysqld.sock)/foo?flag=on&multiStatements=true", s)
}
func TestMySQLCreateDropDatabase(t *testing.T) {
drv := MySQLDriver{}
u := mySQLTestURL(t)
// drop any existing database
err := drv.DropDatabase(u)
require.NoError(t, err)
// create database
err = drv.CreateDatabase(u)
require.NoError(t, err)
// check that database exists and we can connect to it
func() {
db, err := drv.Open(u)
require.NoError(t, err)
defer mustClose(db)
err = db.Ping()
require.NoError(t, err)
}()
// drop the database
err = drv.DropDatabase(u)
require.NoError(t, err)
// check that database no longer exists
func() {
db, err := drv.Open(u)
require.NoError(t, err)
defer mustClose(db)
err = db.Ping()
require.NotNil(t, err)
require.Regexp(t, "Unknown database 'dbmate'", err.Error())
}()
}
func TestMySQLDumpSchema(t *testing.T) {
drv := MySQLDriver{}
u := mySQLTestURL(t)
// prepare database
db := prepTestMySQLDB(t)
defer mustClose(db)
err := drv.CreateMigrationsTable(db)
require.NoError(t, err)
// insert migration
err = drv.InsertMigration(db, "abc1")
require.NoError(t, err)
err = drv.InsertMigration(db, "abc2")
require.NoError(t, err)
// DumpSchema should return schema
schema, err := drv.DumpSchema(u, db)
require.NoError(t, err)
require.Contains(t, string(schema), "CREATE TABLE `schema_migrations`")
require.Contains(t, string(schema), "\n-- Dump completed\n\n"+
"--\n"+
"-- Dbmate schema migrations\n"+
"--\n\n"+
"LOCK TABLES `schema_migrations` WRITE;\n"+
"INSERT INTO `schema_migrations` (version) VALUES\n"+
" ('abc1'),\n"+
" ('abc2');\n"+
"UNLOCK TABLES;\n")
// DumpSchema should return error if command fails
u.Path = "/fakedb"
schema, err = drv.DumpSchema(u, db)
require.Nil(t, schema)
require.EqualError(t, err, "mysqldump: [Warning] Using a password "+
"on the command line interface can be insecure.\n"+
"mysqldump: Got error: 1049: "+
"Unknown database 'fakedb' when selecting the database")
}
func TestMySQLDatabaseExists(t *testing.T) {
drv := MySQLDriver{}
u := mySQLTestURL(t)
// drop any existing database
err := drv.DropDatabase(u)
require.NoError(t, err)
// DatabaseExists should return false
exists, err := drv.DatabaseExists(u)
require.NoError(t, err)
require.Equal(t, false, exists)
// create database
err = drv.CreateDatabase(u)
require.NoError(t, err)
// DatabaseExists should return true
exists, err = drv.DatabaseExists(u)
require.NoError(t, err)
require.Equal(t, true, exists)
}
func TestMySQLDatabaseExists_Error(t *testing.T) {
drv := MySQLDriver{}
u := mySQLTestURL(t)
u.User = url.User("invalid")
exists, err := drv.DatabaseExists(u)
require.Regexp(t, "Access denied for user 'invalid'@", err.Error())
require.Equal(t, false, exists)
}
func TestMySQLCreateMigrationsTable(t *testing.T) {
drv := MySQLDriver{}
db := prepTestMySQLDB(t)
defer mustClose(db)
// migrations table should not exist
count := 0
err := db.QueryRow("select count(*) from schema_migrations").Scan(&count)
require.Regexp(t, "Table 'dbmate.schema_migrations' doesn't exist", err.Error())
// create table
err = drv.CreateMigrationsTable(db)
require.NoError(t, err)
// migrations table should exist
err = db.QueryRow("select count(*) from schema_migrations").Scan(&count)
require.NoError(t, err)
// create table should be idempotent
err = drv.CreateMigrationsTable(db)
require.NoError(t, err)
}
func TestMySQLSelectMigrations(t *testing.T) {
drv := MySQLDriver{}
db := prepTestMySQLDB(t)
defer mustClose(db)
err := drv.CreateMigrationsTable(db)
require.NoError(t, err)
_, err = db.Exec(`insert into schema_migrations (version)
values ('abc2'), ('abc1'), ('abc3')`)
require.NoError(t, err)
migrations, err := drv.SelectMigrations(db, -1)
require.NoError(t, err)
require.Equal(t, true, migrations["abc1"])
require.Equal(t, true, migrations["abc2"])
require.Equal(t, true, migrations["abc2"])
// test limit param
migrations, err = drv.SelectMigrations(db, 1)
require.NoError(t, err)
require.Equal(t, true, migrations["abc3"])
require.Equal(t, false, migrations["abc1"])
require.Equal(t, false, migrations["abc2"])
}
func TestMySQLInsertMigration(t *testing.T) {
drv := MySQLDriver{}
db := prepTestMySQLDB(t)
defer mustClose(db)
err := drv.CreateMigrationsTable(db)
require.NoError(t, err)
count := 0
err = db.QueryRow("select count(*) from schema_migrations").Scan(&count)
require.NoError(t, err)
require.Equal(t, 0, count)
// insert migration
err = drv.InsertMigration(db, "abc1")
require.NoError(t, err)
err = db.QueryRow("select count(*) from schema_migrations where version = 'abc1'").
Scan(&count)
require.NoError(t, err)
require.Equal(t, 1, count)
}
func TestMySQLDeleteMigration(t *testing.T) {
drv := MySQLDriver{}
db := prepTestMySQLDB(t)
defer mustClose(db)
err := drv.CreateMigrationsTable(db)
require.NoError(t, err)
_, err = db.Exec(`insert into schema_migrations (version)
values ('abc1'), ('abc2')`)
require.NoError(t, err)
err = drv.DeleteMigration(db, "abc2")
require.NoError(t, err)
count := 0
err = db.QueryRow("select count(*) from schema_migrations").Scan(&count)
require.NoError(t, err)
require.Equal(t, 1, count)
}
func TestMySQLPing(t *testing.T) {
drv := MySQLDriver{}
u := mySQLTestURL(t)
// drop any existing database
err := drv.DropDatabase(u)
require.NoError(t, err)
// ping database
err = drv.Ping(u)
require.NoError(t, err)
// ping invalid host should return error
u.Host = "mysql:404"
err = drv.Ping(u)
require.Error(t, err)
require.Contains(t, err.Error(), "connect: connection refused")
}

239
pkg/dbmate/postgres.go Normal file
View file

@ -0,0 +1,239 @@
package dbmate
import (
"bytes"
"database/sql"
"fmt"
"net/url"
"strings"
"github.com/lib/pq"
)
func init() {
RegisterDriver(PostgresDriver{}, "postgres")
RegisterDriver(PostgresDriver{}, "postgresql")
}
// PostgresDriver provides top level database functions
type PostgresDriver struct {
}
func normalizePostgresURL(u *url.URL) string {
hostname := u.Hostname()
port := u.Port()
query := u.Query()
// support socket parameter for consistency with mysql
if query.Get("socket") != "" {
query.Set("host", query.Get("socket"))
query.Del("socket")
}
// default hostname
if hostname == "" {
hostname = "localhost"
}
// host param overrides url hostname
if query.Get("host") != "" {
hostname = ""
}
// always specify a port
if query.Get("port") != "" {
port = query.Get("port")
query.Del("port")
}
if port == "" {
port = "5432"
}
// generate output URL
out, _ := url.Parse(u.String())
out.Host = fmt.Sprintf("%s:%s", hostname, port)
out.RawQuery = query.Encode()
return out.String()
}
// Open creates a new database connection
func (drv PostgresDriver) Open(u *url.URL) (*sql.DB, error) {
return sql.Open("postgres", normalizePostgresURL(u))
}
func (drv PostgresDriver) openPostgresDB(u *url.URL) (*sql.DB, error) {
// connect to postgres database
postgresURL := *u
postgresURL.Path = "postgres"
return drv.Open(&postgresURL)
}
// CreateDatabase creates the specified database
func (drv PostgresDriver) CreateDatabase(u *url.URL) error {
name := databaseName(u)
fmt.Printf("Creating: %s\n", name)
db, err := drv.openPostgresDB(u)
if err != nil {
return err
}
defer mustClose(db)
_, err = db.Exec(fmt.Sprintf("create database %s",
pq.QuoteIdentifier(name)))
return err
}
// DropDatabase drops the specified database (if it exists)
func (drv PostgresDriver) DropDatabase(u *url.URL) error {
name := databaseName(u)
fmt.Printf("Dropping: %s\n", name)
db, err := drv.openPostgresDB(u)
if err != nil {
return err
}
defer mustClose(db)
_, err = db.Exec(fmt.Sprintf("drop database if exists %s",
pq.QuoteIdentifier(name)))
return err
}
func postgresSchemaMigrationsDump(db *sql.DB) ([]byte, error) {
// load applied migrations
migrations, err := queryColumn(db,
"select quote_literal(version) from public.schema_migrations order by version asc")
if err != nil {
return nil, err
}
// build schema_migrations table data
var buf bytes.Buffer
buf.WriteString("\n--\n-- Dbmate schema migrations\n--\n\n")
if len(migrations) > 0 {
buf.WriteString("INSERT INTO public.schema_migrations (version) VALUES\n (" +
strings.Join(migrations, "),\n (") +
");\n")
}
return buf.Bytes(), nil
}
// DumpSchema returns the current database schema
func (drv PostgresDriver) DumpSchema(u *url.URL, db *sql.DB) ([]byte, error) {
// load schema
schema, err := runCommand("pg_dump", "--format=plain", "--encoding=UTF8",
"--schema-only", "--no-privileges", "--no-owner", normalizePostgresURL(u))
if err != nil {
return nil, err
}
migrations, err := postgresSchemaMigrationsDump(db)
if err != nil {
return nil, err
}
schema = append(schema, migrations...)
return trimLeadingSQLComments(schema)
}
// DatabaseExists determines whether the database exists
func (drv PostgresDriver) DatabaseExists(u *url.URL) (bool, error) {
name := databaseName(u)
db, err := drv.openPostgresDB(u)
if err != nil {
return false, err
}
defer mustClose(db)
exists := false
err = db.QueryRow("select true from pg_database where datname = $1", name).
Scan(&exists)
if err == sql.ErrNoRows {
return false, nil
}
return exists, err
}
// CreateMigrationsTable creates the schema_migrations table
func (drv PostgresDriver) CreateMigrationsTable(db *sql.DB) error {
_, err := db.Exec("create table if not exists public.schema_migrations " +
"(version varchar(255) primary key)")
return err
}
// SelectMigrations returns a list of applied migrations
// with an optional limit (in descending order)
func (drv PostgresDriver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, error) {
query := "select version from public.schema_migrations order by version desc"
if limit >= 0 {
query = fmt.Sprintf("%s limit %d", query, limit)
}
rows, err := db.Query(query)
if err != nil {
return nil, err
}
defer mustClose(rows)
migrations := map[string]bool{}
for rows.Next() {
var version string
if err := rows.Scan(&version); err != nil {
return nil, err
}
migrations[version] = true
}
return migrations, nil
}
// InsertMigration adds a new migration record
func (drv PostgresDriver) InsertMigration(db Transaction, version string) error {
_, err := db.Exec("insert into public.schema_migrations (version) values ($1)", version)
return err
}
// DeleteMigration removes a migration record
func (drv PostgresDriver) DeleteMigration(db Transaction, version string) error {
_, err := db.Exec("delete from public.schema_migrations where version = $1", version)
return err
}
// Ping verifies a connection to the database server. It does not verify whether the
// specified database exists.
func (drv PostgresDriver) Ping(u *url.URL) error {
// attempt connection to primary database, not "postgres" database
// to support servers with no "postgres" database
// (see https://github.com/amacneil/dbmate/issues/78)
db, err := drv.Open(u)
if err != nil {
return err
}
defer mustClose(db)
err = db.Ping()
if err == nil {
return nil
}
// ignore 'database "foo" does not exist' error
pqErr, ok := err.(*pq.Error)
if ok && pqErr.Code == "3D000" {
return nil
}
return err
}

299
pkg/dbmate/postgres_test.go Normal file
View file

@ -0,0 +1,299 @@
package dbmate
import (
"database/sql"
"net/url"
"testing"
"github.com/stretchr/testify/require"
)
func postgresTestURL(t *testing.T) *url.URL {
u, err := url.Parse("postgres://postgres:postgres@postgres/dbmate?sslmode=disable")
require.NoError(t, err)
return u
}
func prepTestPostgresDB(t *testing.T) *sql.DB {
drv := PostgresDriver{}
u := postgresTestURL(t)
// drop any existing database
err := drv.DropDatabase(u)
require.NoError(t, err)
// create database
err = drv.CreateDatabase(u)
require.NoError(t, err)
// connect database
db, err := sql.Open("postgres", u.String())
require.NoError(t, err)
return db
}
func TestNormalizePostgresURLDefaults(t *testing.T) {
u, err := url.Parse("postgres:///foo")
require.NoError(t, err)
s := normalizePostgresURL(u)
require.Equal(t, "postgres://localhost:5432/foo", s)
}
func TestNormalizePostgresURLCustom(t *testing.T) {
u, err := url.Parse("postgres://bob:secret@myhost:1234/foo?bar=baz")
require.NoError(t, err)
s := normalizePostgresURL(u)
require.Equal(t, "postgres://bob:secret@myhost:1234/foo?bar=baz", s)
}
func TestNormalizePostgresURLHostPortParams(t *testing.T) {
u, err := url.Parse("postgres://bob:secret@myhost:1234/foo?port=9999&bar=baz")
require.NoError(t, err)
s := normalizePostgresURL(u)
require.Equal(t, "postgres://bob:secret@myhost:9999/foo?bar=baz", s)
u, err = url.Parse("postgres://bob:secret@myhost:1234/foo?host=new&port=9999")
require.NoError(t, err)
s = normalizePostgresURL(u)
require.Equal(t, "postgres://bob:secret@:9999/foo?host=new", s)
u, err = url.Parse("postgres://bob:secret@myhost:1234/foo?host=/var/run/postgresql")
require.NoError(t, err)
s = normalizePostgresURL(u)
require.Equal(t, "postgres://bob:secret@:1234/foo?host=%2Fvar%2Frun%2Fpostgresql", s)
}
func TestNormalizePostgresURLSocketParam(t *testing.T) {
u, err := url.Parse("postgres://bob:secret@localhost/foo?socket=/var/run/postgresql")
require.NoError(t, err)
s := normalizePostgresURL(u)
require.Equal(t, "postgres://bob:secret@:5432/foo?host=%2Fvar%2Frun%2Fpostgresql", s)
u, err = url.Parse("postgres:///foo?socket=/var/run/postgresql")
require.NoError(t, err)
s = normalizePostgresURL(u)
require.Equal(t, "postgres://:5432/foo?host=%2Fvar%2Frun%2Fpostgresql", s)
}
func TestPostgresCreateDropDatabase(t *testing.T) {
drv := PostgresDriver{}
u := postgresTestURL(t)
// drop any existing database
err := drv.DropDatabase(u)
require.NoError(t, err)
// create database
err = drv.CreateDatabase(u)
require.NoError(t, err)
// check that database exists and we can connect to it
func() {
db, err := sql.Open("postgres", u.String())
require.NoError(t, err)
defer mustClose(db)
err = db.Ping()
require.NoError(t, err)
}()
// drop the database
err = drv.DropDatabase(u)
require.NoError(t, err)
// check that database no longer exists
func() {
db, err := sql.Open("postgres", u.String())
require.NoError(t, err)
defer mustClose(db)
err = db.Ping()
require.NotNil(t, err)
require.Equal(t, "pq: database \"dbmate\" does not exist", err.Error())
}()
}
func TestPostgresDumpSchema(t *testing.T) {
drv := PostgresDriver{}
u := postgresTestURL(t)
// prepare database
db := prepTestPostgresDB(t)
defer mustClose(db)
err := drv.CreateMigrationsTable(db)
require.NoError(t, err)
// insert migration
err = drv.InsertMigration(db, "abc1")
require.NoError(t, err)
err = drv.InsertMigration(db, "abc2")
require.NoError(t, err)
// DumpSchema should return schema
schema, err := drv.DumpSchema(u, db)
require.NoError(t, err)
require.Contains(t, string(schema), "CREATE TABLE public.schema_migrations")
require.Contains(t, string(schema), "\n--\n"+
"-- PostgreSQL database dump complete\n"+
"--\n\n\n"+
"--\n"+
"-- Dbmate schema migrations\n"+
"--\n\n"+
"INSERT INTO public.schema_migrations (version) VALUES\n"+
" ('abc1'),\n"+
" ('abc2');\n")
// DumpSchema should return error if command fails
u.Path = "/fakedb"
schema, err = drv.DumpSchema(u, db)
require.Nil(t, schema)
require.EqualError(t, err, "pg_dump: [archiver (db)] connection to database "+
"\"fakedb\" failed: FATAL: database \"fakedb\" does not exist")
}
func TestPostgresDatabaseExists(t *testing.T) {
drv := PostgresDriver{}
u := postgresTestURL(t)
// drop any existing database
err := drv.DropDatabase(u)
require.NoError(t, err)
// DatabaseExists should return false
exists, err := drv.DatabaseExists(u)
require.NoError(t, err)
require.Equal(t, false, exists)
// create database
err = drv.CreateDatabase(u)
require.NoError(t, err)
// DatabaseExists should return true
exists, err = drv.DatabaseExists(u)
require.NoError(t, err)
require.Equal(t, true, exists)
}
func TestPostgresDatabaseExists_Error(t *testing.T) {
drv := PostgresDriver{}
u := postgresTestURL(t)
u.User = url.User("invalid")
exists, err := drv.DatabaseExists(u)
require.Equal(t, "pq: password authentication failed for user \"invalid\"", err.Error())
require.Equal(t, false, exists)
}
func TestPostgresCreateMigrationsTable(t *testing.T) {
drv := PostgresDriver{}
db := prepTestPostgresDB(t)
defer mustClose(db)
// migrations table should not exist
count := 0
err := db.QueryRow("select count(*) from public.schema_migrations").Scan(&count)
require.Equal(t, "pq: relation \"public.schema_migrations\" does not exist", err.Error())
// create table
err = drv.CreateMigrationsTable(db)
require.NoError(t, err)
// migrations table should exist
err = db.QueryRow("select count(*) from public.schema_migrations").Scan(&count)
require.NoError(t, err)
// create table should be idempotent
err = drv.CreateMigrationsTable(db)
require.NoError(t, err)
}
func TestPostgresSelectMigrations(t *testing.T) {
drv := PostgresDriver{}
db := prepTestPostgresDB(t)
defer mustClose(db)
err := drv.CreateMigrationsTable(db)
require.NoError(t, err)
_, err = db.Exec(`insert into public.schema_migrations (version)
values ('abc2'), ('abc1'), ('abc3')`)
require.NoError(t, err)
migrations, err := drv.SelectMigrations(db, -1)
require.NoError(t, err)
require.Equal(t, true, migrations["abc1"])
require.Equal(t, true, migrations["abc2"])
require.Equal(t, true, migrations["abc2"])
// test limit param
migrations, err = drv.SelectMigrations(db, 1)
require.NoError(t, err)
require.Equal(t, true, migrations["abc3"])
require.Equal(t, false, migrations["abc1"])
require.Equal(t, false, migrations["abc2"])
}
func TestPostgresInsertMigration(t *testing.T) {
drv := PostgresDriver{}
db := prepTestPostgresDB(t)
defer mustClose(db)
err := drv.CreateMigrationsTable(db)
require.NoError(t, err)
count := 0
err = db.QueryRow("select count(*) from public.schema_migrations").Scan(&count)
require.NoError(t, err)
require.Equal(t, 0, count)
// insert migration
err = drv.InsertMigration(db, "abc1")
require.NoError(t, err)
err = db.QueryRow("select count(*) from public.schema_migrations where version = 'abc1'").
Scan(&count)
require.NoError(t, err)
require.Equal(t, 1, count)
}
func TestPostgresDeleteMigration(t *testing.T) {
drv := PostgresDriver{}
db := prepTestPostgresDB(t)
defer mustClose(db)
err := drv.CreateMigrationsTable(db)
require.NoError(t, err)
_, err = db.Exec(`insert into public.schema_migrations (version)
values ('abc1'), ('abc2')`)
require.NoError(t, err)
err = drv.DeleteMigration(db, "abc2")
require.NoError(t, err)
count := 0
err = db.QueryRow("select count(*) from public.schema_migrations").Scan(&count)
require.NoError(t, err)
require.Equal(t, 1, count)
}
func TestPostgresPing(t *testing.T) {
drv := PostgresDriver{}
u := postgresTestURL(t)
// drop any existing database
err := drv.DropDatabase(u)
require.NoError(t, err)
// ping database
err = drv.Ping(u)
require.NoError(t, err)
// ping invalid host should return error
u.Host = "postgres:404"
err = drv.Ping(u)
require.Error(t, err)
require.Contains(t, err.Error(), "connect: connection refused")
}

179
pkg/dbmate/sqlite.go Normal file
View file

@ -0,0 +1,179 @@
// +build cgo
package dbmate
import (
"bytes"
"database/sql"
"fmt"
"net/url"
"os"
"regexp"
"strings"
_ "github.com/mattn/go-sqlite3" // sqlite driver for database/sql
)
func init() {
RegisterDriver(SQLiteDriver{}, "sqlite")
RegisterDriver(SQLiteDriver{}, "sqlite3")
}
// SQLiteDriver provides top level database functions
type SQLiteDriver struct {
}
func sqlitePath(u *url.URL) string {
// strip one leading slash
// absolute URLs can be specified as sqlite:////tmp/foo.sqlite3
str := regexp.MustCompile("^/").ReplaceAllString(u.Path, "")
return str
}
// Open creates a new database connection
func (drv SQLiteDriver) Open(u *url.URL) (*sql.DB, error) {
return sql.Open("sqlite3", sqlitePath(u))
}
// CreateDatabase creates the specified database
func (drv SQLiteDriver) CreateDatabase(u *url.URL) error {
fmt.Printf("Creating: %s\n", sqlitePath(u))
db, err := drv.Open(u)
if err != nil {
return err
}
defer mustClose(db)
return db.Ping()
}
// DropDatabase drops the specified database (if it exists)
func (drv SQLiteDriver) DropDatabase(u *url.URL) error {
path := sqlitePath(u)
fmt.Printf("Dropping: %s\n", path)
exists, err := drv.DatabaseExists(u)
if err != nil {
return err
}
if !exists {
return nil
}
return os.Remove(path)
}
func sqliteSchemaMigrationsDump(db *sql.DB) ([]byte, error) {
// load applied migrations
migrations, err := queryColumn(db,
"select quote(version) from schema_migrations order by version asc")
if err != nil {
return nil, err
}
// build schema_migrations table data
var buf bytes.Buffer
buf.WriteString("-- Dbmate schema migrations\n")
if len(migrations) > 0 {
buf.WriteString("INSERT INTO schema_migrations (version) VALUES\n (" +
strings.Join(migrations, "),\n (") +
");\n")
}
return buf.Bytes(), nil
}
// DumpSchema returns the current database schema
func (drv SQLiteDriver) DumpSchema(u *url.URL, db *sql.DB) ([]byte, error) {
path := sqlitePath(u)
schema, err := runCommand("sqlite3", path, ".schema")
if err != nil {
return nil, err
}
migrations, err := sqliteSchemaMigrationsDump(db)
if err != nil {
return nil, err
}
schema = append(schema, migrations...)
return trimLeadingSQLComments(schema)
}
// DatabaseExists determines whether the database exists
func (drv SQLiteDriver) DatabaseExists(u *url.URL) (bool, error) {
_, err := os.Stat(sqlitePath(u))
if os.IsNotExist(err) {
return false, nil
}
if err != nil {
return false, err
}
return true, nil
}
// CreateMigrationsTable creates the schema_migrations table
func (drv SQLiteDriver) CreateMigrationsTable(db *sql.DB) error {
_, err := db.Exec("create table if not exists schema_migrations " +
"(version varchar(255) primary key)")
return err
}
// SelectMigrations returns a list of applied migrations
// with an optional limit (in descending order)
func (drv SQLiteDriver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, error) {
query := "select version from schema_migrations order by version desc"
if limit >= 0 {
query = fmt.Sprintf("%s limit %d", query, limit)
}
rows, err := db.Query(query)
if err != nil {
return nil, err
}
defer mustClose(rows)
migrations := map[string]bool{}
for rows.Next() {
var version string
if err := rows.Scan(&version); err != nil {
return nil, err
}
migrations[version] = true
}
return migrations, nil
}
// InsertMigration adds a new migration record
func (drv SQLiteDriver) InsertMigration(db Transaction, version string) error {
_, err := db.Exec("insert into schema_migrations (version) values (?)", version)
return err
}
// DeleteMigration removes a migration record
func (drv SQLiteDriver) DeleteMigration(db Transaction, version string) error {
_, err := db.Exec("delete from schema_migrations where version = ?", version)
return err
}
// Ping verifies a connection to the database. Due to the way SQLite works, by
// testing whether the database is valid, it will automatically create the database
// if it does not already exist.
func (drv SQLiteDriver) Ping(u *url.URL) error {
db, err := drv.Open(u)
if err != nil {
return err
}
defer mustClose(db)
return db.Ping()
}

248
pkg/dbmate/sqlite_test.go Normal file
View file

@ -0,0 +1,248 @@
// +build cgo
package dbmate
import (
"database/sql"
"net/url"
"os"
"testing"
"github.com/stretchr/testify/require"
)
func sqliteTestURL(t *testing.T) *url.URL {
u, err := url.Parse("sqlite3:////tmp/dbmate.sqlite3")
require.NoError(t, err)
return u
}
func prepTestSQLiteDB(t *testing.T) *sql.DB {
drv := SQLiteDriver{}
u := sqliteTestURL(t)
// drop any existing database
err := drv.DropDatabase(u)
require.NoError(t, err)
// create database
err = drv.CreateDatabase(u)
require.NoError(t, err)
// connect database
db, err := drv.Open(u)
require.NoError(t, err)
return db
}
func TestSQLiteCreateDropDatabase(t *testing.T) {
drv := SQLiteDriver{}
u := sqliteTestURL(t)
path := sqlitePath(u)
// drop any existing database
err := drv.DropDatabase(u)
require.NoError(t, err)
// create database
err = drv.CreateDatabase(u)
require.NoError(t, err)
// check that database exists
_, err = os.Stat(path)
require.NoError(t, err)
// drop the database
err = drv.DropDatabase(u)
require.NoError(t, err)
// check that database no longer exists
_, err = os.Stat(path)
require.NotNil(t, err)
require.Equal(t, true, os.IsNotExist(err))
}
func TestSQLiteDumpSchema(t *testing.T) {
drv := SQLiteDriver{}
u := sqliteTestURL(t)
// prepare database
db := prepTestSQLiteDB(t)
defer mustClose(db)
err := drv.CreateMigrationsTable(db)
require.NoError(t, err)
// insert migration
err = drv.InsertMigration(db, "abc1")
require.NoError(t, err)
err = drv.InsertMigration(db, "abc2")
require.NoError(t, err)
// DumpSchema should return schema
schema, err := drv.DumpSchema(u, db)
require.NoError(t, err)
require.Contains(t, string(schema), "CREATE TABLE schema_migrations")
require.Contains(t, string(schema), ");\n-- Dbmate schema migrations\n"+
"INSERT INTO schema_migrations (version) VALUES\n"+
" ('abc1'),\n"+
" ('abc2');\n")
// DumpSchema should return error if command fails
u.Path = "/."
schema, err = drv.DumpSchema(u, db)
require.Nil(t, schema)
require.EqualError(t, err, "Error: unable to open database \".\": "+
"unable to open database file")
}
func TestSQLiteDatabaseExists(t *testing.T) {
drv := SQLiteDriver{}
u := sqliteTestURL(t)
// drop any existing database
err := drv.DropDatabase(u)
require.NoError(t, err)
// DatabaseExists should return false
exists, err := drv.DatabaseExists(u)
require.NoError(t, err)
require.Equal(t, false, exists)
// create database
err = drv.CreateDatabase(u)
require.NoError(t, err)
// DatabaseExists should return true
exists, err = drv.DatabaseExists(u)
require.NoError(t, err)
require.Equal(t, true, exists)
}
func TestSQLiteCreateMigrationsTable(t *testing.T) {
drv := SQLiteDriver{}
db := prepTestSQLiteDB(t)
defer mustClose(db)
// migrations table should not exist
count := 0
err := db.QueryRow("select count(*) from schema_migrations").Scan(&count)
require.Regexp(t, "no such table: schema_migrations", err.Error())
// create table
err = drv.CreateMigrationsTable(db)
require.NoError(t, err)
// migrations table should exist
err = db.QueryRow("select count(*) from schema_migrations").Scan(&count)
require.NoError(t, err)
// create table should be idempotent
err = drv.CreateMigrationsTable(db)
require.NoError(t, err)
}
func TestSQLiteSelectMigrations(t *testing.T) {
drv := SQLiteDriver{}
db := prepTestSQLiteDB(t)
defer mustClose(db)
err := drv.CreateMigrationsTable(db)
require.NoError(t, err)
_, err = db.Exec(`insert into schema_migrations (version)
values ('abc2'), ('abc1'), ('abc3')`)
require.NoError(t, err)
migrations, err := drv.SelectMigrations(db, -1)
require.NoError(t, err)
require.Equal(t, true, migrations["abc1"])
require.Equal(t, true, migrations["abc2"])
require.Equal(t, true, migrations["abc2"])
// test limit param
migrations, err = drv.SelectMigrations(db, 1)
require.NoError(t, err)
require.Equal(t, true, migrations["abc3"])
require.Equal(t, false, migrations["abc1"])
require.Equal(t, false, migrations["abc2"])
}
func TestSQLiteInsertMigration(t *testing.T) {
drv := SQLiteDriver{}
db := prepTestSQLiteDB(t)
defer mustClose(db)
err := drv.CreateMigrationsTable(db)
require.NoError(t, err)
count := 0
err = db.QueryRow("select count(*) from schema_migrations").Scan(&count)
require.NoError(t, err)
require.Equal(t, 0, count)
// insert migration
err = drv.InsertMigration(db, "abc1")
require.NoError(t, err)
err = db.QueryRow("select count(*) from schema_migrations where version = 'abc1'").
Scan(&count)
require.NoError(t, err)
require.Equal(t, 1, count)
}
func TestSQLiteDeleteMigration(t *testing.T) {
drv := SQLiteDriver{}
db := prepTestSQLiteDB(t)
defer mustClose(db)
err := drv.CreateMigrationsTable(db)
require.NoError(t, err)
_, err = db.Exec(`insert into schema_migrations (version)
values ('abc1'), ('abc2')`)
require.NoError(t, err)
err = drv.DeleteMigration(db, "abc2")
require.NoError(t, err)
count := 0
err = db.QueryRow("select count(*) from schema_migrations").Scan(&count)
require.NoError(t, err)
require.Equal(t, 1, count)
}
func TestSQLitePing(t *testing.T) {
drv := SQLiteDriver{}
u := sqliteTestURL(t)
path := sqlitePath(u)
// drop any existing database
err := drv.DropDatabase(u)
require.NoError(t, err)
// ping database
err = drv.Ping(u)
require.NoError(t, err)
// check that the database was created (sqlite-only behavior)
_, err = os.Stat(path)
require.NoError(t, err)
// drop the database
err = drv.DropDatabase(u)
require.NoError(t, err)
// create directory where database file is expected
err = os.Mkdir(path, 0755)
require.NoError(t, err)
defer func() {
err = os.RemoveAll(path)
require.NoError(t, err)
}()
// ping database should fail
err = drv.Ping(u)
require.EqualError(t, err, "unable to open database file: is a directory")
}

View file

@ -1,26 +1,21 @@
package dbutil
package dbmate
import (
"bufio"
"bytes"
"database/sql"
"errors"
"fmt"
"io"
"net/url"
"os"
"os/exec"
"strings"
"unicode"
)
// Transaction can represent a database or open transaction
type Transaction interface {
Exec(query string, args ...interface{}) (sql.Result, error)
Query(query string, args ...interface{}) (*sql.Rows, error)
QueryRow(query string, args ...interface{}) *sql.Row
}
// DatabaseName returns the database name from a URL
func DatabaseName(u *url.URL) string {
// databaseName returns the database name from a URL
func databaseName(u *url.URL) string {
name := u.Path
if len(name) > 0 && name[:1] == "/" {
name = name[1:]
@ -29,15 +24,24 @@ func DatabaseName(u *url.URL) string {
return name
}
// MustClose ensures a stream is closed
func MustClose(c io.Closer) {
// mustClose ensures a stream is closed
func mustClose(c io.Closer) {
if err := c.Close(); err != nil {
panic(err)
}
}
// RunCommand runs a command and returns the stdout if successful
func RunCommand(name string, args ...string) ([]byte, error) {
// ensureDir creates a directory if it does not already exist
func ensureDir(dir string) error {
if err := os.MkdirAll(dir, 0755); err != nil {
return fmt.Errorf("unable to create directory `%s`", dir)
}
return nil
}
// runCommand runs a command and returns the stdout if successful
func runCommand(name string, args ...string) ([]byte, error) {
var stdout, stderr bytes.Buffer
cmd := exec.Command(name, args...)
cmd.Stdout = &stdout
@ -57,10 +61,10 @@ func RunCommand(name string, args ...string) ([]byte, error) {
return stdout.Bytes(), nil
}
// TrimLeadingSQLComments removes sql comments and blank lines from the beginning of text
// trimLeadingSQLComments removes sql comments and blank lines from the beginning of text
// generally when performing sql dumps these contain host-specific information such as
// client/server version numbers
func TrimLeadingSQLComments(data []byte) ([]byte, error) {
func trimLeadingSQLComments(data []byte) ([]byte, error) {
// create decent size buffer
out := bytes.NewBuffer(make([]byte, 0, len(data)))
@ -97,15 +101,15 @@ func TrimLeadingSQLComments(data []byte) ([]byte, error) {
return out.Bytes(), nil
}
// QueryColumn runs a SQL statement and returns a slice of strings
// queryColumn runs a SQL statement and returns a slice of strings
// it is assumed that the statement returns only one column
// e.g. schema_migrations table
func QueryColumn(db Transaction, query string, args ...interface{}) ([]string, error) {
rows, err := db.Query(query, args...)
func queryColumn(db *sql.DB, query string) ([]string, error) {
rows, err := db.Query(query)
if err != nil {
return nil, err
}
defer MustClose(rows)
defer mustClose(rows)
// read into slice
var result []string
@ -124,30 +128,13 @@ func QueryColumn(db Transaction, query string, args ...interface{}) ([]string, e
return result, nil
}
// QueryValue runs a SQL statement and returns a single string
// it is assumed that the statement returns only one row and one column
// sql NULL is returned as empty string
func QueryValue(db Transaction, query string, args ...interface{}) (string, error) {
var result sql.NullString
err := db.QueryRow(query, args...).Scan(&result)
if err != nil || !result.Valid {
return "", err
func printVerbose(result sql.Result) {
lastInsertId, err := result.LastInsertId()
if err == nil {
fmt.Printf("Last insert ID: %d\n", lastInsertId)
}
rowsAffected, err := result.RowsAffected()
if err == nil {
fmt.Printf("Rows affected: %d\n", rowsAffected)
}
return result.String, nil
}
// MustParseURL parses a URL from string, and panics if it fails.
// It is used during testing and in cases where we are parsing a generated URL.
func MustParseURL(s string) *url.URL {
if s == "" {
panic("missing url")
}
u, err := url.Parse(s)
if err != nil {
panic(err)
}
return u
}

35
pkg/dbmate/utils_test.go Normal file
View file

@ -0,0 +1,35 @@
package dbmate
import (
"net/url"
"testing"
"github.com/stretchr/testify/require"
)
func TestDatabaseName(t *testing.T) {
u, err := url.Parse("ignore://localhost/foo?query")
require.NoError(t, err)
name := databaseName(u)
require.Equal(t, "foo", name)
}
func TestDatabaseName_Empty(t *testing.T) {
u, err := url.Parse("ignore://localhost")
require.NoError(t, err)
name := databaseName(u)
require.Equal(t, "", name)
}
func TestTrimLeadingSQLComments(t *testing.T) {
in := "--\n" +
"-- foo\n\n" +
"-- bar\n\n" +
"real stuff\n" +
"-- end\n"
out, err := trimLeadingSQLComments([]byte(in))
require.NoError(t, err)
require.Equal(t, "real stuff\n-- end\n", string(out))
}

View file

@ -1,4 +1,4 @@
package dbmate
// Version of dbmate
const Version = "1.12.1"
const Version = "1.10.0"

View file

@ -1,58 +0,0 @@
package dbutil_test
import (
"database/sql"
"testing"
"github.com/amacneil/dbmate/pkg/dbutil"
_ "github.com/mattn/go-sqlite3" // database/sql driver
"github.com/stretchr/testify/require"
)
func TestDatabaseName(t *testing.T) {
t.Run("valid", func(t *testing.T) {
u := dbutil.MustParseURL("foo://host/dbname?query")
name := dbutil.DatabaseName(u)
require.Equal(t, "dbname", name)
})
t.Run("empty", func(t *testing.T) {
u := dbutil.MustParseURL("foo://host")
name := dbutil.DatabaseName(u)
require.Equal(t, "", name)
})
}
func TestTrimLeadingSQLComments(t *testing.T) {
in := "--\n" +
"-- foo\n\n" +
"-- bar\n\n" +
"real stuff\n" +
"-- end\n"
out, err := dbutil.TrimLeadingSQLComments([]byte(in))
require.NoError(t, err)
require.Equal(t, "real stuff\n-- end\n", string(out))
}
// connect to in-memory sqlite database for testing
const sqliteMemoryDB = "file:dbutil.sqlite3?mode=memory&cache=shared"
func TestQueryColumn(t *testing.T) {
db, err := sql.Open("sqlite3", sqliteMemoryDB)
require.NoError(t, err)
val, err := dbutil.QueryColumn(db, "select 'foo_' || val from (select ? as val union select ?)",
"hi", "there")
require.NoError(t, err)
require.Equal(t, []string{"foo_hi", "foo_there"}, val)
}
func TestQueryValue(t *testing.T) {
db, err := sql.Open("sqlite3", sqliteMemoryDB)
require.NoError(t, err)
val, err := dbutil.QueryValue(db, "select $1 + $2", "5", 2)
require.NoError(t, err)
require.Equal(t, "7", val)
}

View file

@ -1,329 +0,0 @@
package clickhouse
import (
"bytes"
"database/sql"
"fmt"
"io"
"net/url"
"regexp"
"sort"
"strings"
"github.com/amacneil/dbmate/pkg/dbmate"
"github.com/amacneil/dbmate/pkg/dbutil"
"github.com/ClickHouse/clickhouse-go"
)
func init() {
dbmate.RegisterDriver(NewDriver, "clickhouse")
}
// Driver provides top level database functions
type Driver struct {
migrationsTableName string
databaseURL *url.URL
log io.Writer
}
// NewDriver initializes the driver
func NewDriver(config dbmate.DriverConfig) dbmate.Driver {
return &Driver{
migrationsTableName: config.MigrationsTableName,
databaseURL: config.DatabaseURL,
log: config.Log,
}
}
func connectionString(initialURL *url.URL) string {
u := *initialURL
u.Scheme = "tcp"
host := u.Host
if u.Port() == "" {
host = fmt.Sprintf("%s:9000", host)
}
u.Host = host
query := u.Query()
if query.Get("username") == "" && u.User.Username() != "" {
query.Set("username", u.User.Username())
}
password, passwordSet := u.User.Password()
if query.Get("password") == "" && passwordSet {
query.Set("password", password)
}
u.User = nil
if query.Get("database") == "" {
path := strings.Trim(u.Path, "/")
if path != "" {
query.Set("database", path)
u.Path = ""
}
}
u.RawQuery = query.Encode()
return u.String()
}
// Open creates a new database connection
func (drv *Driver) Open() (*sql.DB, error) {
return sql.Open("clickhouse", connectionString(drv.databaseURL))
}
func (drv *Driver) openClickHouseDB() (*sql.DB, error) {
// clone databaseURL
clickhouseURL, err := url.Parse(connectionString(drv.databaseURL))
if err != nil {
return nil, err
}
// connect to clickhouse database
values := clickhouseURL.Query()
values.Set("database", "default")
clickhouseURL.RawQuery = values.Encode()
return sql.Open("clickhouse", clickhouseURL.String())
}
func (drv *Driver) databaseName() string {
name := dbutil.MustParseURL(connectionString(drv.databaseURL)).Query().Get("database")
if name == "" {
name = "default"
}
return name
}
var clickhouseValidIdentifier = regexp.MustCompile(`^[a-zA-Z_][0-9a-zA-Z_]*$`)
func (drv *Driver) quoteIdentifier(str string) string {
if clickhouseValidIdentifier.MatchString(str) {
return str
}
str = strings.Replace(str, `"`, `""`, -1)
return fmt.Sprintf(`"%s"`, str)
}
// CreateDatabase creates the specified database
func (drv *Driver) CreateDatabase() error {
name := drv.databaseName()
fmt.Fprintf(drv.log, "Creating: %s\n", name)
db, err := drv.openClickHouseDB()
if err != nil {
return err
}
defer dbutil.MustClose(db)
_, err = db.Exec("create database " + drv.quoteIdentifier(name))
return err
}
// DropDatabase drops the specified database (if it exists)
func (drv *Driver) DropDatabase() error {
name := drv.databaseName()
fmt.Fprintf(drv.log, "Dropping: %s\n", name)
db, err := drv.openClickHouseDB()
if err != nil {
return err
}
defer dbutil.MustClose(db)
_, err = db.Exec("drop database if exists " + drv.quoteIdentifier(name))
return err
}
func (drv *Driver) schemaDump(db *sql.DB, buf *bytes.Buffer, databaseName string) error {
buf.WriteString("\n--\n-- Database schema\n--\n\n")
buf.WriteString("CREATE DATABASE " + drv.quoteIdentifier(databaseName) + " IF NOT EXISTS;\n\n")
tables, err := dbutil.QueryColumn(db, "show tables")
if err != nil {
return err
}
sort.Strings(tables)
for _, table := range tables {
var clause string
err = db.QueryRow("show create table " + drv.quoteIdentifier(table)).Scan(&clause)
if err != nil {
return err
}
buf.WriteString(clause + ";\n\n")
}
return nil
}
func (drv *Driver) schemaMigrationsDump(db *sql.DB, buf *bytes.Buffer) error {
migrationsTable := drv.quotedMigrationsTableName()
// load applied migrations
migrations, err := dbutil.QueryColumn(db,
fmt.Sprintf("select version from %s final ", migrationsTable)+
"where applied order by version asc",
)
if err != nil {
return err
}
quoter := strings.NewReplacer(`\`, `\\`, `'`, `\'`)
for i := range migrations {
migrations[i] = "'" + quoter.Replace(migrations[i]) + "'"
}
// build schema migrations table data
buf.WriteString("\n--\n-- Dbmate schema migrations\n--\n\n")
if len(migrations) > 0 {
buf.WriteString(
fmt.Sprintf("INSERT INTO %s (version) VALUES\n (", migrationsTable) +
strings.Join(migrations, "),\n (") +
");\n")
}
return nil
}
// DumpSchema returns the current database schema
func (drv *Driver) DumpSchema(db *sql.DB) ([]byte, error) {
var buf bytes.Buffer
var err error
err = drv.schemaDump(db, &buf, drv.databaseName())
if err != nil {
return nil, err
}
err = drv.schemaMigrationsDump(db, &buf)
if err != nil {
return nil, err
}
return buf.Bytes(), nil
}
// DatabaseExists determines whether the database exists
func (drv *Driver) DatabaseExists() (bool, error) {
name := drv.databaseName()
db, err := drv.openClickHouseDB()
if err != nil {
return false, err
}
defer dbutil.MustClose(db)
exists := false
err = db.QueryRow("SELECT 1 FROM system.databases where name = ?", name).
Scan(&exists)
if err == sql.ErrNoRows {
return false, nil
}
return exists, err
}
// CreateMigrationsTable creates the schema migrations table
func (drv *Driver) CreateMigrationsTable(db *sql.DB) error {
_, err := db.Exec(fmt.Sprintf(`
create table if not exists %s (
version String,
ts DateTime default now(),
applied UInt8 default 1
) engine = ReplacingMergeTree(ts)
primary key version
order by version
`, drv.quotedMigrationsTableName()))
return err
}
// SelectMigrations returns a list of applied migrations
// with an optional limit (in descending order)
func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, error) {
query := fmt.Sprintf("select version from %s final where applied order by version desc",
drv.quotedMigrationsTableName())
if limit >= 0 {
query = fmt.Sprintf("%s limit %d", query, limit)
}
rows, err := db.Query(query)
if err != nil {
return nil, err
}
defer dbutil.MustClose(rows)
migrations := map[string]bool{}
for rows.Next() {
var version string
if err := rows.Scan(&version); err != nil {
return nil, err
}
migrations[version] = true
}
if err = rows.Err(); err != nil {
return nil, err
}
return migrations, nil
}
// InsertMigration adds a new migration record
func (drv *Driver) InsertMigration(db dbutil.Transaction, version string) error {
_, err := db.Exec(
fmt.Sprintf("insert into %s (version) values (?)", drv.quotedMigrationsTableName()),
version)
return err
}
// DeleteMigration removes a migration record
func (drv *Driver) DeleteMigration(db dbutil.Transaction, version string) error {
_, err := db.Exec(
fmt.Sprintf("insert into %s (version, applied) values (?, ?)",
drv.quotedMigrationsTableName()),
version, false,
)
return err
}
// Ping verifies a connection to the database server. It does not verify whether the
// specified database exists.
func (drv *Driver) Ping() error {
// attempt connection to primary database, not "clickhouse" database
// to support servers with no "clickhouse" database
// (see https://github.com/amacneil/dbmate/issues/78)
db, err := drv.Open()
if err != nil {
return err
}
defer dbutil.MustClose(db)
err = db.Ping()
if err == nil {
return nil
}
// ignore 'Database foo doesn't exist' error
chErr, ok := err.(*clickhouse.Exception)
if ok && chErr.Code == 81 {
return nil
}
return err
}
func (drv *Driver) quotedMigrationsTableName() string {
return drv.quoteIdentifier(drv.migrationsTableName)
}

View file

@ -1,376 +0,0 @@
package clickhouse
import (
"database/sql"
"net/url"
"os"
"testing"
"github.com/amacneil/dbmate/pkg/dbmate"
"github.com/amacneil/dbmate/pkg/dbutil"
"github.com/stretchr/testify/require"
)
func testClickHouseDriver(t *testing.T) *Driver {
u := dbutil.MustParseURL(os.Getenv("CLICKHOUSE_TEST_URL"))
drv, err := dbmate.New(u).GetDriver()
require.NoError(t, err)
return drv.(*Driver)
}
func prepTestClickHouseDB(t *testing.T) *sql.DB {
drv := testClickHouseDriver(t)
// drop any existing database
err := drv.DropDatabase()
require.NoError(t, err)
// create database
err = drv.CreateDatabase()
require.NoError(t, err)
// connect database
db, err := sql.Open("clickhouse", drv.databaseURL.String())
require.NoError(t, err)
return db
}
func TestGetDriver(t *testing.T) {
db := dbmate.New(dbutil.MustParseURL("clickhouse://"))
drvInterface, err := db.GetDriver()
require.NoError(t, err)
// driver should have URL and default migrations table set
drv, ok := drvInterface.(*Driver)
require.True(t, ok)
require.Equal(t, db.DatabaseURL.String(), drv.databaseURL.String())
require.Equal(t, "schema_migrations", drv.migrationsTableName)
}
func TestConnectionString(t *testing.T) {
t.Run("simple", func(t *testing.T) {
u, err := url.Parse("clickhouse://user:pass@host/db")
require.NoError(t, err)
s := connectionString(u)
require.Equal(t, "tcp://host:9000?database=db&password=pass&username=user", s)
})
t.Run("canonical", func(t *testing.T) {
u, err := url.Parse("clickhouse://host:9000?database=db&password=pass&username=user")
require.NoError(t, err)
s := connectionString(u)
require.Equal(t, "tcp://host:9000?database=db&password=pass&username=user", s)
})
}
func TestClickHouseCreateDropDatabase(t *testing.T) {
drv := testClickHouseDriver(t)
// drop any existing database
err := drv.DropDatabase()
require.NoError(t, err)
// create database
err = drv.CreateDatabase()
require.NoError(t, err)
// check that database exists and we can connect to it
func() {
db, err := sql.Open("clickhouse", drv.databaseURL.String())
require.NoError(t, err)
defer dbutil.MustClose(db)
err = db.Ping()
require.NoError(t, err)
}()
// drop the database
err = drv.DropDatabase()
require.NoError(t, err)
// check that database no longer exists
func() {
db, err := sql.Open("clickhouse", drv.databaseURL.String())
require.NoError(t, err)
defer dbutil.MustClose(db)
err = db.Ping()
require.EqualError(t, err, "code: 81, message: Database dbmate_test doesn't exist")
}()
}
func TestClickHouseDumpSchema(t *testing.T) {
drv := testClickHouseDriver(t)
drv.migrationsTableName = "test_migrations"
// prepare database
db := prepTestClickHouseDB(t)
defer dbutil.MustClose(db)
err := drv.CreateMigrationsTable(db)
require.NoError(t, err)
// insert migration
tx, err := db.Begin()
require.NoError(t, err)
err = drv.InsertMigration(tx, "abc1")
require.NoError(t, err)
err = tx.Commit()
require.NoError(t, err)
tx, err = db.Begin()
require.NoError(t, err)
err = drv.InsertMigration(tx, "abc2")
require.NoError(t, err)
err = tx.Commit()
require.NoError(t, err)
// DumpSchema should return schema
schema, err := drv.DumpSchema(db)
require.NoError(t, err)
require.Contains(t, string(schema), "CREATE TABLE "+drv.databaseName()+".test_migrations")
require.Contains(t, string(schema), "--\n"+
"-- Dbmate schema migrations\n"+
"--\n\n"+
"INSERT INTO test_migrations (version) VALUES\n"+
" ('abc1'),\n"+
" ('abc2');\n")
// DumpSchema should return error if command fails
values := drv.databaseURL.Query()
values.Set("database", "fakedb")
drv.databaseURL.RawQuery = values.Encode()
db, err = sql.Open("clickhouse", drv.databaseURL.String())
require.NoError(t, err)
schema, err = drv.DumpSchema(db)
require.Nil(t, schema)
require.EqualError(t, err, "code: 81, message: Database fakedb doesn't exist")
}
func TestClickHouseDatabaseExists(t *testing.T) {
drv := testClickHouseDriver(t)
// drop any existing database
err := drv.DropDatabase()
require.NoError(t, err)
// DatabaseExists should return false
exists, err := drv.DatabaseExists()
require.NoError(t, err)
require.Equal(t, false, exists)
// create database
err = drv.CreateDatabase()
require.NoError(t, err)
// DatabaseExists should return true
exists, err = drv.DatabaseExists()
require.NoError(t, err)
require.Equal(t, true, exists)
}
func TestClickHouseDatabaseExists_Error(t *testing.T) {
drv := testClickHouseDriver(t)
values := drv.databaseURL.Query()
values.Set("username", "invalid")
drv.databaseURL.RawQuery = values.Encode()
exists, err := drv.DatabaseExists()
require.EqualError(t, err, "code: 192, message: Unknown user invalid")
require.Equal(t, false, exists)
}
func TestClickHouseCreateMigrationsTable(t *testing.T) {
t.Run("default table", func(t *testing.T) {
drv := testClickHouseDriver(t)
db := prepTestClickHouseDB(t)
defer dbutil.MustClose(db)
// migrations table should not exist
count := 0
err := db.QueryRow("select count(*) from schema_migrations").Scan(&count)
require.EqualError(t, err, "code: 60, message: Table dbmate_test.schema_migrations doesn't exist.")
// create table
err = drv.CreateMigrationsTable(db)
require.NoError(t, err)
// migrations table should exist
err = db.QueryRow("select count(*) from schema_migrations").Scan(&count)
require.NoError(t, err)
// create table should be idempotent
err = drv.CreateMigrationsTable(db)
require.NoError(t, err)
})
t.Run("custom table", func(t *testing.T) {
drv := testClickHouseDriver(t)
drv.migrationsTableName = "testMigrations"
db := prepTestClickHouseDB(t)
defer dbutil.MustClose(db)
// migrations table should not exist
count := 0
err := db.QueryRow("select count(*) from \"testMigrations\"").Scan(&count)
require.EqualError(t, err, "code: 60, message: Table dbmate_test.testMigrations doesn't exist.")
// create table
err = drv.CreateMigrationsTable(db)
require.NoError(t, err)
// migrations table should exist
err = db.QueryRow("select count(*) from \"testMigrations\"").Scan(&count)
require.NoError(t, err)
// create table should be idempotent
err = drv.CreateMigrationsTable(db)
require.NoError(t, err)
})
}
func TestClickHouseSelectMigrations(t *testing.T) {
drv := testClickHouseDriver(t)
drv.migrationsTableName = "test_migrations"
db := prepTestClickHouseDB(t)
defer dbutil.MustClose(db)
err := drv.CreateMigrationsTable(db)
require.NoError(t, err)
tx, err := db.Begin()
require.NoError(t, err)
stmt, err := tx.Prepare("insert into test_migrations (version) values (?)")
require.NoError(t, err)
_, err = stmt.Exec("abc2")
require.NoError(t, err)
_, err = stmt.Exec("abc1")
require.NoError(t, err)
_, err = stmt.Exec("abc3")
require.NoError(t, err)
err = tx.Commit()
require.NoError(t, err)
migrations, err := drv.SelectMigrations(db, -1)
require.NoError(t, err)
require.Equal(t, true, migrations["abc1"])
require.Equal(t, true, migrations["abc2"])
require.Equal(t, true, migrations["abc2"])
// test limit param
migrations, err = drv.SelectMigrations(db, 1)
require.NoError(t, err)
require.Equal(t, true, migrations["abc3"])
require.Equal(t, false, migrations["abc1"])
require.Equal(t, false, migrations["abc2"])
}
func TestClickHouseInsertMigration(t *testing.T) {
drv := testClickHouseDriver(t)
drv.migrationsTableName = "test_migrations"
db := prepTestClickHouseDB(t)
defer dbutil.MustClose(db)
err := drv.CreateMigrationsTable(db)
require.NoError(t, err)
count := 0
err = db.QueryRow("select count(*) from test_migrations").Scan(&count)
require.NoError(t, err)
require.Equal(t, 0, count)
// insert migration
tx, err := db.Begin()
require.NoError(t, err)
err = drv.InsertMigration(tx, "abc1")
require.NoError(t, err)
err = tx.Commit()
require.NoError(t, err)
err = db.QueryRow("select count(*) from test_migrations where version = 'abc1'").Scan(&count)
require.NoError(t, err)
require.Equal(t, 1, count)
}
func TestClickHouseDeleteMigration(t *testing.T) {
drv := testClickHouseDriver(t)
drv.migrationsTableName = "test_migrations"
db := prepTestClickHouseDB(t)
defer dbutil.MustClose(db)
err := drv.CreateMigrationsTable(db)
require.NoError(t, err)
tx, err := db.Begin()
require.NoError(t, err)
stmt, err := tx.Prepare("insert into test_migrations (version) values (?)")
require.NoError(t, err)
_, err = stmt.Exec("abc2")
require.NoError(t, err)
_, err = stmt.Exec("abc1")
require.NoError(t, err)
err = tx.Commit()
require.NoError(t, err)
tx, err = db.Begin()
require.NoError(t, err)
err = drv.DeleteMigration(tx, "abc2")
require.NoError(t, err)
err = tx.Commit()
require.NoError(t, err)
count := 0
err = db.QueryRow("select count(*) from test_migrations final where applied").Scan(&count)
require.NoError(t, err)
require.Equal(t, 1, count)
}
func TestClickHousePing(t *testing.T) {
drv := testClickHouseDriver(t)
// drop any existing database
err := drv.DropDatabase()
require.NoError(t, err)
// ping database
err = drv.Ping()
require.NoError(t, err)
// ping invalid host should return error
drv.databaseURL.Host = "clickhouse:404"
err = drv.Ping()
require.Error(t, err)
require.Contains(t, err.Error(), "connect: connection refused")
}
func TestClickHouseQuotedMigrationsTableName(t *testing.T) {
t.Run("default name", func(t *testing.T) {
drv := testClickHouseDriver(t)
name := drv.quotedMigrationsTableName()
require.Equal(t, "schema_migrations", name)
})
t.Run("custom name", func(t *testing.T) {
drv := testClickHouseDriver(t)
drv.migrationsTableName = "fooMigrations"
name := drv.quotedMigrationsTableName()
require.Equal(t, "fooMigrations", name)
})
t.Run("quoted name", func(t *testing.T) {
drv := testClickHouseDriver(t)
drv.migrationsTableName = "bizarre\"$name"
name := drv.quotedMigrationsTableName()
require.Equal(t, `"bizarre""$name"`, name)
})
}

View file

@ -1,290 +0,0 @@
package mysql
import (
"bytes"
"database/sql"
"fmt"
"io"
"net/url"
"strings"
"github.com/amacneil/dbmate/pkg/dbmate"
"github.com/amacneil/dbmate/pkg/dbutil"
_ "github.com/go-sql-driver/mysql" // database/sql driver
)
func init() {
dbmate.RegisterDriver(NewDriver, "mysql")
}
// Driver provides top level database functions
type Driver struct {
migrationsTableName string
databaseURL *url.URL
log io.Writer
}
// NewDriver initializes the driver
func NewDriver(config dbmate.DriverConfig) dbmate.Driver {
return &Driver{
migrationsTableName: config.MigrationsTableName,
databaseURL: config.DatabaseURL,
log: config.Log,
}
}
func connectionString(u *url.URL) string {
query := u.Query()
query.Set("multiStatements", "true")
host := u.Host
protocol := "tcp"
if query.Get("socket") != "" {
protocol = "unix"
host = query.Get("socket")
query.Del("socket")
} else if u.Port() == "" {
// set default port
host = fmt.Sprintf("%s:3306", host)
}
// Get decoded user:pass
userPassEncoded := u.User.String()
userPass, _ := url.PathUnescape(userPassEncoded)
// Build DSN w/ user:pass percent-decoded
normalizedString := ""
if userPass != "" { // user:pass can be empty
normalizedString = userPass + "@"
}
// connection string format required by go-sql-driver/mysql
normalizedString = fmt.Sprintf("%s%s(%s)%s?%s", normalizedString,
protocol, host, u.Path, query.Encode())
return normalizedString
}
// Open creates a new database connection
func (drv *Driver) Open() (*sql.DB, error) {
return sql.Open("mysql", connectionString(drv.databaseURL))
}
func (drv *Driver) openRootDB() (*sql.DB, error) {
// clone databaseURL
rootURL, err := url.Parse(drv.databaseURL.String())
if err != nil {
return nil, err
}
// connect to no particular database
rootURL.Path = "/"
return sql.Open("mysql", connectionString(rootURL))
}
func (drv *Driver) quoteIdentifier(str string) string {
str = strings.Replace(str, "`", "\\`", -1)
return fmt.Sprintf("`%s`", str)
}
// CreateDatabase creates the specified database
func (drv *Driver) CreateDatabase() error {
name := dbutil.DatabaseName(drv.databaseURL)
fmt.Fprintf(drv.log, "Creating: %s\n", name)
db, err := drv.openRootDB()
if err != nil {
return err
}
defer dbutil.MustClose(db)
_, err = db.Exec(fmt.Sprintf("create database %s",
drv.quoteIdentifier(name)))
return err
}
// DropDatabase drops the specified database (if it exists)
func (drv *Driver) DropDatabase() error {
name := dbutil.DatabaseName(drv.databaseURL)
fmt.Fprintf(drv.log, "Dropping: %s\n", name)
db, err := drv.openRootDB()
if err != nil {
return err
}
defer dbutil.MustClose(db)
_, err = db.Exec(fmt.Sprintf("drop database if exists %s",
drv.quoteIdentifier(name)))
return err
}
func (drv *Driver) mysqldumpArgs() []string {
// generate CLI arguments
args := []string{"--opt", "--routines", "--no-data",
"--skip-dump-date", "--skip-add-drop-table"}
if hostname := drv.databaseURL.Hostname(); hostname != "" {
args = append(args, "--host="+hostname)
}
if port := drv.databaseURL.Port(); port != "" {
args = append(args, "--port="+port)
}
if username := drv.databaseURL.User.Username(); username != "" {
args = append(args, "--user="+username)
}
// mysql recommends against using environment variables to supply password
// https://dev.mysql.com/doc/refman/5.7/en/password-security-user.html
if password, set := drv.databaseURL.User.Password(); set {
args = append(args, "--password="+password)
}
// add database name
args = append(args, dbutil.DatabaseName(drv.databaseURL))
return args
}
func (drv *Driver) schemaMigrationsDump(db *sql.DB) ([]byte, error) {
migrationsTable := drv.quotedMigrationsTableName()
// load applied migrations
migrations, err := dbutil.QueryColumn(db,
fmt.Sprintf("select quote(version) from %s order by version asc", migrationsTable))
if err != nil {
return nil, err
}
// build schema_migrations table data
var buf bytes.Buffer
buf.WriteString("\n--\n-- Dbmate schema migrations\n--\n\n" +
fmt.Sprintf("LOCK TABLES %s WRITE;\n", migrationsTable))
if len(migrations) > 0 {
buf.WriteString(
fmt.Sprintf("INSERT INTO %s (version) VALUES\n (", migrationsTable) +
strings.Join(migrations, "),\n (") +
");\n")
}
buf.WriteString("UNLOCK TABLES;\n")
return buf.Bytes(), nil
}
// DumpSchema returns the current database schema
func (drv *Driver) DumpSchema(db *sql.DB) ([]byte, error) {
schema, err := dbutil.RunCommand("mysqldump", drv.mysqldumpArgs()...)
if err != nil {
return nil, err
}
migrations, err := drv.schemaMigrationsDump(db)
if err != nil {
return nil, err
}
schema = append(schema, migrations...)
return dbutil.TrimLeadingSQLComments(schema)
}
// DatabaseExists determines whether the database exists
func (drv *Driver) DatabaseExists() (bool, error) {
name := dbutil.DatabaseName(drv.databaseURL)
db, err := drv.openRootDB()
if err != nil {
return false, err
}
defer dbutil.MustClose(db)
exists := false
err = db.QueryRow("select true from information_schema.schemata "+
"where schema_name = ?", name).Scan(&exists)
if err == sql.ErrNoRows {
return false, nil
}
return exists, err
}
// CreateMigrationsTable creates the schema_migrations table
func (drv *Driver) CreateMigrationsTable(db *sql.DB) error {
_, err := db.Exec(fmt.Sprintf("create table if not exists %s "+
"(version varchar(255) primary key) character set latin1 collate latin1_bin",
drv.quotedMigrationsTableName()))
return err
}
// SelectMigrations returns a list of applied migrations
// with an optional limit (in descending order)
func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, error) {
query := fmt.Sprintf("select version from %s order by version desc", drv.quotedMigrationsTableName())
if limit >= 0 {
query = fmt.Sprintf("%s limit %d", query, limit)
}
rows, err := db.Query(query)
if err != nil {
return nil, err
}
defer dbutil.MustClose(rows)
migrations := map[string]bool{}
for rows.Next() {
var version string
if err := rows.Scan(&version); err != nil {
return nil, err
}
migrations[version] = true
}
if err = rows.Err(); err != nil {
return nil, err
}
return migrations, nil
}
// InsertMigration adds a new migration record
func (drv *Driver) InsertMigration(db dbutil.Transaction, version string) error {
_, err := db.Exec(
fmt.Sprintf("insert into %s (version) values (?)", drv.quotedMigrationsTableName()),
version)
return err
}
// DeleteMigration removes a migration record
func (drv *Driver) DeleteMigration(db dbutil.Transaction, version string) error {
_, err := db.Exec(
fmt.Sprintf("delete from %s where version = ?", drv.quotedMigrationsTableName()),
version)
return err
}
// Ping verifies a connection to the database server. It does not verify whether the
// specified database exists.
func (drv *Driver) Ping() error {
db, err := drv.openRootDB()
if err != nil {
return err
}
defer dbutil.MustClose(db)
return db.Ping()
}
func (drv *Driver) quotedMigrationsTableName() string {
return drv.quoteIdentifier(drv.migrationsTableName)
}

View file

@ -1,352 +0,0 @@
package mysql
import (
"database/sql"
"net/url"
"os"
"testing"
"github.com/amacneil/dbmate/pkg/dbmate"
"github.com/amacneil/dbmate/pkg/dbutil"
"github.com/stretchr/testify/require"
)
func testMySQLDriver(t *testing.T) *Driver {
u := dbutil.MustParseURL(os.Getenv("MYSQL_TEST_URL"))
drv, err := dbmate.New(u).GetDriver()
require.NoError(t, err)
return drv.(*Driver)
}
func prepTestMySQLDB(t *testing.T) *sql.DB {
drv := testMySQLDriver(t)
// drop any existing database
err := drv.DropDatabase()
require.NoError(t, err)
// create database
err = drv.CreateDatabase()
require.NoError(t, err)
// connect database
db, err := drv.Open()
require.NoError(t, err)
return db
}
func TestGetDriver(t *testing.T) {
db := dbmate.New(dbutil.MustParseURL("mysql://"))
drvInterface, err := db.GetDriver()
require.NoError(t, err)
// driver should have URL and default migrations table set
drv, ok := drvInterface.(*Driver)
require.True(t, ok)
require.Equal(t, db.DatabaseURL.String(), drv.databaseURL.String())
require.Equal(t, "schema_migrations", drv.migrationsTableName)
}
func TestConnectionString(t *testing.T) {
t.Run("defaults", func(t *testing.T) {
u, err := url.Parse("mysql://host/foo")
require.NoError(t, err)
require.Equal(t, "", u.Port())
s := connectionString(u)
require.Equal(t, "tcp(host:3306)/foo?multiStatements=true", s)
})
t.Run("custom", func(t *testing.T) {
u, err := url.Parse("mysql://bob:secret@host:123/foo?flag=on")
require.NoError(t, err)
require.Equal(t, "123", u.Port())
s := connectionString(u)
require.Equal(t, "bob:secret@tcp(host:123)/foo?flag=on&multiStatements=true", s)
})
t.Run("special chars", func(t *testing.T) {
u, err := url.Parse("mysql://duhfsd7s:123!@123!@@host:123/foo?flag=on")
require.NoError(t, err)
require.Equal(t, "123", u.Port())
s := connectionString(u)
require.Equal(t, "duhfsd7s:123!@123!@@tcp(host:123)/foo?flag=on&multiStatements=true", s)
})
t.Run("url encoding", func(t *testing.T) {
u, err := url.Parse("mysql://bob%2Balice:secret%5E%5B%2A%28%29@host:123/foo")
require.NoError(t, err)
require.Equal(t, "bob+alice:secret%5E%5B%2A%28%29", u.User.String())
require.Equal(t, "123", u.Port())
s := connectionString(u)
// ensure that '+' is correctly encoded by url.PathUnescape as '+'
// (not whitespace as url.QueryUnescape generates)
require.Equal(t, "bob+alice:secret^[*()@tcp(host:123)/foo?multiStatements=true", s)
})
t.Run("socket", func(t *testing.T) {
// test with no user/pass
u, err := url.Parse("mysql:///foo?socket=/var/run/mysqld/mysqld.sock&flag=on")
require.NoError(t, err)
require.Equal(t, "", u.Host)
s := connectionString(u)
require.Equal(t, "unix(/var/run/mysqld/mysqld.sock)/foo?flag=on&multiStatements=true", s)
// test with user/pass
u, err = url.Parse("mysql://bob:secret@fakehost/foo?socket=/var/run/mysqld/mysqld.sock&flag=on")
require.NoError(t, err)
s = connectionString(u)
require.Equal(t, "bob:secret@unix(/var/run/mysqld/mysqld.sock)/foo?flag=on&multiStatements=true", s)
})
}
func TestMySQLCreateDropDatabase(t *testing.T) {
drv := testMySQLDriver(t)
// drop any existing database
err := drv.DropDatabase()
require.NoError(t, err)
// create database
err = drv.CreateDatabase()
require.NoError(t, err)
// check that database exists and we can connect to it
func() {
db, err := drv.Open()
require.NoError(t, err)
defer dbutil.MustClose(db)
err = db.Ping()
require.NoError(t, err)
}()
// drop the database
err = drv.DropDatabase()
require.NoError(t, err)
// check that database no longer exists
func() {
db, err := drv.Open()
require.NoError(t, err)
defer dbutil.MustClose(db)
err = db.Ping()
require.Error(t, err)
require.Regexp(t, "Unknown database 'dbmate_test'", err.Error())
}()
}
func TestMySQLDumpSchema(t *testing.T) {
drv := testMySQLDriver(t)
drv.migrationsTableName = "test_migrations"
// prepare database
db := prepTestMySQLDB(t)
defer dbutil.MustClose(db)
err := drv.CreateMigrationsTable(db)
require.NoError(t, err)
// insert migration
err = drv.InsertMigration(db, "abc1")
require.NoError(t, err)
err = drv.InsertMigration(db, "abc2")
require.NoError(t, err)
// DumpSchema should return schema
schema, err := drv.DumpSchema(db)
require.NoError(t, err)
require.Contains(t, string(schema), "CREATE TABLE `test_migrations`")
require.Contains(t, string(schema), "\n-- Dump completed\n\n"+
"--\n"+
"-- Dbmate schema migrations\n"+
"--\n\n"+
"LOCK TABLES `test_migrations` WRITE;\n"+
"INSERT INTO `test_migrations` (version) VALUES\n"+
" ('abc1'),\n"+
" ('abc2');\n"+
"UNLOCK TABLES;\n")
// DumpSchema should return error if command fails
drv.databaseURL.Path = "/fakedb"
schema, err = drv.DumpSchema(db)
require.Nil(t, schema)
require.Error(t, err)
require.Contains(t, err.Error(), "Unknown database 'fakedb'")
}
func TestMySQLDatabaseExists(t *testing.T) {
drv := testMySQLDriver(t)
// drop any existing database
err := drv.DropDatabase()
require.NoError(t, err)
// DatabaseExists should return false
exists, err := drv.DatabaseExists()
require.NoError(t, err)
require.Equal(t, false, exists)
// create database
err = drv.CreateDatabase()
require.NoError(t, err)
// DatabaseExists should return true
exists, err = drv.DatabaseExists()
require.NoError(t, err)
require.Equal(t, true, exists)
}
func TestMySQLDatabaseExists_Error(t *testing.T) {
drv := testMySQLDriver(t)
drv.databaseURL.User = url.User("invalid")
exists, err := drv.DatabaseExists()
require.Error(t, err)
require.Regexp(t, "Access denied for user 'invalid'@", err.Error())
require.Equal(t, false, exists)
}
func TestMySQLCreateMigrationsTable(t *testing.T) {
drv := testMySQLDriver(t)
drv.migrationsTableName = "test_migrations"
db := prepTestMySQLDB(t)
defer dbutil.MustClose(db)
// migrations table should not exist
count := 0
err := db.QueryRow("select count(*) from test_migrations").Scan(&count)
require.Error(t, err)
require.Regexp(t, "Table 'dbmate_test.test_migrations' doesn't exist", err.Error())
// create table
err = drv.CreateMigrationsTable(db)
require.NoError(t, err)
// migrations table should exist
err = db.QueryRow("select count(*) from test_migrations").Scan(&count)
require.NoError(t, err)
// create table should be idempotent
err = drv.CreateMigrationsTable(db)
require.NoError(t, err)
}
func TestMySQLSelectMigrations(t *testing.T) {
drv := testMySQLDriver(t)
drv.migrationsTableName = "test_migrations"
db := prepTestMySQLDB(t)
defer dbutil.MustClose(db)
err := drv.CreateMigrationsTable(db)
require.NoError(t, err)
_, err = db.Exec(`insert into test_migrations (version)
values ('abc2'), ('abc1'), ('abc3')`)
require.NoError(t, err)
migrations, err := drv.SelectMigrations(db, -1)
require.NoError(t, err)
require.Equal(t, true, migrations["abc1"])
require.Equal(t, true, migrations["abc2"])
require.Equal(t, true, migrations["abc2"])
// test limit param
migrations, err = drv.SelectMigrations(db, 1)
require.NoError(t, err)
require.Equal(t, true, migrations["abc3"])
require.Equal(t, false, migrations["abc1"])
require.Equal(t, false, migrations["abc2"])
}
func TestMySQLInsertMigration(t *testing.T) {
drv := testMySQLDriver(t)
drv.migrationsTableName = "test_migrations"
db := prepTestMySQLDB(t)
defer dbutil.MustClose(db)
err := drv.CreateMigrationsTable(db)
require.NoError(t, err)
count := 0
err = db.QueryRow("select count(*) from test_migrations").Scan(&count)
require.NoError(t, err)
require.Equal(t, 0, count)
// insert migration
err = drv.InsertMigration(db, "abc1")
require.NoError(t, err)
err = db.QueryRow("select count(*) from test_migrations where version = 'abc1'").
Scan(&count)
require.NoError(t, err)
require.Equal(t, 1, count)
}
func TestMySQLDeleteMigration(t *testing.T) {
drv := testMySQLDriver(t)
drv.migrationsTableName = "test_migrations"
db := prepTestMySQLDB(t)
defer dbutil.MustClose(db)
err := drv.CreateMigrationsTable(db)
require.NoError(t, err)
_, err = db.Exec(`insert into test_migrations (version)
values ('abc1'), ('abc2')`)
require.NoError(t, err)
err = drv.DeleteMigration(db, "abc2")
require.NoError(t, err)
count := 0
err = db.QueryRow("select count(*) from test_migrations").Scan(&count)
require.NoError(t, err)
require.Equal(t, 1, count)
}
func TestMySQLPing(t *testing.T) {
drv := testMySQLDriver(t)
// drop any existing database
err := drv.DropDatabase()
require.NoError(t, err)
// ping database
err = drv.Ping()
require.NoError(t, err)
// ping invalid host should return error
drv.databaseURL.Host = "mysql:404"
err = drv.Ping()
require.Error(t, err)
require.Contains(t, err.Error(), "connect: connection refused")
}
func TestMySQLQuotedMigrationsTableName(t *testing.T) {
t.Run("default name", func(t *testing.T) {
drv := testMySQLDriver(t)
name := drv.quotedMigrationsTableName()
require.Equal(t, "`schema_migrations`", name)
})
t.Run("custom name", func(t *testing.T) {
drv := testMySQLDriver(t)
drv.migrationsTableName = "fooMigrations"
name := drv.quotedMigrationsTableName()
require.Equal(t, "`fooMigrations`", name)
})
}

View file

@ -1,393 +0,0 @@
package postgres
import (
"bytes"
"database/sql"
"fmt"
"io"
"net/url"
"runtime"
"strings"
"github.com/amacneil/dbmate/pkg/dbmate"
"github.com/amacneil/dbmate/pkg/dbutil"
"github.com/lib/pq"
)
func init() {
dbmate.RegisterDriver(NewDriver, "postgres")
dbmate.RegisterDriver(NewDriver, "postgresql")
}
// Driver provides top level database functions
type Driver struct {
migrationsTableName string
databaseURL *url.URL
log io.Writer
}
// NewDriver initializes the driver
func NewDriver(config dbmate.DriverConfig) dbmate.Driver {
return &Driver{
migrationsTableName: config.MigrationsTableName,
databaseURL: config.DatabaseURL,
log: config.Log,
}
}
func connectionString(u *url.URL) string {
hostname := u.Hostname()
port := u.Port()
query := u.Query()
// support socket parameter for consistency with mysql
if query.Get("socket") != "" {
query.Set("host", query.Get("socket"))
query.Del("socket")
}
// default hostname
if hostname == "" {
switch runtime.GOOS {
case "linux":
query.Set("host", "/var/run/postgresql")
case "darwin", "freebsd", "dragonfly", "openbsd", "netbsd":
query.Set("host", "/tmp")
default:
hostname = "localhost"
}
}
// host param overrides url hostname
if query.Get("host") != "" {
hostname = ""
}
// always specify a port
if query.Get("port") != "" {
port = query.Get("port")
query.Del("port")
}
if port == "" {
port = "5432"
}
// generate output URL
out, _ := url.Parse(u.String())
out.Host = fmt.Sprintf("%s:%s", hostname, port)
out.RawQuery = query.Encode()
return out.String()
}
func connectionArgsForDump(u *url.URL) []string {
u = dbutil.MustParseURL(connectionString(u))
// find schemas from search_path
query := u.Query()
schemas := strings.Split(query.Get("search_path"), ",")
query.Del("search_path")
u.RawQuery = query.Encode()
out := []string{}
for _, schema := range schemas {
schema = strings.TrimSpace(schema)
if schema != "" {
out = append(out, "--schema", schema)
}
}
out = append(out, u.String())
return out
}
// Open creates a new database connection
func (drv *Driver) Open() (*sql.DB, error) {
return sql.Open("postgres", connectionString(drv.databaseURL))
}
func (drv *Driver) openPostgresDB() (*sql.DB, error) {
// clone databaseURL
postgresURL, err := url.Parse(connectionString(drv.databaseURL))
if err != nil {
return nil, err
}
// connect to postgres database
postgresURL.Path = "postgres"
return sql.Open("postgres", postgresURL.String())
}
// CreateDatabase creates the specified database
func (drv *Driver) CreateDatabase() error {
name := dbutil.DatabaseName(drv.databaseURL)
fmt.Fprintf(drv.log, "Creating: %s\n", name)
db, err := drv.openPostgresDB()
if err != nil {
return err
}
defer dbutil.MustClose(db)
_, err = db.Exec(fmt.Sprintf("create database %s",
pq.QuoteIdentifier(name)))
return err
}
// DropDatabase drops the specified database (if it exists)
func (drv *Driver) DropDatabase() error {
name := dbutil.DatabaseName(drv.databaseURL)
fmt.Fprintf(drv.log, "Dropping: %s\n", name)
db, err := drv.openPostgresDB()
if err != nil {
return err
}
defer dbutil.MustClose(db)
_, err = db.Exec(fmt.Sprintf("drop database if exists %s",
pq.QuoteIdentifier(name)))
return err
}
func (drv *Driver) schemaMigrationsDump(db *sql.DB) ([]byte, error) {
migrationsTable, err := drv.quotedMigrationsTableName(db)
if err != nil {
return nil, err
}
// load applied migrations
migrations, err := dbutil.QueryColumn(db,
"select quote_literal(version) from "+migrationsTable+" order by version asc")
if err != nil {
return nil, err
}
// build migrations table data
var buf bytes.Buffer
buf.WriteString("\n--\n-- Dbmate schema migrations\n--\n\n")
if len(migrations) > 0 {
buf.WriteString("INSERT INTO " + migrationsTable + " (version) VALUES\n (" +
strings.Join(migrations, "),\n (") +
");\n")
}
return buf.Bytes(), nil
}
// DumpSchema returns the current database schema
func (drv *Driver) DumpSchema(db *sql.DB) ([]byte, error) {
// load schema
args := append([]string{"--format=plain", "--encoding=UTF8", "--schema-only",
"--no-privileges", "--no-owner"}, connectionArgsForDump(drv.databaseURL)...)
schema, err := dbutil.RunCommand("pg_dump", args...)
if err != nil {
return nil, err
}
migrations, err := drv.schemaMigrationsDump(db)
if err != nil {
return nil, err
}
schema = append(schema, migrations...)
return dbutil.TrimLeadingSQLComments(schema)
}
// DatabaseExists determines whether the database exists
func (drv *Driver) DatabaseExists() (bool, error) {
name := dbutil.DatabaseName(drv.databaseURL)
db, err := drv.openPostgresDB()
if err != nil {
return false, err
}
defer dbutil.MustClose(db)
exists := false
err = db.QueryRow("select true from pg_database where datname = $1", name).
Scan(&exists)
if err == sql.ErrNoRows {
return false, nil
}
return exists, err
}
// CreateMigrationsTable creates the schema_migrations table
func (drv *Driver) CreateMigrationsTable(db *sql.DB) error {
schema, migrationsTable, err := drv.quotedMigrationsTableNameParts(db)
if err != nil {
return err
}
// first attempt at creating migrations table
createTableStmt := fmt.Sprintf("create table if not exists %s.%s", schema, migrationsTable) +
" (version varchar(255) primary key)"
_, err = db.Exec(createTableStmt)
if err == nil {
// table exists or created successfully
return nil
}
// catch 'schema does not exist' error
pqErr, ok := err.(*pq.Error)
if !ok || pqErr.Code != "3F000" {
// unknown error
return err
}
// in theory we could attempt to create the schema every time, but we avoid that
// in case the user doesn't have permissions to create schemas
fmt.Fprintf(drv.log, "Creating schema: %s\n", schema)
_, err = db.Exec(fmt.Sprintf("create schema if not exists %s", schema))
if err != nil {
return err
}
// second and final attempt at creating migrations table
_, err = db.Exec(createTableStmt)
return err
}
// SelectMigrations returns a list of applied migrations
// with an optional limit (in descending order)
func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, error) {
migrationsTable, err := drv.quotedMigrationsTableName(db)
if err != nil {
return nil, err
}
query := "select version from " + migrationsTable + " order by version desc"
if limit >= 0 {
query = fmt.Sprintf("%s limit %d", query, limit)
}
rows, err := db.Query(query)
if err != nil {
return nil, err
}
defer dbutil.MustClose(rows)
migrations := map[string]bool{}
for rows.Next() {
var version string
if err := rows.Scan(&version); err != nil {
return nil, err
}
migrations[version] = true
}
if err = rows.Err(); err != nil {
return nil, err
}
return migrations, nil
}
// InsertMigration adds a new migration record
func (drv *Driver) InsertMigration(db dbutil.Transaction, version string) error {
migrationsTable, err := drv.quotedMigrationsTableName(db)
if err != nil {
return err
}
_, err = db.Exec("insert into "+migrationsTable+" (version) values ($1)", version)
return err
}
// DeleteMigration removes a migration record
func (drv *Driver) DeleteMigration(db dbutil.Transaction, version string) error {
migrationsTable, err := drv.quotedMigrationsTableName(db)
if err != nil {
return err
}
_, err = db.Exec("delete from "+migrationsTable+" where version = $1", version)
return err
}
// Ping verifies a connection to the database server. It does not verify whether the
// specified database exists.
func (drv *Driver) Ping() error {
// attempt connection to primary database, not "postgres" database
// to support servers with no "postgres" database
// (see https://github.com/amacneil/dbmate/issues/78)
db, err := drv.Open()
if err != nil {
return err
}
defer dbutil.MustClose(db)
err = db.Ping()
if err == nil {
return nil
}
// ignore 'database does not exist' error
pqErr, ok := err.(*pq.Error)
if ok && pqErr.Code == "3D000" {
return nil
}
return err
}
func (drv *Driver) quotedMigrationsTableName(db dbutil.Transaction) (string, error) {
schema, name, err := drv.quotedMigrationsTableNameParts(db)
if err != nil {
return "", err
}
return schema + "." + name, nil
}
func (drv *Driver) quotedMigrationsTableNameParts(db dbutil.Transaction) (string, string, error) {
schema := ""
tableNameParts := strings.Split(drv.migrationsTableName, ".")
if len(tableNameParts) > 1 {
// schema specified as part of table name
schema, tableNameParts = tableNameParts[0], tableNameParts[1:]
}
if schema == "" {
// no schema specified with table name, try URL search path if available
searchPath := strings.Split(drv.databaseURL.Query().Get("search_path"), ",")
schema = strings.TrimSpace(searchPath[0])
}
var err error
if schema == "" {
// if no URL available, use current schema
// this is a hack because we don't always have the URL context available
schema, err = dbutil.QueryValue(db, "select current_schema()")
if err != nil {
return "", "", err
}
}
// fall back to public schema as last resort
if schema == "" {
schema = "public"
}
// quote all parts
// use server rather than client to do this to avoid unnecessary quotes
// (which would change schema.sql diff)
tableNameParts = append([]string{schema}, tableNameParts...)
quotedNameParts, err := dbutil.QueryColumn(db, "select quote_ident(unnest($1::text[]))", pq.Array(tableNameParts))
if err != nil {
return "", "", err
}
// if more than one part, we already have a schema
return quotedNameParts[0], strings.Join(quotedNameParts[1:], "."), nil
}

View file

@ -1,581 +0,0 @@
package postgres
import (
"database/sql"
"net/url"
"os"
"runtime"
"testing"
"github.com/amacneil/dbmate/pkg/dbmate"
"github.com/amacneil/dbmate/pkg/dbutil"
"github.com/stretchr/testify/require"
)
func testPostgresDriver(t *testing.T) *Driver {
u := dbutil.MustParseURL(os.Getenv("POSTGRES_TEST_URL"))
drv, err := dbmate.New(u).GetDriver()
require.NoError(t, err)
return drv.(*Driver)
}
func prepTestPostgresDB(t *testing.T) *sql.DB {
drv := testPostgresDriver(t)
// drop any existing database
err := drv.DropDatabase()
require.NoError(t, err)
// create database
err = drv.CreateDatabase()
require.NoError(t, err)
// connect database
db, err := sql.Open("postgres", drv.databaseURL.String())
require.NoError(t, err)
return db
}
func TestGetDriver(t *testing.T) {
db := dbmate.New(dbutil.MustParseURL("postgres://"))
drvInterface, err := db.GetDriver()
require.NoError(t, err)
// driver should have URL and default migrations table set
drv, ok := drvInterface.(*Driver)
require.True(t, ok)
require.Equal(t, db.DatabaseURL.String(), drv.databaseURL.String())
require.Equal(t, "schema_migrations", drv.migrationsTableName)
}
func defaultConnString() string {
switch runtime.GOOS {
case "linux":
return "postgres://:5432/foo?host=%2Fvar%2Frun%2Fpostgresql"
case "darwin", "freebsd", "dragonfly", "openbsd", "netbsd":
return "postgres://:5432/foo?host=%2Ftmp"
default:
return "postgres://localhost:5432/foo"
}
}
func TestConnectionString(t *testing.T) {
cases := []struct {
input string
expected string
}{
// defaults
{"postgres:///foo", defaultConnString()},
// support custom url params
{"postgres://bob:secret@myhost:1234/foo?bar=baz", "postgres://bob:secret@myhost:1234/foo?bar=baz"},
// support `host` and `port` via url params
{"postgres://bob:secret@myhost:1234/foo?host=new&port=9999", "postgres://bob:secret@:9999/foo?host=new"},
{"postgres://bob:secret@myhost:1234/foo?port=9999&bar=baz", "postgres://bob:secret@myhost:9999/foo?bar=baz"},
// support unix sockets via `host` or `socket` param
{"postgres://bob:secret@myhost:1234/foo?host=/var/run/postgresql", "postgres://bob:secret@:1234/foo?host=%2Fvar%2Frun%2Fpostgresql"},
{"postgres://bob:secret@localhost/foo?socket=/var/run/postgresql", "postgres://bob:secret@:5432/foo?host=%2Fvar%2Frun%2Fpostgresql"},
{"postgres:///foo?socket=/var/run/postgresql", "postgres://:5432/foo?host=%2Fvar%2Frun%2Fpostgresql"},
}
for _, c := range cases {
t.Run(c.input, func(t *testing.T) {
u, err := url.Parse(c.input)
require.NoError(t, err)
actual := connectionString(u)
require.Equal(t, c.expected, actual)
})
}
}
func TestConnectionArgsForDump(t *testing.T) {
cases := []struct {
input string
expected []string
}{
// defaults
{"postgres:///foo", []string{defaultConnString()}},
// support single schema
{"postgres:///foo?search_path=foo", []string{"--schema", "foo", defaultConnString()}},
// support multiple schemas
{"postgres:///foo?search_path=foo,public", []string{"--schema", "foo", "--schema", "public", defaultConnString()}},
}
for _, c := range cases {
t.Run(c.input, func(t *testing.T) {
u, err := url.Parse(c.input)
require.NoError(t, err)
actual := connectionArgsForDump(u)
require.Equal(t, c.expected, actual)
})
}
}
func TestPostgresCreateDropDatabase(t *testing.T) {
drv := testPostgresDriver(t)
// drop any existing database
err := drv.DropDatabase()
require.NoError(t, err)
// create database
err = drv.CreateDatabase()
require.NoError(t, err)
// check that database exists and we can connect to it
func() {
db, err := sql.Open("postgres", drv.databaseURL.String())
require.NoError(t, err)
defer dbutil.MustClose(db)
err = db.Ping()
require.NoError(t, err)
}()
// drop the database
err = drv.DropDatabase()
require.NoError(t, err)
// check that database no longer exists
func() {
db, err := sql.Open("postgres", drv.databaseURL.String())
require.NoError(t, err)
defer dbutil.MustClose(db)
err = db.Ping()
require.Error(t, err)
require.Equal(t, "pq: database \"dbmate_test\" does not exist", err.Error())
}()
}
func TestPostgresDumpSchema(t *testing.T) {
t.Run("default migrations table", func(t *testing.T) {
drv := testPostgresDriver(t)
// prepare database
db := prepTestPostgresDB(t)
defer dbutil.MustClose(db)
err := drv.CreateMigrationsTable(db)
require.NoError(t, err)
// insert migration
err = drv.InsertMigration(db, "abc1")
require.NoError(t, err)
err = drv.InsertMigration(db, "abc2")
require.NoError(t, err)
// DumpSchema should return schema
schema, err := drv.DumpSchema(db)
require.NoError(t, err)
require.Contains(t, string(schema), "CREATE TABLE public.schema_migrations")
require.Contains(t, string(schema), "\n--\n"+
"-- PostgreSQL database dump complete\n"+
"--\n\n\n"+
"--\n"+
"-- Dbmate schema migrations\n"+
"--\n\n"+
"INSERT INTO public.schema_migrations (version) VALUES\n"+
" ('abc1'),\n"+
" ('abc2');\n")
// DumpSchema should return error if command fails
drv.databaseURL.Path = "/fakedb"
schema, err = drv.DumpSchema(db)
require.Nil(t, schema)
require.Error(t, err)
require.Contains(t, err.Error(), "database \"fakedb\" does not exist")
})
t.Run("custom migrations table with schema", func(t *testing.T) {
drv := testPostgresDriver(t)
drv.migrationsTableName = "camelSchema.testMigrations"
// prepare database
db := prepTestPostgresDB(t)
defer dbutil.MustClose(db)
err := drv.CreateMigrationsTable(db)
require.NoError(t, err)
// insert migration
err = drv.InsertMigration(db, "abc1")
require.NoError(t, err)
err = drv.InsertMigration(db, "abc2")
require.NoError(t, err)
// DumpSchema should return schema
schema, err := drv.DumpSchema(db)
require.NoError(t, err)
require.Contains(t, string(schema), "CREATE TABLE \"camelSchema\".\"testMigrations\"")
require.Contains(t, string(schema), "\n--\n"+
"-- PostgreSQL database dump complete\n"+
"--\n\n\n"+
"--\n"+
"-- Dbmate schema migrations\n"+
"--\n\n"+
"INSERT INTO \"camelSchema\".\"testMigrations\" (version) VALUES\n"+
" ('abc1'),\n"+
" ('abc2');\n")
})
}
func TestPostgresDatabaseExists(t *testing.T) {
drv := testPostgresDriver(t)
// drop any existing database
err := drv.DropDatabase()
require.NoError(t, err)
// DatabaseExists should return false
exists, err := drv.DatabaseExists()
require.NoError(t, err)
require.Equal(t, false, exists)
// create database
err = drv.CreateDatabase()
require.NoError(t, err)
// DatabaseExists should return true
exists, err = drv.DatabaseExists()
require.NoError(t, err)
require.Equal(t, true, exists)
}
func TestPostgresDatabaseExists_Error(t *testing.T) {
drv := testPostgresDriver(t)
drv.databaseURL.User = url.User("invalid")
exists, err := drv.DatabaseExists()
require.Error(t, err)
require.Equal(t, "pq: password authentication failed for user \"invalid\"", err.Error())
require.Equal(t, false, exists)
}
func TestPostgresCreateMigrationsTable(t *testing.T) {
t.Run("default schema", func(t *testing.T) {
drv := testPostgresDriver(t)
db := prepTestPostgresDB(t)
defer dbutil.MustClose(db)
// migrations table should not exist
count := 0
err := db.QueryRow("select count(*) from public.schema_migrations").Scan(&count)
require.Error(t, err)
require.Equal(t, "pq: relation \"public.schema_migrations\" does not exist", err.Error())
// create table
err = drv.CreateMigrationsTable(db)
require.NoError(t, err)
// migrations table should exist
err = db.QueryRow("select count(*) from public.schema_migrations").Scan(&count)
require.NoError(t, err)
// create table should be idempotent
err = drv.CreateMigrationsTable(db)
require.NoError(t, err)
})
t.Run("custom search path", func(t *testing.T) {
drv := testPostgresDriver(t)
drv.migrationsTableName = "testMigrations"
u, err := url.Parse(drv.databaseURL.String() + "&search_path=camelFoo")
require.NoError(t, err)
drv.databaseURL = u
db := prepTestPostgresDB(t)
defer dbutil.MustClose(db)
// delete schema
_, err = db.Exec("drop schema if exists \"camelFoo\"")
require.NoError(t, err)
// drop any testMigrations table in public schema
_, err = db.Exec("drop table if exists public.\"testMigrations\"")
require.NoError(t, err)
// migrations table should not exist in either schema
count := 0
err = db.QueryRow("select count(*) from \"camelFoo\".\"testMigrations\"").Scan(&count)
require.Error(t, err)
require.Equal(t, "pq: relation \"camelFoo.testMigrations\" does not exist", err.Error())
err = db.QueryRow("select count(*) from public.\"testMigrations\"").Scan(&count)
require.Error(t, err)
require.Equal(t, "pq: relation \"public.testMigrations\" does not exist", err.Error())
// create table
err = drv.CreateMigrationsTable(db)
require.NoError(t, err)
// camelFoo schema should be created, and migrations table should exist only in camelFoo schema
err = db.QueryRow("select count(*) from \"camelFoo\".\"testMigrations\"").Scan(&count)
require.NoError(t, err)
err = db.QueryRow("select count(*) from public.\"testMigrations\"").Scan(&count)
require.Error(t, err)
require.Equal(t, "pq: relation \"public.testMigrations\" does not exist", err.Error())
// create table should be idempotent
err = drv.CreateMigrationsTable(db)
require.NoError(t, err)
})
t.Run("custom schema", func(t *testing.T) {
drv := testPostgresDriver(t)
drv.migrationsTableName = "camelSchema.testMigrations"
u, err := url.Parse(drv.databaseURL.String() + "&search_path=foo")
require.NoError(t, err)
drv.databaseURL = u
db := prepTestPostgresDB(t)
defer dbutil.MustClose(db)
// delete schemas
_, err = db.Exec("drop schema if exists foo")
require.NoError(t, err)
_, err = db.Exec("drop schema if exists \"camelSchema\"")
require.NoError(t, err)
// migrations table should not exist
count := 0
err = db.QueryRow("select count(*) from \"camelSchema\".\"testMigrations\"").Scan(&count)
require.Error(t, err)
require.Equal(t, "pq: relation \"camelSchema.testMigrations\" does not exist", err.Error())
// create table
err = drv.CreateMigrationsTable(db)
require.NoError(t, err)
// camelSchema should be created, and testMigrations table should exist
err = db.QueryRow("select count(*) from \"camelSchema\".\"testMigrations\"").Scan(&count)
require.NoError(t, err)
// testMigrations table should not exist in foo schema because
// schema specified with migrations table name takes priority over search path
err = db.QueryRow("select count(*) from foo.\"testMigrations\"").Scan(&count)
require.Error(t, err)
require.Equal(t, "pq: relation \"foo.testMigrations\" does not exist", err.Error())
// create table should be idempotent
err = drv.CreateMigrationsTable(db)
require.NoError(t, err)
})
}
func TestPostgresSelectMigrations(t *testing.T) {
drv := testPostgresDriver(t)
drv.migrationsTableName = "test_migrations"
db := prepTestPostgresDB(t)
defer dbutil.MustClose(db)
err := drv.CreateMigrationsTable(db)
require.NoError(t, err)
_, err = db.Exec(`insert into public.test_migrations (version)
values ('abc2'), ('abc1'), ('abc3')`)
require.NoError(t, err)
migrations, err := drv.SelectMigrations(db, -1)
require.NoError(t, err)
require.Equal(t, true, migrations["abc1"])
require.Equal(t, true, migrations["abc2"])
require.Equal(t, true, migrations["abc2"])
// test limit param
migrations, err = drv.SelectMigrations(db, 1)
require.NoError(t, err)
require.Equal(t, true, migrations["abc3"])
require.Equal(t, false, migrations["abc1"])
require.Equal(t, false, migrations["abc2"])
}
func TestPostgresInsertMigration(t *testing.T) {
drv := testPostgresDriver(t)
drv.migrationsTableName = "test_migrations"
db := prepTestPostgresDB(t)
defer dbutil.MustClose(db)
err := drv.CreateMigrationsTable(db)
require.NoError(t, err)
count := 0
err = db.QueryRow("select count(*) from public.test_migrations").Scan(&count)
require.NoError(t, err)
require.Equal(t, 0, count)
// insert migration
err = drv.InsertMigration(db, "abc1")
require.NoError(t, err)
err = db.QueryRow("select count(*) from public.test_migrations where version = 'abc1'").
Scan(&count)
require.NoError(t, err)
require.Equal(t, 1, count)
}
func TestPostgresDeleteMigration(t *testing.T) {
drv := testPostgresDriver(t)
drv.migrationsTableName = "test_migrations"
db := prepTestPostgresDB(t)
defer dbutil.MustClose(db)
err := drv.CreateMigrationsTable(db)
require.NoError(t, err)
_, err = db.Exec(`insert into public.test_migrations (version)
values ('abc1'), ('abc2')`)
require.NoError(t, err)
err = drv.DeleteMigration(db, "abc2")
require.NoError(t, err)
count := 0
err = db.QueryRow("select count(*) from public.test_migrations").Scan(&count)
require.NoError(t, err)
require.Equal(t, 1, count)
}
func TestPostgresPing(t *testing.T) {
drv := testPostgresDriver(t)
// drop any existing database
err := drv.DropDatabase()
require.NoError(t, err)
// ping database
err = drv.Ping()
require.NoError(t, err)
// ping invalid host should return error
drv.databaseURL.Host = "postgres:404"
err = drv.Ping()
require.Error(t, err)
require.Contains(t, err.Error(), "connect: connection refused")
}
func TestPostgresQuotedMigrationsTableName(t *testing.T) {
t.Run("default schema", func(t *testing.T) {
drv := testPostgresDriver(t)
db := prepTestPostgresDB(t)
defer dbutil.MustClose(db)
name, err := drv.quotedMigrationsTableName(db)
require.NoError(t, err)
require.Equal(t, "public.schema_migrations", name)
})
t.Run("custom schema", func(t *testing.T) {
drv := testPostgresDriver(t)
u, err := url.Parse(drv.databaseURL.String() + "&search_path=foo,bar,public")
require.NoError(t, err)
drv.databaseURL = u
db := prepTestPostgresDB(t)
defer dbutil.MustClose(db)
_, err = db.Exec("drop schema if exists foo")
require.NoError(t, err)
_, err = db.Exec("drop schema if exists bar")
require.NoError(t, err)
// should use first schema from search path
name, err := drv.quotedMigrationsTableName(db)
require.NoError(t, err)
require.Equal(t, "foo.schema_migrations", name)
})
t.Run("no schema", func(t *testing.T) {
drv := testPostgresDriver(t)
db := prepTestPostgresDB(t)
defer dbutil.MustClose(db)
// this is an unlikely edge case, but if for some reason there is
// no current schema then we should default to "public"
_, err := db.Exec("select pg_catalog.set_config('search_path', '', false)")
require.NoError(t, err)
name, err := drv.quotedMigrationsTableName(db)
require.NoError(t, err)
require.Equal(t, "public.schema_migrations", name)
})
t.Run("custom table name", func(t *testing.T) {
drv := testPostgresDriver(t)
db := prepTestPostgresDB(t)
defer dbutil.MustClose(db)
drv.migrationsTableName = "simple_name"
name, err := drv.quotedMigrationsTableName(db)
require.NoError(t, err)
require.Equal(t, "public.simple_name", name)
})
t.Run("custom table name quoted", func(t *testing.T) {
drv := testPostgresDriver(t)
db := prepTestPostgresDB(t)
defer dbutil.MustClose(db)
// this table name will need quoting
drv.migrationsTableName = "camelCase"
name, err := drv.quotedMigrationsTableName(db)
require.NoError(t, err)
require.Equal(t, "public.\"camelCase\"", name)
})
t.Run("custom table name with custom schema", func(t *testing.T) {
drv := testPostgresDriver(t)
u, err := url.Parse(drv.databaseURL.String() + "&search_path=foo")
require.NoError(t, err)
drv.databaseURL = u
db := prepTestPostgresDB(t)
defer dbutil.MustClose(db)
_, err = db.Exec("create schema if not exists foo")
require.NoError(t, err)
drv.migrationsTableName = "simple_name"
name, err := drv.quotedMigrationsTableName(db)
require.NoError(t, err)
require.Equal(t, "foo.simple_name", name)
})
t.Run("custom table name overrides schema", func(t *testing.T) {
drv := testPostgresDriver(t)
u, err := url.Parse(drv.databaseURL.String() + "&search_path=foo")
require.NoError(t, err)
drv.databaseURL = u
db := prepTestPostgresDB(t)
defer dbutil.MustClose(db)
_, err = db.Exec("create schema if not exists foo")
require.NoError(t, err)
_, err = db.Exec("create schema if not exists bar")
require.NoError(t, err)
// if schema is specified as part of table name, it should override search_path
drv.migrationsTableName = "bar.simple_name"
name, err := drv.quotedMigrationsTableName(db)
require.NoError(t, err)
require.Equal(t, "bar.simple_name", name)
// schema and table name should be quoted if necessary
drv.migrationsTableName = "barName.camelTable"
name, err = drv.quotedMigrationsTableName(db)
require.NoError(t, err)
require.Equal(t, "\"barName\".\"camelTable\"", name)
// more than 2 components is unexpected but we will quote and pass it along anyway
drv.migrationsTableName = "whyWould.i.doThis"
name, err = drv.quotedMigrationsTableName(db)
require.NoError(t, err)
require.Equal(t, "\"whyWould\".i.\"doThis\"", name)
})
}

View file

@ -1,224 +0,0 @@
//go:build cgo
// +build cgo
package sqlite
import (
"bytes"
"database/sql"
"fmt"
"io"
"net/url"
"os"
"regexp"
"strings"
"github.com/amacneil/dbmate/pkg/dbmate"
"github.com/amacneil/dbmate/pkg/dbutil"
"github.com/lib/pq"
_ "github.com/mattn/go-sqlite3" // database/sql driver
)
func init() {
dbmate.RegisterDriver(NewDriver, "sqlite")
dbmate.RegisterDriver(NewDriver, "sqlite3")
}
// Driver provides top level database functions
type Driver struct {
migrationsTableName string
databaseURL *url.URL
log io.Writer
}
// NewDriver initializes the driver
func NewDriver(config dbmate.DriverConfig) dbmate.Driver {
return &Driver{
migrationsTableName: config.MigrationsTableName,
databaseURL: config.DatabaseURL,
log: config.Log,
}
}
// ConnectionString converts a URL into a valid connection string
func ConnectionString(u *url.URL) string {
// duplicate URL and remove scheme
newURL := *u
newURL.Scheme = ""
// trim duplicate leading slashes
str := regexp.MustCompile("^//+").ReplaceAllString(newURL.String(), "/")
return str
}
// Open creates a new database connection
func (drv *Driver) Open() (*sql.DB, error) {
return sql.Open("sqlite3", ConnectionString(drv.databaseURL))
}
// CreateDatabase creates the specified database
func (drv *Driver) CreateDatabase() error {
fmt.Fprintf(drv.log, "Creating: %s\n", ConnectionString(drv.databaseURL))
db, err := drv.Open()
if err != nil {
return err
}
defer dbutil.MustClose(db)
return db.Ping()
}
// DropDatabase drops the specified database (if it exists)
func (drv *Driver) DropDatabase() error {
path := ConnectionString(drv.databaseURL)
fmt.Fprintf(drv.log, "Dropping: %s\n", path)
exists, err := drv.DatabaseExists()
if err != nil {
return err
}
if !exists {
return nil
}
return os.Remove(path)
}
func (drv *Driver) schemaMigrationsDump(db *sql.DB) ([]byte, error) {
migrationsTable := drv.quotedMigrationsTableName()
// load applied migrations
migrations, err := dbutil.QueryColumn(db,
fmt.Sprintf("select quote(version) from %s order by version asc", migrationsTable))
if err != nil {
return nil, err
}
// build schema migrations table data
var buf bytes.Buffer
buf.WriteString("-- Dbmate schema migrations\n")
if len(migrations) > 0 {
buf.WriteString(
fmt.Sprintf("INSERT INTO %s (version) VALUES\n (", migrationsTable) +
strings.Join(migrations, "),\n (") +
");\n")
}
return buf.Bytes(), nil
}
// DumpSchema returns the current database schema
func (drv *Driver) DumpSchema(db *sql.DB) ([]byte, error) {
path := ConnectionString(drv.databaseURL)
schema, err := dbutil.RunCommand("sqlite3", path, ".schema")
if err != nil {
return nil, err
}
migrations, err := drv.schemaMigrationsDump(db)
if err != nil {
return nil, err
}
schema = append(schema, migrations...)
return dbutil.TrimLeadingSQLComments(schema)
}
// DatabaseExists determines whether the database exists
func (drv *Driver) DatabaseExists() (bool, error) {
_, err := os.Stat(ConnectionString(drv.databaseURL))
if os.IsNotExist(err) {
return false, nil
}
if err != nil {
return false, err
}
return true, nil
}
// CreateMigrationsTable creates the schema migrations table
func (drv *Driver) CreateMigrationsTable(db *sql.DB) error {
_, err := db.Exec(
fmt.Sprintf("create table if not exists %s ", drv.quotedMigrationsTableName()) +
"(version varchar(255) primary key)")
return err
}
// SelectMigrations returns a list of applied migrations
// with an optional limit (in descending order)
func (drv *Driver) SelectMigrations(db *sql.DB, limit int) (map[string]bool, error) {
query := fmt.Sprintf("select version from %s order by version desc", drv.quotedMigrationsTableName())
if limit >= 0 {
query = fmt.Sprintf("%s limit %d", query, limit)
}
rows, err := db.Query(query)
if err != nil {
return nil, err
}
defer dbutil.MustClose(rows)
migrations := map[string]bool{}
for rows.Next() {
var version string
if err := rows.Scan(&version); err != nil {
return nil, err
}
migrations[version] = true
}
if err = rows.Err(); err != nil {
return nil, err
}
return migrations, nil
}
// InsertMigration adds a new migration record
func (drv *Driver) InsertMigration(db dbutil.Transaction, version string) error {
_, err := db.Exec(
fmt.Sprintf("insert into %s (version) values (?)", drv.quotedMigrationsTableName()),
version)
return err
}
// DeleteMigration removes a migration record
func (drv *Driver) DeleteMigration(db dbutil.Transaction, version string) error {
_, err := db.Exec(
fmt.Sprintf("delete from %s where version = ?", drv.quotedMigrationsTableName()),
version)
return err
}
// Ping verifies a connection to the database. Due to the way SQLite works, by
// testing whether the database is valid, it will automatically create the database
// if it does not already exist.
func (drv *Driver) Ping() error {
db, err := drv.Open()
if err != nil {
return err
}
defer dbutil.MustClose(db)
return db.Ping()
}
func (drv *Driver) quotedMigrationsTableName() string {
return drv.quoteIdentifier(drv.migrationsTableName)
}
// quoteIdentifier quotes a table or column name
// we fall back to lib/pq implementation since both use ansi standard (double quotes)
// and mattn/go-sqlite3 doesn't provide a sqlite-specific equivalent
func (drv *Driver) quoteIdentifier(s string) string {
return pq.QuoteIdentifier(s)
}

View file

@ -1,337 +0,0 @@
//go:build cgo
// +build cgo
package sqlite
import (
"database/sql"
"os"
"testing"
"github.com/amacneil/dbmate/pkg/dbmate"
"github.com/amacneil/dbmate/pkg/dbutil"
"github.com/stretchr/testify/require"
)
func testSQLiteDriver(t *testing.T) *Driver {
u := dbutil.MustParseURL(os.Getenv("SQLITE_TEST_URL"))
drv, err := dbmate.New(u).GetDriver()
require.NoError(t, err)
return drv.(*Driver)
}
func prepTestSQLiteDB(t *testing.T) *sql.DB {
drv := testSQLiteDriver(t)
// drop any existing database
err := drv.DropDatabase()
require.NoError(t, err)
// create database
err = drv.CreateDatabase()
require.NoError(t, err)
// connect database
db, err := drv.Open()
require.NoError(t, err)
return db
}
func TestGetDriver(t *testing.T) {
db := dbmate.New(dbutil.MustParseURL("sqlite://"))
drvInterface, err := db.GetDriver()
require.NoError(t, err)
// driver should have URL and default migrations table set
drv, ok := drvInterface.(*Driver)
require.True(t, ok)
require.Equal(t, db.DatabaseURL.String(), drv.databaseURL.String())
require.Equal(t, "schema_migrations", drv.migrationsTableName)
}
func TestConnectionString(t *testing.T) {
t.Run("relative", func(t *testing.T) {
u := dbutil.MustParseURL("sqlite:foo/bar.sqlite3?mode=ro")
require.Equal(t, "foo/bar.sqlite3?mode=ro", ConnectionString(u))
})
t.Run("absolute", func(t *testing.T) {
u := dbutil.MustParseURL("sqlite:/tmp/foo.sqlite3?mode=ro")
require.Equal(t, "/tmp/foo.sqlite3?mode=ro", ConnectionString(u))
})
t.Run("three slashes", func(t *testing.T) {
// interpreted as absolute path
u := dbutil.MustParseURL("sqlite:///tmp/foo.sqlite3?mode=ro")
require.Equal(t, "/tmp/foo.sqlite3?mode=ro", ConnectionString(u))
})
t.Run("four slashes", func(t *testing.T) {
// interpreted as absolute path
// supported for backwards compatibility
u := dbutil.MustParseURL("sqlite:////tmp/foo.sqlite3?mode=ro")
require.Equal(t, "/tmp/foo.sqlite3?mode=ro", ConnectionString(u))
})
}
func TestSQLiteCreateDropDatabase(t *testing.T) {
drv := testSQLiteDriver(t)
path := ConnectionString(drv.databaseURL)
// drop any existing database
err := drv.DropDatabase()
require.NoError(t, err)
// create database
err = drv.CreateDatabase()
require.NoError(t, err)
// check that database exists
_, err = os.Stat(path)
require.NoError(t, err)
// drop the database
err = drv.DropDatabase()
require.NoError(t, err)
// check that database no longer exists
_, err = os.Stat(path)
require.NotNil(t, err)
require.Equal(t, true, os.IsNotExist(err))
}
func TestSQLiteDumpSchema(t *testing.T) {
drv := testSQLiteDriver(t)
drv.migrationsTableName = "test_migrations"
// prepare database
db := prepTestSQLiteDB(t)
defer dbutil.MustClose(db)
err := drv.CreateMigrationsTable(db)
require.NoError(t, err)
// insert migration
err = drv.InsertMigration(db, "abc1")
require.NoError(t, err)
err = drv.InsertMigration(db, "abc2")
require.NoError(t, err)
// DumpSchema should return schema
schema, err := drv.DumpSchema(db)
require.NoError(t, err)
require.Contains(t, string(schema), "CREATE TABLE IF NOT EXISTS \"test_migrations\"")
require.Contains(t, string(schema), ");\n-- Dbmate schema migrations\n"+
"INSERT INTO \"test_migrations\" (version) VALUES\n"+
" ('abc1'),\n"+
" ('abc2');\n")
// DumpSchema should return error if command fails
drv.databaseURL = dbutil.MustParseURL(".")
schema, err = drv.DumpSchema(db)
require.Nil(t, schema)
require.Error(t, err)
require.EqualError(t, err, "Error: unable to open database \".\": "+
"unable to open database file")
}
func TestSQLiteDatabaseExists(t *testing.T) {
drv := testSQLiteDriver(t)
// drop any existing database
err := drv.DropDatabase()
require.NoError(t, err)
// DatabaseExists should return false
exists, err := drv.DatabaseExists()
require.NoError(t, err)
require.Equal(t, false, exists)
// create database
err = drv.CreateDatabase()
require.NoError(t, err)
// DatabaseExists should return true
exists, err = drv.DatabaseExists()
require.NoError(t, err)
require.Equal(t, true, exists)
}
func TestSQLiteCreateMigrationsTable(t *testing.T) {
t.Run("default table", func(t *testing.T) {
drv := testSQLiteDriver(t)
db := prepTestSQLiteDB(t)
defer dbutil.MustClose(db)
// migrations table should not exist
count := 0
err := db.QueryRow("select count(*) from schema_migrations").Scan(&count)
require.Error(t, err)
require.Regexp(t, "no such table: schema_migrations", err.Error())
// create table
err = drv.CreateMigrationsTable(db)
require.NoError(t, err)
// migrations table should exist
err = db.QueryRow("select count(*) from schema_migrations").Scan(&count)
require.NoError(t, err)
// create table should be idempotent
err = drv.CreateMigrationsTable(db)
require.NoError(t, err)
})
t.Run("custom table", func(t *testing.T) {
drv := testSQLiteDriver(t)
drv.migrationsTableName = "test_migrations"
db := prepTestSQLiteDB(t)
defer dbutil.MustClose(db)
// migrations table should not exist
count := 0
err := db.QueryRow("select count(*) from test_migrations").Scan(&count)
require.Error(t, err)
require.Regexp(t, "no such table: test_migrations", err.Error())
// create table
err = drv.CreateMigrationsTable(db)
require.NoError(t, err)
// migrations table should exist
err = db.QueryRow("select count(*) from test_migrations").Scan(&count)
require.NoError(t, err)
// create table should be idempotent
err = drv.CreateMigrationsTable(db)
require.NoError(t, err)
})
}
func TestSQLiteSelectMigrations(t *testing.T) {
drv := testSQLiteDriver(t)
drv.migrationsTableName = "test_migrations"
db := prepTestSQLiteDB(t)
defer dbutil.MustClose(db)
err := drv.CreateMigrationsTable(db)
require.NoError(t, err)
_, err = db.Exec(`insert into test_migrations (version)
values ('abc2'), ('abc1'), ('abc3')`)
require.NoError(t, err)
migrations, err := drv.SelectMigrations(db, -1)
require.NoError(t, err)
require.Equal(t, true, migrations["abc1"])
require.Equal(t, true, migrations["abc2"])
require.Equal(t, true, migrations["abc2"])
// test limit param
migrations, err = drv.SelectMigrations(db, 1)
require.NoError(t, err)
require.Equal(t, true, migrations["abc3"])
require.Equal(t, false, migrations["abc1"])
require.Equal(t, false, migrations["abc2"])
}
func TestSQLiteInsertMigration(t *testing.T) {
drv := testSQLiteDriver(t)
drv.migrationsTableName = "test_migrations"
db := prepTestSQLiteDB(t)
defer dbutil.MustClose(db)
err := drv.CreateMigrationsTable(db)
require.NoError(t, err)
count := 0
err = db.QueryRow("select count(*) from test_migrations").Scan(&count)
require.NoError(t, err)
require.Equal(t, 0, count)
// insert migration
err = drv.InsertMigration(db, "abc1")
require.NoError(t, err)
err = db.QueryRow("select count(*) from test_migrations where version = 'abc1'").
Scan(&count)
require.NoError(t, err)
require.Equal(t, 1, count)
}
func TestSQLiteDeleteMigration(t *testing.T) {
drv := testSQLiteDriver(t)
drv.migrationsTableName = "test_migrations"
db := prepTestSQLiteDB(t)
defer dbutil.MustClose(db)
err := drv.CreateMigrationsTable(db)
require.NoError(t, err)
_, err = db.Exec(`insert into test_migrations (version)
values ('abc1'), ('abc2')`)
require.NoError(t, err)
err = drv.DeleteMigration(db, "abc2")
require.NoError(t, err)
count := 0
err = db.QueryRow("select count(*) from test_migrations").Scan(&count)
require.NoError(t, err)
require.Equal(t, 1, count)
}
func TestSQLitePing(t *testing.T) {
drv := testSQLiteDriver(t)
path := ConnectionString(drv.databaseURL)
// drop any existing database
err := drv.DropDatabase()
require.NoError(t, err)
// ping database
err = drv.Ping()
require.NoError(t, err)
// check that the database was created (sqlite-only behavior)
_, err = os.Stat(path)
require.NoError(t, err)
// drop the database
err = drv.DropDatabase()
require.NoError(t, err)
// create directory where database file is expected
err = os.Mkdir(path, 0755)
require.NoError(t, err)
defer func() {
err = os.RemoveAll(path)
require.NoError(t, err)
}()
// ping database should fail
err = drv.Ping()
require.EqualError(t, err, "unable to open database file: is a directory")
}
func TestSQLiteQuotedMigrationsTableName(t *testing.T) {
t.Run("default name", func(t *testing.T) {
drv := testSQLiteDriver(t)
name := drv.quotedMigrationsTableName()
require.Equal(t, `"schema_migrations"`, name)
})
t.Run("custom name", func(t *testing.T) {
drv := testSQLiteDriver(t)
drv.migrationsTableName = "fooMigrations"
name := drv.quotedMigrationsTableName()
require.Equal(t, `"fooMigrations"`, name)
})
}

View file

@ -1,9 +0,0 @@
-- migrate:up
create table categories (
id integer,
title varchar(50),
slug varchar(100)
);
-- migrate:down
drop table categories;