diff --git a/activate b/activate new file mode 100644 index 0000000..04433ea --- /dev/null +++ b/activate @@ -0,0 +1,7 @@ +# This is a convenience for development purposes, +# for deployments you should set the appropriate +# environment variables explicitly and use fully +# qualified paths to the interpreter, as is done +# for example in the Dockerfile + +source <(make activate;); diff --git a/docker_base.Dockerfile b/docker_base.Dockerfile index bf00033..47c588d 100644 --- a/docker_base.Dockerfile +++ b/docker_base.Dockerfile @@ -1,13 +1,13 @@ # Stages: -# alpine_base : Common base image, both for the builder and for the final image. -# This contains only minimal dependencies required in both cases -# for miniconda and the makefile. -# builder : stage in which the conda envrionment is created -# and dependencies are installed -# final : the final image containing only the required environment files, -# and none of the infrastructure required to generate them. +# base_image : Common base image, both for the builder and for the final image. +# This contains only minimal dependencies required in both cases +# for miniconda and the makefile. +# builder : stage in which the conda envrionment is created +# and dependencies are installed +# final : the final image containing only the required environment files, +# and none of the infrastructure required to generate them. -FROM frolvlad/alpine-glibc AS alpine_base +FROM debian:stable-slim AS base_image ENV LC_ALL=C.UTF-8 ENV LANG=C.UTF-8 @@ -17,14 +17,27 @@ ENV CONDA_DIR /opt/conda ENV PATH $CONDA_DIR/bin:$PATH ENV SHELL /bin/bash -RUN apk add --no-cache bash make sed grep gawk curl bzip2 unzip -RUN apk add --no-cache git mercurial +RUN if [ $(which apk) ]; then \ + apk add --no-cache bash make sed grep gawk curl git bzip2 unzip; \ + elif [ $(which apt-get) ]; then \ + apt-get update && apt-get install --yes bash make sed grep gawk curl git bzip2 unzip; \ + else \ + echo "Invalid Distro: $(uname -a)"; \ + exit 1; \ + fi CMD [ "/bin/bash" ] -FROM alpine_base AS builder +FROM base_image AS builder -RUN apk add --no-cache ca-certificates openssh-client openssh-keygen +RUN if [ $(which apk) ]; then \ + apk add --no-cache ca-certificates openssh-client openssh-keygen; \ + elif [ $(which apt-get) ]; then \ + apt-get --yes install ca-certificates openssh-client; \ + else \ + echo "Invalid Distro: $(uname -a)"; \ + exit 1; \ + fi ENV MINICONDA_VER latest ENV MINICONDA Miniconda3-$MINICONDA_VER-Linux-x86_64.sh @@ -90,7 +103,7 @@ RUN conda clean --all --yes && \ rm -rf /opt/conda/pkgs/ -FROM alpine_base +FROM base_image COPY --from=builder /opt/conda/ /opt/conda/ COPY --from=builder /vendor/ /vendor diff --git a/makefile b/makefile index 3b4300b..feb8484 100644 --- a/makefile +++ b/makefile @@ -14,10 +14,6 @@ SHELL := /bin/bash PROJECT_DIR := $(notdir $(abspath .)) -ifndef MODULE_SRC_PATH - MODULE_SRC_PATH := $(notdir $(abspath .)) -endif - ifndef DEVELOPMENT_PYTHON_VERSION DEVELOPMENT_PYTHON_VERSION := python=3.6 endif @@ -27,7 +23,6 @@ ifndef SUPPORTED_PYTHON_VERSIONS endif PKG_NAME := $(PACKAGE_NAME) -MODULE_SRC_PATH = src/$(PKG_NAME)/ # TODO (mb 2018-09-23): Support for bash on windows # perhaps we need to install conda using this @@ -35,7 +30,7 @@ MODULE_SRC_PATH = src/$(PKG_NAME)/ PLATFORM = $(shell uname -s) # miniconda is shared between projects -CONDA_ROOT := $(shell if [[ -d /opt/conda ]]; then echo "/opt/conda"; else echo "$$HOME/miniconda3"; fi;) +CONDA_ROOT := $(shell if [[ -d /opt/conda/envs ]]; then echo "/opt/conda"; else echo "$$HOME/miniconda3"; fi;) CONDA_BIN := $(CONDA_ROOT)/bin/conda ENV_PREFIX := $(CONDA_ROOT)/envs @@ -49,6 +44,15 @@ CONDA_ENV_NAMES := \ CONDA_ENV_PATHS := \ $(subst py,${ENV_PREFIX}/$(PKG_NAME)_py,$(subst .,,$(subst =,,$(subst thon,,$(SUPPORTED_PYTHON_VERSIONS))))) +literal_space := $() $() + +BDIST_WHEEL_PYTHON_TAG := \ + $(subst python,py,$(subst $(literal_space),.,$(subst .,,$(subst =,,$(SUPPORTED_PYTHON_VERSIONS))))) + +SDIST_FILE_CMD = ls -1t dist/*.tar.gz | head -n 1 + +BDIST_WHEEL_FILE_CMD = ls -1t dist/*.whl | head -n 1 + # default version for development DEV_ENV := $(ENV_PREFIX)/$(DEV_ENV_NAME) @@ -56,6 +60,7 @@ DEV_ENV_PY := $(DEV_ENV)/bin/python RSA_KEY_PATH := ${HOME}/.ssh/${PKG_NAME}_gitlab_runner_id_rsa +DOCKER_BASE_IMAGE := registry.gitlab.com/mbarkhau/pycalver/base:latest build/envs.txt: requirements/conda.txt @mkdir -p build/; @@ -130,11 +135,9 @@ build/deps.txt: build/envs.txt requirements/*.txt @mv build/deps.txt.tmp build/deps.txt -## This help message +## Short help message for each task. .PHONY: help help: - @printf "Available make targets for \033[97m$(PKG_NAME)\033[0m:\n"; - @awk '{ \ if ($$0 ~ /^.PHONY: [a-zA-Z\-\_0-9]+$$/) { \ helpCommand = substr($$0, index($$0, ":") + 2); \ @@ -143,15 +146,20 @@ help: helpCommand, helpMessage; \ helpMessage = ""; \ } \ - } else if ($$0 ~ /^##/) { \ + } else if ($$0 ~ /^[a-zA-Z\-\_0-9.]+:/) { \ + helpCommand = substr($$0, 0, index($$0, ":")); \ if (helpMessage) { \ - helpMessage = helpMessage"\n "substr($$0, 3); \ - } else { \ + printf "\033[36m%-20s\033[0m %s\n", \ + helpCommand, helpMessage; \ + helpMessage = ""; \ + } \ + } else if ($$0 ~ /^##/) { \ + if (! (helpMessage)) { \ helpMessage = substr($$0, 3); \ } \ } else { \ if (helpMessage) { \ - print "\n "helpMessage"\n" \ + print " "helpMessage \ } \ helpMessage = ""; \ } \ @@ -177,6 +185,42 @@ help: fi +## Full help message for each task. +.PHONY: fullhelp +fullhelp: + @printf "Available make targets for \033[97m$(PKG_NAME)\033[0m:\n"; + + @awk '{ \ + if ($$0 ~ /^.PHONY: [a-zA-Z\-\_0-9]+$$/) { \ + helpCommand = substr($$0, index($$0, ":") + 2); \ + if (helpMessage) { \ + printf "\033[36m%-20s\033[0m %s\n", \ + helpCommand, helpMessage; \ + helpMessage = ""; \ + } \ + } else if ($$0 ~ /^[a-zA-Z\-\_0-9.]+:/) { \ + helpCommand = substr($$0, 0, index($$0, ":")); \ + if (helpMessage) { \ + printf "\033[36m%-20s\033[0m %s\n", \ + helpCommand, helpMessage; \ + helpMessage = ""; \ + } \ + } else if ($$0 ~ /^##/) { \ + if (helpMessage) { \ + helpMessage = helpMessage"\n "substr($$0, 3); \ + } else { \ + helpMessage = substr($$0, 3); \ + } \ + } else { \ + if (helpMessage) { \ + print "\n "helpMessage"\n" \ + } \ + helpMessage = ""; \ + } \ + }' \ + $(MAKEFILE_LIST) + + ## -- Project Setup -- @@ -236,21 +280,7 @@ git_hooks: ln -s "${PWD}/scripts/pre-push-hook.sh" "${PWD}/.git/hooks/pre-push" - -# TODO make target to publish on pypi -# .PHONY: publish -# publish: -# echo "Not Implemented" - - -## -- Development -- - - -## Run code formatter on src/ and test/ -.PHONY: fmt -fmt: - @$(DEV_ENV)/bin/sjfmt --py36 --skip-string-normalization --line-length=100 \ - src/ test/ +## -- Integration -- ## Run flake8 linter @@ -294,24 +324,24 @@ test: @rm -rf "src/__pycache__"; @rm -rf "test/__pycache__"; -ifndef FILTER + # First we test the local source tree using the dev environment ENV=$${ENV-dev} PYTHONPATH=src/:vendor/:$$PYTHONPATH \ $(DEV_ENV_PY) -m pytest -v \ --doctest-modules \ + --verbose \ --cov-report html \ --cov-report term \ - $(shell ls -1 src/ | awk '{ print "--cov "$$1 }') \ + $(shell cd src/ && ls -1 */__init__.py | awk '{ print "--cov "substr($$1,0,index($$1,"/")-1) }') \ test/ src/; -else - ENV=$${ENV-dev} PYTHONPATH=src/:vendor/:$$PYTHONPATH \ - $(DEV_ENV_PY) -m pytest -v \ - --doctest-modules \ - --cov-report html \ - --cov-report term \ - $(shell ls -1 src/ | awk '{ print "--cov "$$1 }') \ - -k $(FILTER) \ - test/ src/; -endif + + # Next we install the package and run the test suite against it. + + IFS=' ' read -r -a env_paths <<< "$(CONDA_ENV_PATHS)"; \ + for i in $${!env_paths[@]}; do \ + env_py=$${env_paths[i]}/bin/python; \ + $${env_py} -m pip install --upgrade .; \ + ENV=$${ENV-dev} $${env_py} -m pytest test/; \ + done; @rm -rf ".pytest_cache"; @rm -rf "src/__pycache__"; @@ -321,27 +351,58 @@ endif ## -- Helpers -- +## Run code formatter on src/ and test/ +.PHONY: fmt +fmt: + @$(DEV_ENV)/bin/sjfmt --py36 --skip-string-normalization --line-length=100 \ + src/ test/ + + ## Shortcut for make fmt lint pylint test .PHONY: check check: fmt lint mypy test -## Start shell with environ variables set. -.PHONY: env -env: +## Start subshell with environ variables set. +.PHONY: env_subshell +env_subshell: @bash --init-file <(echo '\ source $$HOME/.bashrc; \ -<<<<<<< HEAD - source $(CONDA_ROOT)/etc/profile.d/conda.sh \ -======= source $(CONDA_ROOT)/etc/profile.d/conda.sh; \ ->>>>>>> fix make/bash escape issue export ENV=$${ENV-dev}; \ export PYTHONPATH="src/:vendor/:$$PYTHONPATH"; \ conda activate $(DEV_ENV_NAME) \ ') +## Usage: "source activate", to deactivate: "deactivate" +.PHONY: activate +activate: + @echo 'source $(CONDA_ROOT)/etc/profile.d/conda.sh;' + @echo 'if [[ -z $$ENV ]]; then' + @echo ' export _env_before_activate_$(DEV_ENV_NAME)=$${ENV};' + @echo 'fi' + @echo 'if [[ -z $$PYTHONPATH ]]; then' + @echo ' export _pythonpath_before_activate_$(DEV_ENV_NAME)=$${PYTHONPATH};' + @echo 'fi' + @echo 'export ENV=$${ENV-dev};' + @echo 'export PYTHONPATH="src/:vendor/:$$PYTHONPATH";' + @echo 'conda activate $(DEV_ENV_NAME);' + @echo 'function deactivate {' + @echo ' if [[ -z $${_env_before_activate_$(DEV_ENV_NAME)} ]]; then' + @echo ' export ENV=$${_env_before_activate_$(DEV_ENV_NAME)}; ' + @echo ' else' + @echo ' unset ENV;' + @echo ' fi' + @echo ' if [[ -z $${_pythonpath_before_activate_$(DEV_ENV_NAME)} ]]; then' + @echo ' export PYTHONPATH=$${_pythonpath_before_activate_$(DEV_ENV_NAME)}; ' + @echo ' else' + @echo ' unset PYTHONPATH;' + @echo ' fi' + @echo ' conda deactivate;' + @echo '};' + + ## Drop into an ipython shell with correct env variables set .PHONY: ipy ipy: @@ -356,8 +417,7 @@ devtest: @rm -rf "src/__pycache__"; @rm -rf "test/__pycache__"; - -ifndef FILTER +ifdef FILTER ENV=$${ENV-dev} PYTHONPATH=src/:vendor/:$$PYTHONPATH \ $(DEV_ENV_PY) -m pytest -v \ --doctest-modules \ @@ -365,6 +425,7 @@ ifndef FILTER --verbose \ --capture=no \ --exitfirst \ + -k $(FILTER) \ test/ src/; else ENV=$${ENV-dev} PYTHONPATH=src/:vendor/:$$PYTHONPATH \ @@ -374,7 +435,6 @@ else --verbose \ --capture=no \ --exitfirst \ - -k $(FILTER) \ test/ src/; endif @@ -386,34 +446,54 @@ endif ## -- Build/Deploy -- -## Generate Documentation +# Generate Documentation # .PHONY: doc # doc: # echo "Not Implemented" -## Bump Version number in all files -# .PHONY: bump_version -# bump_version: -# echo "Not Implemented" - - ## Freeze dependencies of the current development env. ## The requirements files this produces should be used ## in order to have reproducable builds, otherwise you ## should minimize the number of pinned versions in ## your requirements. -# .PHONY: freeze -# freeze: -# echo "Not Implemented" +.PHONY: freeze +freeze: + $(DEV_ENV_PY) -m pip freeze \ + > requirements/$(shell date -u +"%Y%m%dt%H%M%S")_freeze.txt -## Create python sdist and bdist_wheel distributions -.PHONY: build_dist -build_dist: - $(DEV_ENV_PY) setup.py sdist bdist_wheel - twine check dist/* - echo "To a PUBLIC release on pypi run:\n\t\$(DEV_ENV_PY) setup.py upload" +## Bump Version number in all files +.PHONY: bump_version +bump_version: + echo "Not Implemented" + # $(DEV_ENV)/bin/pycalver bump; + + +## Create python sdist and bdist_wheel files +.PHONY: build_dists +build_dists: + $(DEV_ENV_PY) setup.py sdist; + $(DEV_ENV_PY) setup.py bdist_wheel --python-tag=$(BDIST_WHEEL_PYTHON_TAG); + + +## Upload sdist and bdist files to pypi +.PHONY: upload_dists +upload_dists: + @if [[ "1" != "1" ]]; then \ + echo "FAILSAFE! Not publishing a private package."; \ + echo " To avoid this set IS_PUBLIC=1 in bootstrap.sh and run it."; \ + exit 1; \ + fi + + $(DEV_ENV)/bin/twine check $$($(SDIST_FILE_CMD)); + $(DEV_ENV)/bin/twine check $$($(BDIST_WHEEL_FILE_CMD)); + $(DEV_ENV)/bin/twine upload $$($(SDIST_FILE_CMD)) $$($(BDIST_WHEEL_FILE_CMD)); + + +## Publish on pypi +.PHONY: publish +publish: bump_version build_dists upload_dists ## Build docker images. Must be run when dependencies are added @@ -430,16 +510,18 @@ build_docker: docker build \ --build-arg SSH_PRIVATE_RSA_KEY="$$(cat '${RSA_KEY_PATH}')" \ --file docker_base.Dockerfile \ - --tag $(DOCKER_REGISTRY_URL)/base:latest \ + --tag $(DOCKER_BASE_IMAGE) \ .; \ else \ docker build \ --file docker_base.Dockerfile \ - --tag $(DOCKER_REGISTRY_URL)/base:latest \ + --tag $(DOCKER_BASE_IMAGE) \ .; \ fi - docker push $(DOCKER_REGISTRY_URL)/base:latest + docker push $(DOCKER_BASE_IMAGE) +## -- Extra/Custom/Project Specific Tasks -- + -include makefile.extra.make diff --git a/makefile.config.make b/makefile.config.make index 6d6578a..53ebe7d 100644 --- a/makefile.config.make +++ b/makefile.config.make @@ -9,8 +9,9 @@ DOCKER_REGISTRY_URL := registry.gitlab.com/mbarkhau/pycalver # - `make devtest` DEVELOPMENT_PYTHON_VERSION := python=3.6 -# These must be valid conda package names. A separate -# conda environment will be created for each of these. +# These must be valid (space separated) conda package names. +# A separate conda environment will be created for each of these. +# # Some valid options are: # - python=2.7 # - python=3.5 diff --git a/requirements/conda.txt b/requirements/conda.txt index bdd0e41..5f5e4bb 100644 --- a/requirements/conda.txt +++ b/requirements/conda.txt @@ -24,11 +24,15 @@ ujson # The hot new pkdf on the block is argon2, winner of # the https://password-hashing.net/ competition. -argon2_cffi +# argon2_cffi # https://blake2.net/ # BLAKE2 is a cryptographic hash function faster than MD5, SHA-1, # SHA-2, and SHA-3, yet is at least as secure as the latest standard # SHA-3. BLAKE2 has been adopted by many projects due to its high # speed, security, and simplicity. -pyblake2 +# pyblake2 + +# pytest is required in every environment to run the test suite +# against the installed modules. +pytest diff --git a/requirements/development.txt b/requirements/development.txt index ab63a13..b80a28d 100644 --- a/requirements/development.txt +++ b/requirements/development.txt @@ -34,4 +34,3 @@ snakeviz # add one after you've tested it and found it to be actually useful. ipython # nuff said - diff --git a/requirements/integration.txt b/requirements/integration.txt index 96947d3..36eeaba 100644 --- a/requirements/integration.txt +++ b/requirements/integration.txt @@ -18,6 +18,7 @@ pytest pytest-cov pylint +readme_renderer[md] twine md-toc diff --git a/setup.cfg b/setup.cfg index 5ba6fd9..127f0f3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -64,19 +64,20 @@ addopts = --doctest-modules [pycalver] -current_version = v201811.0007-beta +current_version = v201812.0007-beta commit = True tag = True push = True - [pycalver:file_patterns] -"setup.cfg" = +bootstrapit.sh = + PACKAGE_VERSION="{version}" +setup.cfg = current_version = {version} -"setup.py" = +setup.py = version="{pep440_version}" -"src/pycalver/__init__.py" = +src/pycalver/__init__.py = __version__ = "{version}" -"README.md" = +README.md = [PyCalVer {calver}{build}-{release}] img.shields.io/badge/PyCalVer-{calver}{build}--{release}-blue