From 091747fa48f9dbcbb4ea60a398a0ef1c69bba6fa Mon Sep 17 00:00:00 2001 From: Manuel Barkhau Date: Thu, 14 Feb 2019 23:19:18 +0100 Subject: [PATCH] bootstrap updates --- CONTRIBUTING.md | 188 ++++++++++++++++++------------- LICENSE | 7 +- bootstrapit.sh | 2 +- docker_base.Dockerfile | 4 +- license.header | 2 +- makefile | 49 ++++---- requirements/conda.txt | 1 - scripts/pre-push-hook.sh | 0 scripts/setup_conda_envs.sh | 0 scripts/update_conda_env_deps.sh | 0 src/pycalver/__init__.py | 4 +- src/pycalver/__main__.py | 2 +- 12 files changed, 149 insertions(+), 110 deletions(-) mode change 100644 => 100755 scripts/pre-push-hook.sh mode change 100644 => 100755 scripts/setup_conda_envs.sh mode change 100644 => 100755 scripts/update_conda_env_deps.sh diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e687936..40f8e8b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -136,25 +136,38 @@ You can also activate the default virtual environment as follows. ```shell (myproject_py36) dev@host:~/myproject -$ conda env activate myproject_py36 +$ source ./activate +$ which python /home/dev/miniconda3/envs/myproject_py36/bin/python $ ipython -Python 3.6.6 | packaged by conda-forge | (default, Jul 26 2018, 09:53:17) +Python 3.6.6 |Anaconda, Inc.| (default, Jun 28 2018, 17:14:51) t Type 'copyright', 'credits' or 'license' for more information IPython 6.5.0 -- An enhanced Interactive Python. Type '?' for help. -In [1]: +In [1]: import sys + +In [2]: sys.path +Out[2]: +['/home/dev/miniconda3/envs/pycalver_py36/bin', + '/home/dev/myproject/src', + '/home/dev/myproject/vendor', + ... +In [3]: import myproject + +In [4]: myproject.__file__ +Out[4]: '/home/dev/myproject/src/myproject/__init__.py' ``` - -Note however, that this invocation does not have the correct -`PYTHONPATH` set up to import modules of the project. You can -review the definition for ``make ipy`` to see how to set up -`PYTHONPATH` correctly. +Note that the `PYTHONPATH` has been set up to import modules +of the project. You can review the definition for `make ipy` +to see how to set up `PYTHONPATH` correctly. ```shell +$ make ipy --dry-run +ENV=${ENV-dev} PYTHONPATH=src/:vendor/:$PYTHONPATH \ + /home/dev/miniconda3/envs/myproject_py36/bin/ipython $ make ipy Python 3.6.6 |Anaconda, Inc.| (default, Jun 28 2018, 17:14:51) Type 'copyright', 'credits' or 'license' for more information @@ -163,7 +176,7 @@ IPython 6.5.0 -- An enhanced Interactive Python. Type '?' for help. In [1]: import myproject In [2]: myproject.__file__ -Out[2]: '/mnt/c/Users/ManuelBarkhau/myproject/src/myproject/__init__.py' +Out[2]: '/home/dev/myproject/src/myproject/__init__.py' ``` @@ -199,10 +212,15 @@ projects by reducing the burdon of project setup to a minimum. README.md # project overview and status CONTRIBUTING.md # guide for developers - CHANGELOG.md # for public libraries + CHANGELOG.md # short documentation of release history LICENSE # for public libraries (MIT preferred) - makefile # main project and environment management file + makefile # main project and environment management file + makefile.config.make # project configuration variables + makefile.extra.make # project specific make targets + + docker_base.Dockerfile # base image for CI (only conda envs) + Dockerfile # image with source of the project ### Dependency Management @@ -223,9 +241,9 @@ requirements/development.txt # useful packgages for development/debugging requirements/integration.txt # used for linting/testing/packaging # These are the requirements produced for specific builds. They can be -# used to debug version compatatbility issues . They are generated -# using pip freeze -requirements/build-0123.freeze +# used to debug version compatatbility issues. They are generated +# using make freeze +requirements/20190214t212403_freeze.txt ``` @@ -233,14 +251,14 @@ When adding a new dependency please consider: - Only specify direct dependencies of the project, not transitive dependencies of other projects. These are installed via their - dependency declarations. -- The default specifier for a package should be only its name without - a version specifier. With this as the default, the project remains - up to date in terms of security fixes and other library - improvements. -- Some packages consider some of their dependancies to be optional, in - which case you will have to specify their transitive dependencies - + own respective dependency declarations. +- Whenever possible, the specifier for a package should be only its + name without a version specifier. With this as the default, the + project remains up to date in terms of security fixes and other + library improvements. +- Some packages consider some of their dependancies to be optional, + in which case you will have to specify their transitive + dependencies. - Only specify/pin/freeze a specific (older) version if there are known issues, or your project requires features from an unstable (alpha/beta) version of the package. Each pinned version should @@ -270,27 +288,28 @@ Further Reading: Dependencies are installed in this order: - - ``conda.txt`` - - ``pypi.txt`` - - ``vendor.txt`` - - ``development.txt`` - - ``integration.txt`` + - `conda.txt` + - `pypi.txt` + - `vendor.txt` + - `development.txt` + - `integration.txt` -Please review the documentation header at the beginning of each file -to determine which file is appropriate for the dependency you want to -add. +Please review the documentation header at the beginning of each +`requirements/*.txt` file to determine which file is appropriate +for the dependency you want to add. Choose a file: -- ``conda.txt`` is appropriate for non python packages and packages - which would require compilation if they were downloaded from pypi. -- ``pypi.txt`` is for dependencies on python packages, be they from +- `conda.txt` is appropriate for non python packages and packages + which would require compilation if they were downloaded from pypi + or cannot be downloaded from pypi (such as openjdk or node). +- `pypi.txt` is for dependencies on python packages, be they from pypi or git repositories. -- ``vendor.txt`` is appropriate for pure python libaries which are +- `vendor.txt` is appropriate for pure python libaries which are written using mypy. This allows the mypy type checker to work with types defined in other packages -After adding a new dependency, you can run ``make update`` +After adding a new dependency, you can run `make update` ```shell @@ -303,11 +322,16 @@ requests-2.19.1 | 94 KB conda-forge ... ``` +Normally make update only does something if you update one of the +`requirements/*.txt` files has changed. If you know a dependency +was updated, and `make update` is not having an effect, you can +force the update using `make force update`. + ### Vendoring Vendored dependencies are usually committed to git, but if you -trust the package maintainer and the installation via vendor.txt, +trust the package maintainer and the installation via `vendor.txt`, then it's not required. There are a few reasons to vendor a dependency: @@ -326,30 +350,57 @@ contribute to the upstream project when possible. ## Development -TODO: document development tasks like lint, type checking in a -platform independent way, ideally they work with PyCharm. Until -then, these are platform agnostic commands that have to be -entered manually. +The typical commands used during development are: + +- `make install`: Setup virtual environment +- `source activate`: Activate virtual environment +- `make help`: Overview of tasks +- `make fmt`: Format code +- `make lint`: Linting +- `make mypy`: Typecheck +- `make devtest`: Run unittests with dev interpreter against code from `src/`. + +Slightly less common but good to run before doing `git push`. + +- `make test`: Run unitests on all supported interpreters after installing + using `python setup.py install`. This tests the code as the users of your + library will have installed. +- `make citest`: Run `make test` but inside a docker container, which is as + close to the ci environment as possible. This is quite useful if you don't + want to trigger dozens of CI builds to debug a tricky issue. -### Linting +### Docker + +The base image of the project is `docker_base.Dockerfile` which is +used to create images that have only the conda virtual environment needed +to run the project. The CI environment uses the image generated by +`make docker_build`. While this means that the CI setup is simpler and faster, +as you don't have to build the image for the test run in the CI environment, it does mean that you have to run `make docker_build` every time one of your dependencies is updated. + +The `docker_base.Dockerfile` uses the multi stage builder pattern, so that 1. +your private key doesn't end up in the published image 2. the published image +is as small as possible. -```shell -flake8 src/ -sjfmt --py36 src/ +``` +$ make docker_build +Sending build context to Docker daemon 7.761MB +Step 1/20 : FROM registry.gitlab.com/mbarkhau/bootstrapit/env_builder AS builder +... +conda create --name myproject_py36 python=3.6 ... +Solving environment: ...working... done +... +conda create --name myproject_py35 python=3.5 ... +Solving environment: ...working... done + +docker push ``` - -### Type Checking - - -TODO: This is left open, until the mypy setup is complete - -```shell -mypy src/ -pytest test/ -``` +As is the case for your local development setup, every version of python +that you have configured to be supported, is installed in the image. If +you want to create a minimal image for a production system, you may wish +to trim this down. ### Documentation @@ -360,28 +411,13 @@ decent cross platform editor. TODO: `make doc` -### Setup to run docker +### Editor Setup -TODO: - - -### PyCharm - +https://gitlab.com/mbarkhau/straitjacket#editortooling-integration TODO: Expand how to set editor, possibly by sharing editor config files? -Recoomended plugins: - -https://plugins.jetbrains.com/plugin/10563-black-pycharm -https://plugins.jetbrains.com/plugin/7642-save-actions - - -### Sublime Text - - -https://github.com/jgirardet/sublack - ## Best Practices @@ -409,12 +445,12 @@ Please read, view at your leasure: Keep in mind, that all of this is about the form of your code, and catching common pitfalls or gotchas. None of this releives you of the burdon of thinking about your code. The reason to use linters and type -checking is not to make the code correct, but to help you make your -code correct. +checking is not to have a tool to make your code correct, but to +support you to make your code correct. For now I won't go into the effort of writing yet another style guide. -Instead, if your code passes `make lint`, then it's acceptable. Every -time you encounter a linting error, consider it as an opportinity to -learn a best practice. +Instead, if your code passes `make fmt lint`, then it's acceptable. +Every time you encounter a linting error, consider it as an opportinity +to learn a best practice and look up the error code. [^1]: Linux, MacOS and [WSL](https://docs.microsoft.com/en-us/windows/wsl/install-win10) diff --git a/LICENSE b/LICENSE index 7cd63b4..b775855 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2018 Manuel Barkhau (mbarkhau@gmail.com) +Copyright (c) 2019 Manuel Barkhau (mbarkhau@gmail.com) Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -9,8 +9,9 @@ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. +The above copyright notice and this permission notice (including the next +paragraph) shall be included in all copies or substantial portions of the +Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS diff --git a/bootstrapit.sh b/bootstrapit.sh index ed70075..27be8c8 100644 --- a/bootstrapit.sh +++ b/bootstrapit.sh @@ -5,7 +5,7 @@ AUTHOR_NAME="Manuel Barkhau" AUTHOR_EMAIL="mbarkhau@gmail.com" KEYWORDS="version versioning bumpversion calver" -DESCRIPTION="CalVer for python libraries." +DESCRIPTION="CalVer for python packages." LICENSE_ID="MIT" diff --git a/docker_base.Dockerfile b/docker_base.Dockerfile index 9c8006c..00b270b 100644 --- a/docker_base.Dockerfile +++ b/docker_base.Dockerfile @@ -17,7 +17,7 @@ ARG SSH_PRIVATE_RSA_KEY ENV ENV_SSH_PRIVATE_RSA_KEY=${SSH_PRIVATE_RSA_KEY} # Write private key and generate public key -RUN if [[ "$ENV_SSH_PRIVATE_RSA_KEY" ]]; then \ +RUN if ! test -z "${ENV_SSH_PRIVATE_RSA_KEY}"; then \ echo -n "-----BEGIN RSA PRIVATE KEY-----" >> /root/.ssh/id_rsa && \ echo -n ${ENV_SSH_PRIVATE_RSA_KEY} \ | sed 's/-----BEGIN RSA PRIVATE KEY-----//' \ @@ -41,7 +41,7 @@ RUN make install RUN rm -f /root/.ssh/id_rsa # Deleting pkgs implies that `conda install` -# will at have to pull all packages again. +# will have to pull all packages again. RUN conda clean --all --yes # Conda docs say that it is not safe to delete pkgs # because there may be symbolic links, so we verify diff --git a/license.header b/license.header index f28c8c9..7fb5895 100644 --- a/license.header +++ b/license.header @@ -3,7 +3,7 @@ Individual files contain the following tag instead of the full license text. This file is part of the pycalver project https://gitlab.com/mbarkhau/pycalver - Copyright (c) 2018 Manuel Barkhau (mbarkhau@gmail.com) - MIT License + Copyright (c) 2019 Manuel Barkhau (mbarkhau@gmail.com) - MIT License SPDX-License-Identifier: MIT This enables machine processing of license information based on the SPDX diff --git a/makefile b/makefile index 66eec0c..2262f11 100644 --- a/makefile +++ b/makefile @@ -42,10 +42,11 @@ CONDA_ENV_NAMES := \ $(subst pypy,$(PKG_NAME)_pypy,$(subst python=,$(PKG_NAME)_py,$(subst .,,$(SUPPORTED_PYTHON_VERSIONS)))) CONDA_ENV_PATHS := \ - $(subst pypy,${ENV_PREFIX}/$(PKG_NAME)_pypy,$(subst python=,${ENV_PREFIX}/$(PKG_NAME)_py,$(subst .,,$(SUPPORTED_PYTHON_VERSIONS)))) + $(subst pypy,$(ENV_PREFIX)/$(PKG_NAME)_pypy,$(subst python=,$(ENV_PREFIX)/$(PKG_NAME)_py,$(subst .,,$(SUPPORTED_PYTHON_VERSIONS)))) -literal_space := $() $() +empty := +literal_space := $(empty) $(empty) BDIST_WHEEL_PYTHON_TAG := \ $(subst python,py,$(subst $(literal_space),.,$(subst .,,$(subst =,,$(SUPPORTED_PYTHON_VERSIONS))))) @@ -59,11 +60,12 @@ BDIST_WHEEL_FILE_CMD = ls -1t dist/*.whl | head -n 1 DEV_ENV := $(ENV_PREFIX)/$(DEV_ENV_NAME) DEV_ENV_PY := $(DEV_ENV)/bin/python -RSA_KEY_PATH := ${HOME}/.ssh/${PKG_NAME}_gitlab_runner_id_rsa +RSA_KEY_PATH := $(HOME)/.ssh/$(PKG_NAME)_gitlab_runner_id_rsa DOCKER_BASE_IMAGE := registry.gitlab.com/mbarkhau/pycalver/base -DOCKER_IMAGE_VERSION := $(shell date -u +'%Y%m%dt%H%M%S')_$(shell git rev-parse --short HEAD) +GIT_HEAD_REV = $(shell git rev-parse --short HEAD) +DOCKER_IMAGE_VERSION = $(shell date -u +'%Y%m%dt%H%M%S')_$(GIT_HEAD_REV) build/envs.txt: requirements/conda.txt @@ -130,8 +132,8 @@ build/deps.txt: build/envs.txt requirements/*.txt @rm -f build/deps.txt.tmp; @for env_name in $(CONDA_ENV_NAMES); do \ - env_py="${ENV_PREFIX}/$${env_name}/bin/python"; \ - printf "\npip freeze for $${env_name}:\n" >> build/deps.txt.tmp; \ + env_py="$(ENV_PREFIX)/$${env_name}/bin/python"; \ + printf "\n# pip freeze for $${env_name}:\n" >> build/deps.txt.tmp; \ $${env_py} -m pip freeze >> build/deps.txt.tmp; \ printf "\n\n" >> build/deps.txt.tmp; \ done @@ -190,8 +192,8 @@ help: ## Full help message for each task. -.PHONY: fullhelp -fullhelp: +.PHONY: helpverbose +helpverbose: @printf "Available make targets for \033[97m$(PKG_NAME)\033[0m:\n"; @awk '{ \ @@ -232,7 +234,7 @@ fullhelp: .PHONY: clean clean: @for env_name in $(CONDA_ENV_NAMES); do \ - env_py="${ENV_PREFIX}/$${env_name}/bin/python"; \ + env_py="$(ENV_PREFIX)/$${env_name}/bin/python"; \ if [[ -f $${env_py} ]]; then \ $(CONDA_BIN) env remove --name $${env_name} --yes; \ fi; \ @@ -280,8 +282,8 @@ update: build/deps.txt ## Install git pre-push hooks .PHONY: git_hooks git_hooks: - @rm -f "${PWD}/.git/hooks/pre-push" - ln -s "${PWD}/scripts/pre-push-hook.sh" "${PWD}/.git/hooks/pre-push" + @rm -f "$(PWD)/.git/hooks/pre-push" + ln -s "$(PWD)/scripts/pre-push-hook.sh" "$(PWD)/.git/hooks/pre-push" ## -- Integration -- @@ -355,7 +357,7 @@ fmt: src/ test/ -## Shortcut for make fmt lint pylint test +## Shortcut for make fmt lint mypy test .PHONY: check check: fmt lint mypy test @@ -372,27 +374,27 @@ env_subshell: ') -## Usage: "source activate", to deactivate: "deactivate" +## Usage: "source ./activate", to deactivate: "deactivate" .PHONY: activate activate: @echo 'source $(CONDA_ROOT)/etc/profile.d/conda.sh;' @echo 'if [[ -z $$ENV ]]; then' - @echo ' export _env_before_activate_$(DEV_ENV_NAME)=$${ENV};' + @echo ' export _env_before_activate=$${ENV};' @echo 'fi' @echo 'if [[ -z $$PYTHONPATH ]]; then' - @echo ' export _pythonpath_before_activate_$(DEV_ENV_NAME)=$${PYTHONPATH};' + @echo ' export _pythonpath_before_activate=$${PYTHONPATH};' @echo 'fi' @echo 'export ENV=$${ENV-dev};' @echo 'export PYTHONPATH="src/:vendor/:$$PYTHONPATH";' @echo 'conda activate $(DEV_ENV_NAME);' @echo 'function deactivate {' - @echo ' if [[ -z $${_env_before_activate_$(DEV_ENV_NAME)} ]]; then' - @echo ' export ENV=$${_env_before_activate_$(DEV_ENV_NAME)}; ' + @echo ' if [[ -z $${_env_before_activate} ]]; then' + @echo ' export ENV=$${_env_before_activate}; ' @echo ' else' @echo ' unset ENV;' @echo ' fi' - @echo ' if [[ -z $${_pythonpath_before_activate_$(DEV_ENV_NAME)} ]]; then' - @echo ' export PYTHONPATH=$${_pythonpath_before_activate_$(DEV_ENV_NAME)}; ' + @echo ' if [[ -z $${_pythonpath_before_activate} ]]; then' + @echo ' export PYTHONPATH=$${_pythonpath_before_activate}; ' @echo ' else' @echo ' unset PYTHONPATH;' @echo ' fi' @@ -478,6 +480,7 @@ bump_version: build_dists: $(DEV_ENV_PY) setup.py sdist; $(DEV_ENV_PY) setup.py bdist_wheel --python-tag=$(BDIST_WHEEL_PYTHON_TAG); + @rm -rf src/*.egg-info ## Upload sdist and bdist files to pypi @@ -494,14 +497,14 @@ upload_dists: $(DEV_ENV)/bin/twine upload $$($(SDIST_FILE_CMD)) $$($(BDIST_WHEEL_FILE_CMD)); -## Publish on pypi +## bump_version build_dists upload_dists .PHONY: publish publish: bump_version build_dists upload_dists ## Build docker images. Must be run when dependencies are added ## or updated. The main reasons this can fail are: -## 1. No ssh key at $(HOME)/.ssh/${PKG_NAME}_gitlab_runner_id_rsa +## 1. No ssh key at $(HOME)/.ssh/$(PKG_NAME)_gitlab_runner_id_rsa ## (which is needed to install packages from private repos ## and is copied into a temp container during the build). ## 2. Your docker daemon is not running @@ -509,9 +512,9 @@ publish: bump_version build_dists upload_dists ## 4. You're using WSL but didn't do export DOCKER_HOST="tcp://localhost:2375" .PHONY: build_docker build_docker: - @if [[ -f "${RSA_KEY_PATH}" ]]; then \ + @if [[ -f "$(RSA_KEY_PATH)" ]]; then \ docker build \ - --build-arg SSH_PRIVATE_RSA_KEY="$$(cat '${RSA_KEY_PATH}')" \ + --build-arg SSH_PRIVATE_RSA_KEY="$$(cat '$(RSA_KEY_PATH)')" \ --file docker_base.Dockerfile \ --tag $(DOCKER_BASE_IMAGE):$(DOCKER_IMAGE_VERSION) \ --tag $(DOCKER_BASE_IMAGE) \ diff --git a/requirements/conda.txt b/requirements/conda.txt index 329869e..67ba6a0 100644 --- a/requirements/conda.txt +++ b/requirements/conda.txt @@ -35,7 +35,6 @@ # pytest is required in every environment to run the test suite # against the installed modules. -# Unpin when this is released https://github.com/pytest-dev/pytest-cov/pull/240 pytest # needed for mypy coverage report diff --git a/scripts/pre-push-hook.sh b/scripts/pre-push-hook.sh old mode 100644 new mode 100755 diff --git a/scripts/setup_conda_envs.sh b/scripts/setup_conda_envs.sh old mode 100644 new mode 100755 diff --git a/scripts/update_conda_env_deps.sh b/scripts/update_conda_env_deps.sh old mode 100644 new mode 100755 diff --git a/src/pycalver/__init__.py b/src/pycalver/__init__.py index 9e61b25..2818c0a 100644 --- a/src/pycalver/__init__.py +++ b/src/pycalver/__init__.py @@ -1,8 +1,8 @@ # This file is part of the pycalver project # https://gitlab.com/mbarkhau/pycalver # -# Copyright (c) 2018 Manuel Barkhau (mbarkhau@gmail.com) - MIT License +# Copyright (c) 2019 Manuel Barkhau (mbarkhau@gmail.com) - MIT License # SPDX-License-Identifier: MIT -"""PyCalVer: Automatic CalVer Versioning for Python Packages.""" +"""PyCalVer: CalVer for Python Packages.""" __version__ = "v201812.0018" diff --git a/src/pycalver/__main__.py b/src/pycalver/__main__.py index 1e3a9c5..b01e4ea 100644 --- a/src/pycalver/__main__.py +++ b/src/pycalver/__main__.py @@ -2,7 +2,7 @@ # This file is part of the pycalver project # https://gitlab.com/mbarkhau/pycalver # -# Copyright (c) 2018 Manuel Barkhau (mbarkhau@gmail.com) - MIT License +# Copyright (c) 2019 Manuel Barkhau (mbarkhau@gmail.com) - MIT License # SPDX-License-Identifier: MIT """ CLI module for PyCalVer.