bootstrapit update

.gitignore

@@ -46,6 +46,7 @@ pip-delete-this-directory.txt
 # Unit test / coverage reports
 htmlcov/
 mypycov/
+reports/
 .tox/
 .coverage
 .cache
@@ -71,3 +72,4 @@ README.html
 envs.txt
 test_build_logs/*.log
 build/miniconda3.sh
+compat_test/

CONTRIBUTING.md

@@ -160,7 +160,7 @@ Out[4]: '/home/dev/myproject/src/myproject/__init__.py'
 ```
 
 Note that the `PYTHONPATH` has been set up to import modules
-of the project. You can review the definition for `make ipy` 
+of the project. You can review the definition for `make ipy`
 to see how to set up `PYTHONPATH` correctly.
 
 
@@ -215,9 +215,9 @@ projects by reducing the burdon of project setup to a minimum.
     CHANGELOG.md       # short documentation of release history
     LICENSE            # for public libraries (MIT preferred)
 
-    makefile                # main project and environment management file
+    makefile                    # project specific configuration
-    makefile.config.make    # project configuration variables
+                                # variables and make targets
-    makefile.extra.make     # project specific make targets
+    makefile.bootstrapit.make   # bootstrapit make include library
 
     docker_base.Dockerfile  # base image for CI (only conda envs)
     Dockerfile              # image with source of the project
@@ -294,7 +294,7 @@ Dependencies are installed in this order:
  - `development.txt`
  - `integration.txt`
 
-Please review the documentation header at the beginning of each 
+Please review the documentation header at the beginning of each
 `requirements/*.txt` file to determine which file is appropriate
 for the dependency you want to add.
 
@@ -355,7 +355,7 @@ The typical commands used during development are:
 - `make install`: Setup virtual environment
 - `source activate`: Activate virtual environment
 - `make help`: Overview of tasks
-- `make fmt`: Format code 
+- `make fmt`: Format code
 - `make lint`: Linting
 - `make mypy`: Typecheck
 - `make devtest`: Run unittests with dev interpreter against code from `src/`.
@@ -374,7 +374,7 @@ Slightly less common but good to run before doing `git push`.
 
 The base image of the project is `docker_base.Dockerfile` which is
 used to create images that have only the conda virtual environment needed
-to run the project. The CI environment uses the image generated by 
+to run the project. The CI environment uses the image generated by
 `make docker_build`. While this means that the CI setup is simpler and faster,
 as you don't have to build the image for the test run in the CI environment, it does mean that you have to run `make docker_build` every time one of your dependencies is updated.
 
@@ -394,7 +394,7 @@ Solving environment: ...working... done
 conda create --name myproject_py35 python=3.5 ...
 Solving environment: ...working... done
 
-docker push 
+docker push
 ```
 
 As is the case for your local development setup, every version of python
@@ -445,11 +445,11 @@ Please read, view at your leasure:
 Keep in mind, that all of this is about the form of your code, and
 catching common pitfalls or gotchas. None of this releives you of the
 burdon of thinking about your code. The reason to use linters and type
-checking is not to have a tool to make your code correct, but to 
+checking is not to have a tool to make your code correct, but to
 support you to make your code correct.
 
 For now I won't go into the effort of writing yet another style guide.
-Instead, if your code passes `make fmt lint`, then it's acceptable. 
+Instead, if your code passes `make fmt lint`, then it's acceptable.
 Every time you encounter a linting error, consider it as an opportinity
 to learn a best practice and look up the error code.
 

Dockerfile

@@ -10,9 +10,8 @@ ADD README.md README.md
 ADD CHANGELOG.md CHANGELOG.md
 ADD LICENSE LICENSE
 ADD makefile makefile
-ADD makefile.config.make makefile.config.make
+ADD makefile.bootstrapit.make makefile.bootstrapit.make
-ADD makefile.extra.make makefile.extra.make
 
 ENV PYTHONPATH="src/:vendor/"
 
-CMD make test_compat
+CMD make lint test_compat

LICENSE

@@ -1,6 +1,4 @@
-MIT License
+MIT License Copyright (c) 2020 Manuel Barkhau (mbarkhau@gmail.com)
-
-Copyright (c) 2019 Manuel Barkhau (mbarkhau@gmail.com)
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

docker_base.Dockerfile

@@ -32,8 +32,7 @@ RUN if ! test -z "${ENV_SSH_PRIVATE_RSA_KEY}"; then \
 ADD requirements/ requirements/
 ADD scripts/ scripts/
 
-ADD makefile.extra.make makefile.extra.make
+ADD makefile.bootstrapit.make makefile.bootstrapit.make
-ADD makefile.config.make makefile.config.make
 ADD makefile makefile
 
 RUN make install

license.header

@@ -3,7 +3,7 @@ Individual files contain the following tag instead of the full license text.
     This file is part of the pycalver project
     https://gitlab.com/mbarkhau/pycalver
 
-    Copyright (c) 2019 Manuel Barkhau (mbarkhau@gmail.com) - MIT License
+    Copyright (c) 2020 Manuel Barkhau (mbarkhau@gmail.com) - MIT License
     SPDX-License-Identifier: MIT
 
 This enables machine processing of license information based on the SPDX

makefile.bootstrapit.make

@@ -10,7 +10,6 @@ SHELL := /bin/bash
 .DEFAULT_GOAL := help
 .SUFFIXES:
 
-
 PROJECT_DIR := $(notdir $(abspath .))
 
 ifndef DEVELOPMENT_PYTHON_VERSION
@@ -48,7 +47,7 @@ CONDA_ENV_PATHS := \
 # envname/bin/pypy3
 CONDA_ENV_BIN_PYTHON_PATHS := \
 	$(shell echo "$(CONDA_ENV_PATHS)" \
-	| sed 's!\(_py[[:digit:]]\+\)!\1/bin/python!g' \
+	| sed 's!\(_py[[:digit:]]\{1,\}\)!\1/bin/python!g' \
 	| sed 's!\(_pypy2[[:digit:]]\)!\1/bin/pypy!g' \
 	| sed 's!\(_pypy3[[:digit:]]\)!\1/bin/pypy3!g' \
 )
@@ -85,13 +84,13 @@ build/envs.txt: requirements/conda.txt
 	@if [[ ! -f $(CONDA_BIN) ]]; then \
 		echo "installing miniconda ..."; \
 		if [[ $(PLATFORM) == "Linux" ]]; then \
-			curl "https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh" \
+			curl "https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh" --location \
 				> build/miniconda3.sh; \
 		elif [[ $(PLATFORM) == "MINGW64_NT-10.0" ]]; then \
-			curl "https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh" \
+			curl "https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh" --location \
 				> build/miniconda3.sh; \
 		elif [[ $(PLATFORM) == "Darwin" ]]; then \
-			curl "https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh" \
+			curl "https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh" --location \
 				> build/miniconda3.sh; \
 		fi; \
 		bash build/miniconda3.sh -b -p $(CONDA_ROOT); \
@@ -164,7 +163,7 @@ help:
 						helpCommand, helpMessage; \
 					helpMessage = ""; \
 				} \
-			} else if ($$0 ~ /^[a-zA-Z\-\_0-9.]+:/) { \
+			} else if ($$0 ~ /^[a-zA-Z\-\_0-9.\/]+:/) { \
 				helpCommand = substr($$0, 0, index($$0, ":")); \
 				if (helpMessage) { \
 					printf "\033[36m%-20s\033[0m %s\n", \
@@ -182,7 +181,7 @@ help:
 				helpMessage = ""; \
 			} \
 		}' \
-		$(MAKEFILE_LIST)
+		makefile.bootstrapit.make makefile
 
 	@if [[ ! -f $(DEV_ENV_PY) ]]; then \
 	echo "Missing python interpreter at $(DEV_ENV_PY) !"; \
@@ -216,7 +215,7 @@ helpverbose:
 						helpCommand, helpMessage; \
 					helpMessage = ""; \
 				} \
-			} else if ($$0 ~ /^[a-zA-Z\-\_0-9.]+:/) { \
+			} else if ($$0 ~ /^[a-zA-Z\-\_0-9.\/]+:/) { \
 				helpCommand = substr($$0, 0, index($$0, ":")); \
 				if (helpMessage) { \
 					printf "\033[36m%-20s\033[0m %s\n", \
@@ -236,7 +235,7 @@ helpverbose:
 				helpMessage = ""; \
 			} \
 		}' \
-		$(MAKEFILE_LIST)
+		makefile.bootstrapit.make makefile
 
 
 ## -- Project Setup --
@@ -301,11 +300,18 @@ git_hooks:
 ## -- Integration --
 
 
-## Run flake8 linter
+## Run flake8 linter and check for fmt
 .PHONY: lint
 lint:
-	@printf "flake8 ..\n"
+	@printf "isort ..\n"
-	@$(DEV_ENV)/bin/flake8 src/
+	@$(DEV_ENV)/bin/isort \
+		--check-only \
+		--force-single-line-imports \
+		--length-sort \
+		--recursive \
+		--line-width=$(MAX_LINE_LEN) \
+		--project $(PKG_NAME) \
+		src/ test/
 	@printf "\e[1F\e[9C ok\n"
 
 	@printf "sjfmt ..\n"
@@ -317,6 +323,10 @@ lint:
 		src/ test/ 2>&1 | sed "/All done/d" | sed "/left unchanged/d"
 	@printf "\e[1F\e[9C ok\n"
 
+	@printf "flake8 ..\n"
+	@$(DEV_ENV)/bin/flake8 src/
+	@printf "\e[1F\e[9C ok\n"
+
 
 ## Run mypy type checker
 .PHONY: mypy
@@ -326,6 +336,7 @@ mypy:
 	@printf "mypy ....\n"
 	@MYPYPATH=stubs/:vendor/ $(DEV_ENV_PY) -m mypy \
 		--html-report mypycov \
+		--no-error-summary \
 		src/ | sed "/Generated HTML report/d"
 	@printf "\e[1F\e[9C ok\n"
 
@@ -354,15 +365,27 @@ test:
 	@rm -rf "test/__pycache__";
 
 	# First we test the local source tree using the dev environment
-	ENV=$${ENV-dev} PYTHONPATH=src/:vendor/:$$PYTHONPATH \
+	ENV=$${ENV-dev} \
+		PYTHONPATH=src/:vendor/:$$PYTHONPATH \
+		PATH=$(DEV_ENV)/bin:$$PATH \
 		$(DEV_ENV_PY) -m pytest -v \
 		--doctest-modules \
 		--verbose \
 		--cov-report html \
 		--cov-report term \
-		$(shell cd src/ && ls -1 */__init__.py | awk '{ print "--cov "substr($$1,0,index($$1,"/")-1) }') \
+		-k "$${PYTEST_FILTER}" \
+		$(shell cd src/ && ls -1 */__init__.py | awk '{ sub(/\/__init__.py/, "", $$1); print "--cov "$$1 }') \
 		test/ src/;
 
+	# Next we install the package and run the test suite against it.
+
+	IFS=' ' read -r -a env_py_paths <<< "$(CONDA_ENV_BIN_PYTHON_PATHS)"; \
+	for i in $${!env_py_paths[@]}; do \
+		env_py=$${env_py_paths[i]}; \
+		$${env_py} -m pip install --upgrade .; \
+		PYTHONPATH="" ENV=$${ENV-dev} $${env_py} -m pytest test/; \
+	done;
+
 	@rm -rf ".pytest_cache";
 	@rm -rf "src/__pycache__";
 	@rm -rf "test/__pycache__";
@@ -374,11 +397,20 @@ test:
 ## Run code formatter on src/ and test/
 .PHONY: fmt
 fmt:
+	@$(DEV_ENV)/bin/isort \
+		--force-single-line-imports \
+		--length-sort \
+		--recursive \
+		--line-width=$(MAX_LINE_LEN) \
+		--project $(PKG_NAME) \
+		src/ test/;
+
 	@$(DEV_ENV)/bin/sjfmt \
 		--target-version=py36 \
 		--skip-string-normalization \
 		--line-length=$(MAX_LINE_LEN) \
-		 src/ test/
+		src/ test/;
+
 
 
 ## Shortcut for make fmt lint mypy test
@@ -429,7 +461,9 @@ activate:
 ## Drop into an ipython shell with correct env variables set
 .PHONY: ipy
 ipy:
-	@ENV=$${ENV-dev} PYTHONPATH=src/:vendor/:$$PYTHONPATH \
+	@ENV=$${ENV-dev} \
+		PYTHONPATH=src/:vendor/:$$PYTHONPATH \
+		PATH=$(DEV_ENV)/bin:$$PATH \
 		$(DEV_ENV)/bin/ipython
 
 
@@ -439,28 +473,19 @@ devtest:
 	@rm -rf "src/__pycache__";
 	@rm -rf "test/__pycache__";
 
-ifdef FILTER
+	ENV=$${ENV-dev} \
-	ENV=$${ENV-dev} PYTHONPATH=src/:vendor/:$$PYTHONPATH \
+		PYTHONPATH=src/:vendor/:$$PYTHONPATH \
+		PATH=$(DEV_ENV)/bin:$$PATH \
 		$(DEV_ENV_PY) -m pytest -v \
 		--doctest-modules \
 		--no-cov \
+		--durations 5 \
 		--verbose \
 		--capture=no \
 		--exitfirst \
 		--failed-first \
-		-k $(FILTER) \
+		-k "$${PYTEST_FILTER}" \
 		test/ src/;
-else
-	ENV=$${ENV-dev} PYTHONPATH=src/:vendor/:$$PYTHONPATH \
-		$(DEV_ENV_PY) -m pytest -v \
-		--doctest-modules \
-		--no-cov \
-		--verbose \
-		--capture=no \
-		--exitfirst \
-		--failed-first \
-		test/ src/;
-endif
 
 	@rm -rf "src/__pycache__";
 	@rm -rf "test/__pycache__";
@@ -518,7 +543,8 @@ dist_upload:
 
 	$(DEV_ENV)/bin/twine check $$($(SDIST_FILE_CMD));
 	$(DEV_ENV)/bin/twine check $$($(BDIST_WHEEL_FILE_CMD));
-	$(DEV_ENV)/bin/twine upload $$($(SDIST_FILE_CMD)) $$($(BDIST_WHEEL_FILE_CMD));
+	$(DEV_ENV)/bin/twine upload --skip-existing \
+		$$($(SDIST_FILE_CMD)) $$($(BDIST_WHEEL_FILE_CMD));
 
 
 ## bump_version dist_build dist_upload
@@ -552,6 +578,3 @@ docker_build:
 	fi
 
 	docker push $(DOCKER_BASE_IMAGE)
-
-
-

requirements/integration.txt

@@ -7,18 +7,28 @@
 #
 # No dependencies required for production should be listed here.
 
-# oh ffs: https://gitlab.com/pycqa/flake8-docstrings/issues/36
+# http://www.pydocstyle.org/en/4.0.0/release_notes.html
+# Support for Python 2.x and PyPy has been dropped (#340).
 pydocstyle<4
 
+
 flake8
 flake8-bugbear
 flake8-docstrings
 flake8-builtins
 flake8-comprehensions
+flake8-junit-report
 pylint
 mypy
+isort
+
+# http://doc.pytest.org/en/latest/py27-py34-deprecation.html
+# The pytest 4.6 series will be the last to support Python 2.7
+# and 3.4, and is scheduled to be released by mid-2019.
+# pytest 5.0 and onwards will support only Python 3.5+.
 pytest<5.0
 pytest-cov
+pytest-html
 
 readme_renderer[md]
 twine
@@ -26,7 +36,6 @@ twine
 md-toc
 straitjacket
 pycalver
-lib3to6
 
 # needed for mypy coverage report
 lxml

setup.cfg

@@ -10,6 +10,7 @@ disallow_untyped_calls = True
 follow_imports = silent
 strict_optional = True
 ignore_missing_imports = True
+show_error_codes = True
 
 [flake8]
 max-line-length = 100
@@ -27,10 +28,12 @@ ignore =
     E221
     # Multiple spaces after operand
     E222
-    # Multiple spaces before keyword
+    # Multiple spaces after ':'
-    E272
+    E241
     # Spaces around keyword/parameter equals
     E251
+    # Multiple spaces before keyword
+    E272
     # Line too long (B950 is used instead)
     E501
     # Line break before binary op
@@ -45,10 +48,14 @@ ignore =
     D102
     # Missing docstring in public function
     D103
+    # Missing docstring in magic method
+    # D105
     # Missing docstring on __init__
     D107
     # No blank lines allowed after function docstring
     D202
+    # First line should end in a period
+    D400
     # First line should be in imperative mood
     D401
 select = A,AAA,D,C,E,F,W,H,B,D212,D404,D405,D406,B901,B950