bootstrapit updates

.gitignore

@@ -73,3 +73,147 @@ envs.txt
 test_build_logs/*.log
 build/miniconda3.sh
 compat_test/
+
+# https://github.com/github/gitignore/blob/master/Global/Linux.gitignore
+*~
+
+# temporary files which can be created if a process still has a handle open of a deleted file
+.fuse_hidden*
+
+# KDE directory preferences
+.directory
+
+# Linux trash folder which might appear on any partition or disk
+.Trash-*
+
+# .nfs files are created when an open file is removed but is still being accessed
+.nfs*
+
+# https://github.com/github/gitignore/blob/master/Global/Windows.gitignore
+# Windows thumbnail cache files
+Thumbs.db
+Thumbs.db:encryptable
+ehthumbs.db
+ehthumbs_vista.db
+
+# Dump file
+*.stackdump
+
+# Folder config file
+[Dd]esktop.ini
+
+# Recycle Bin used on file shares
+$RECYCLE.BIN/
+
+# Windows Installer files
+*.cab
+*.msi
+*.msix
+*.msm
+*.msp
+
+# Windows shortcuts
+*.lnk
+
+# https://raw.githubusercontent.com/github/gitignore/master/Global/macOS.gitignore
+# General
+.DS_Store
+.AppleDouble
+.LSOverride
+
+# Icon must end with two \r
+Icon
+
+
+# Thumbnails
+._*
+
+# Files that might appear in the root of a volume
+.DocumentRevisions-V100
+.fseventsd
+.Spotlight-V100
+.TemporaryItems
+.Trashes
+.VolumeIcon.icns
+.com.apple.timemachine.donotpresent
+
+# Directories potentially created on remote AFP share
+.AppleDB
+.AppleDesktop
+Network Trash Folder
+Temporary Items
+.apdisk
+
+
+# https://raw.githubusercontent.com/github/gitignore/master/Global/JetBrains.gitignore
+# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider
+# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
+
+# User-specific stuff
+.idea/**/workspace.xml
+.idea/**/tasks.xml
+.idea/**/usage.statistics.xml
+.idea/**/dictionaries
+.idea/**/shelf
+
+# Generated files
+.idea/**/contentModel.xml
+
+# Sensitive or high-churn files
+.idea/**/dataSources/
+.idea/**/dataSources.ids
+.idea/**/dataSources.local.xml
+.idea/**/sqlDataSources.xml
+.idea/**/dynamic.xml
+.idea/**/uiDesigner.xml
+.idea/**/dbnavigator.xml
+
+# Gradle
+.idea/**/gradle.xml
+.idea/**/libraries
+
+# Gradle and Maven with auto-import
+# When using Gradle or Maven with auto-import, you should exclude module files,
+# since they will be recreated, and may cause churn.  Uncomment if using
+# auto-import.
+# .idea/artifacts
+# .idea/compiler.xml
+# .idea/jarRepositories.xml
+# .idea/modules.xml
+# .idea/*.iml
+# .idea/modules
+# *.iml
+# *.ipr
+
+# CMake
+cmake-build-*/
+
+# Mongo Explorer plugin
+.idea/**/mongoSettings.xml
+
+# File-based project format
+*.iws
+
+# IntelliJ
+out/
+
+# mpeltonen/sbt-idea plugin
+.idea_modules/
+
+# JIRA plugin
+atlassian-ide-plugin.xml
+
+# Cursive Clojure plugin
+.idea/replstate.xml
+
+# Crashlytics plugin (for Android Studio and IntelliJ)
+com_crashlytics_export_strings.xml
+crashlytics.properties
+crashlytics-build.properties
+fabric.properties
+
+# Editor-based Rest Client
+.idea/httpRequests
+
+# Android studio 3.1+ serialized cache file
+.idea/caches/build_file_checksums.ser

.gitlab-ci.yml

@@ -13,9 +13,13 @@ unit:
     - make test_compat
   coverage: '/^(TOTAL|src).*?(\d+\%)$/'
   artifacts:
+    reports:
+      junit:
+        - reports/flake8.xml
+        - reports/pytest.xml
     paths:
-      - htmlcov/
+      - reports/htmlcov/
-      - mypycov/
+      - reports/mypycov/
   allow_failure: false
 
 
@@ -24,8 +28,8 @@ pages:
   script:
     - mkdir -p public/cov
     - mkdir -p public/mypycov
-    - cp -r htmlcov/* public/cov/
+    - cp -r reports/htmlcov/* public/cov/
-    - cp -r mypycov/* public/mypycov/
+    - cp -r reports/mypycov/* public/mypycov/
   artifacts:
     paths:
       - public

makefile.bootstrapit.make

@@ -300,9 +300,9 @@ git_hooks:
 ## -- Integration --
 
 
-## Run flake8 linter and check for fmt
+## Run isort with --check-only
-.PHONY: lint
+.PHONY: lint_isort
-lint:
+lint_isort:
 	@printf "isort ..\n"
 	@$(DEV_ENV)/bin/isort \
 		--check-only \
@@ -314,6 +314,10 @@ lint:
 		src/ test/
 	@printf "\e[1F\e[9C ok\n"
 
+
+## Run sjfmt with --check
+.PHONY: lint_sjfmt
+lint_sjfmt:
 	@printf "sjfmt ..\n"
 	@$(DEV_ENV)/bin/sjfmt \
 		--target-version=py36 \
@@ -323,19 +327,36 @@ lint:
 		src/ test/ 2>&1 | sed "/All done/d" | sed "/left unchanged/d"
 	@printf "\e[1F\e[9C ok\n"
 
+
+## Run flake8
+.PHONY: lint_flake8
+lint_flake8:
+	@rm -f reports/flake8*;
+	@mkdir -p "reports/";
+
 	@printf "flake8 ..\n"
-	@$(DEV_ENV)/bin/flake8 src/
+	@$(DEV_ENV)/bin/flake8 src/ --tee --output-file reports/flake8.txt || exit 0;
+	@$(DEV_ENV)/bin/flake8_junit reports/flake8.txt reports/flake8.xml >> /dev/null;
+	@$(DEV_ENV_PY) scripts/exit_0_if_empty.py reports/flake8.txt;
+
 	@printf "\e[1F\e[9C ok\n"
 
 
+## Run flake8 linter and check for fmt
+.PHONY: lint
+lint: lint_isort lint_sjfmt lint_flake8
+
+
 ## Run mypy type checker
 .PHONY: mypy
 mypy:
 	@rm -rf ".mypy_cache";
+	@rm -rf "reports/mypycov";
+	@mkdir -p "reports/";
 
 	@printf "mypy ....\n"
 	@MYPYPATH=stubs/:vendor/ $(DEV_ENV_PY) -m mypy \
-		--html-report mypycov \
+		--html-report reports/mypycov \
 		--no-error-summary \
 		src/ | sed "/Generated HTML report/d"
 	@printf "\e[1F\e[9C ok\n"
@@ -344,16 +365,10 @@ mypy:
 ## Run pylint. Should not break the build yet
 .PHONY: pylint
 pylint:
-	@printf "pylint ..\n";
+	@mkdir -p "reports/";
-	@$(DEV_ENV)/bin/pylint --jobs=4 --output-format=colorized --score=no \
-		 --disable=C0103,C0301,C0330,C0326,C0330,C0411,R0903,W1619,W1618,W1203 \
-		 --extension-pkg-whitelist=ujson,lxml,PIL,numpy,pandas,sklearn,pyblake2 \
-		 src/
-	@$(DEV_ENV)/bin/pylint --jobs=4 --output-format=colorized --score=no \
-		 --disable=C0103,C0111,C0301,C0330,C0326,C0330,C0411,R0903,W1619,W1618,W1203 \
-		 --extension-pkg-whitelist=ujson,lxml,PIL,numpy,pandas,sklearn,pyblake2 \
-		 test/
 
+	@printf "pylint ..\n";
+	@$(DEV_ENV)/bin/pylint --rcfile=setup.cfg src/ test/
 	@printf "\e[1F\e[9C ok\n"
 
 
@@ -363,6 +378,9 @@ test:
 	@rm -rf ".pytest_cache";
 	@rm -rf "src/__pycache__";
 	@rm -rf "test/__pycache__";
+	@rm -rf "reports/testcov/";
+	@rm -f "reports/pytest*";
+	@mkdir -p "reports/";
 
 	# First we test the local source tree using the dev environment
 	ENV=$${ENV-dev} \
@@ -371,8 +389,10 @@ test:
 		$(DEV_ENV_PY) -m pytest -v \
 		--doctest-modules \
 		--verbose \
-		--cov-report html \
+		--cov-report "html:reports/testcov/" \
 		--cov-report term \
+		--html=reports/pytest/index.html \
+		--junitxml reports/pytest.xml \
 		-k "$${PYTEST_FILTER}" \
 		$(shell cd src/ && ls -1 */__init__.py | awk '{ sub(/\/__init__.py/, "", $$1); print "--cov "$$1 }') \
 		test/ src/;
@@ -394,9 +414,9 @@ test:
 ## -- Helpers --
 
 
-## Run code formatter on src/ and test/
+## Run import sorting on src/ and test/
-.PHONY: fmt
+.PHONY: fmt_isort
-fmt:
+fmt_isort:
 	@$(DEV_ENV)/bin/isort \
 		--force-single-line-imports \
 		--length-sort \
@@ -405,6 +425,10 @@ fmt:
 		--project $(PKG_NAME) \
 		src/ test/;
 
+
+## Run code formatter on src/ and test/
+.PHONY: fmt_sjfmt
+fmt_sjfmt:
 	@$(DEV_ENV)/bin/sjfmt \
 		--target-version=py36 \
 		--skip-string-normalization \
@@ -412,10 +436,14 @@ fmt:
 		src/ test/;
 
 
+## Run code formatters
+.PHONY: fmt
+fmt: fmt_isort fmt_sjfmt
 
-## Shortcut for make fmt lint mypy test
+
+## Shortcut for make fmt lint mypy devtest test
 .PHONY: check
-check:  fmt lint mypy test
+check: fmt lint mypy devtest test
 
 
 ## Start subshell with environ variables set.

requirements/integration.txt

@@ -20,7 +20,9 @@ flake8-comprehensions
 flake8-junit-report
 pylint
 mypy
-isort
+# pylint doesn't support isort>=5 for now
+#   https://github.com/PyCQA/pylint/issues/3722
+isort<5
 
 # http://doc.pytest.org/en/latest/py27-py34-deprecation.html
 # The pytest 4.6 series will be the last to support Python 2.7
@@ -28,7 +30,9 @@ isort
 # pytest 5.0 and onwards will support only Python 3.5+.
 pytest<5.0
 pytest-cov
-pytest-html
+# https://github.com/pytest-dev/pytest-html/blob/master/CHANGES.rst
+# pytest-html 2.0+ doesn't support python2.7
+pytest-html<2.0
 
 readme_renderer[md]
 twine

setup.cfg

@@ -12,6 +12,13 @@ strict_optional = True
 ignore_missing_imports = True
 show_error_codes = True
 
+[tool:isort]
+known_third_party = pathlib2
+force_single_line = True
+length_sort = True
+line_length = 100
+
+
 [flake8]
 max-line-length = 100
 max-complexity = 10
@@ -48,6 +55,8 @@ ignore =
     D102
     # Missing docstring in public function
     D103
+    # Missing docstring in public package
+    # D104
     # Missing docstring in magic method
     # D105
     # Missing docstring on __init__
@@ -97,3 +106,39 @@ README.md =
     img.shields.io/static/v1.svg?label=PyCalVer&message={version}&color=blue
     Successfully installed pycalver-{pep440_version}
     pycalver, version {version}
+
+
+[tool:pylint]
+score = yes
+
+# pylint-ignore only works with jobs = 1
+jobs = 1
+
+# Set the output format. Available formats are text, parseable, colorized,
+# msvs (visual studio) and html. You can also give a reporter class, eg
+# mypackage.mymodule.MyReporterClass.
+output-format = colorized
+
+# Maximum number of locals for function / method body
+max-locals = 20
+
+# Maximum number of arguments for function / method
+max-args = 8
+
+good-names = logger,i,ex
+
+# These are packages that are implemented as c extensions and
+# which pylint cannot do introspection on.
+extension-pkg-whitelist = numpy,pandas,lxml,PIL,sklearn,pyblake2
+
+notes=TODO,FIXME,XXX,SLOW,BUG
+
+# https://pylint.pycqa.org/en/stable/technical_reference/features.html
+disable =
+    bad-continuation,
+    bad-whitespace,
+    logging-fstring-interpolation,
+    missing-docstring,
+    missing-module-docstring,
+    missing-class-docstring,
+    missing-function-docstring,