From f1e17562b8cfc53a2ac11ce0619aa5fe7b04b6d2 Mon Sep 17 00:00:00 2001 From: Manuel Barkhau Date: Sun, 19 Jul 2020 13:58:33 +0000 Subject: [PATCH] bootstrapit updates --- .gitignore | 144 +++++++++++++++++++++++++++++++++++ .gitlab-ci.yml | 12 ++- makefile.bootstrapit.make | 68 ++++++++++++----- requirements/integration.txt | 8 +- setup.cfg | 45 +++++++++++ 5 files changed, 251 insertions(+), 26 deletions(-) diff --git a/.gitignore b/.gitignore index 665f68a..c899f50 100644 --- a/.gitignore +++ b/.gitignore @@ -73,3 +73,147 @@ envs.txt test_build_logs/*.log build/miniconda3.sh compat_test/ + +# https://github.com/github/gitignore/blob/master/Global/Linux.gitignore +*~ + +# temporary files which can be created if a process still has a handle open of a deleted file +.fuse_hidden* + +# KDE directory preferences +.directory + +# Linux trash folder which might appear on any partition or disk +.Trash-* + +# .nfs files are created when an open file is removed but is still being accessed +.nfs* + +# https://github.com/github/gitignore/blob/master/Global/Windows.gitignore +# Windows thumbnail cache files +Thumbs.db +Thumbs.db:encryptable +ehthumbs.db +ehthumbs_vista.db + +# Dump file +*.stackdump + +# Folder config file +[Dd]esktop.ini + +# Recycle Bin used on file shares +$RECYCLE.BIN/ + +# Windows Installer files +*.cab +*.msi +*.msix +*.msm +*.msp + +# Windows shortcuts +*.lnk + +# https://raw.githubusercontent.com/github/gitignore/master/Global/macOS.gitignore +# General +.DS_Store +.AppleDouble +.LSOverride + +# Icon must end with two \r +Icon + + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + + +# https://raw.githubusercontent.com/github/gitignore/master/Global/JetBrains.gitignore +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/**/usage.statistics.xml +.idea/**/dictionaries +.idea/**/shelf + +# Generated files +.idea/**/contentModel.xml + +# Sensitive or high-churn files +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml +.idea/**/dbnavigator.xml + +# Gradle +.idea/**/gradle.xml +.idea/**/libraries + +# Gradle and Maven with auto-import +# When using Gradle or Maven with auto-import, you should exclude module files, +# since they will be recreated, and may cause churn. Uncomment if using +# auto-import. +# .idea/artifacts +# .idea/compiler.xml +# .idea/jarRepositories.xml +# .idea/modules.xml +# .idea/*.iml +# .idea/modules +# *.iml +# *.ipr + +# CMake +cmake-build-*/ + +# Mongo Explorer plugin +.idea/**/mongoSettings.xml + +# File-based project format +*.iws + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +# Editor-based Rest Client +.idea/httpRequests + +# Android studio 3.1+ serialized cache file +.idea/caches/build_file_checksums.ser \ No newline at end of file diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7d7ae98..df0c428 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -13,9 +13,13 @@ unit: - make test_compat coverage: '/^(TOTAL|src).*?(\d+\%)$/' artifacts: + reports: + junit: + - reports/flake8.xml + - reports/pytest.xml paths: - - htmlcov/ - - mypycov/ + - reports/htmlcov/ + - reports/mypycov/ allow_failure: false @@ -24,8 +28,8 @@ pages: script: - mkdir -p public/cov - mkdir -p public/mypycov - - cp -r htmlcov/* public/cov/ - - cp -r mypycov/* public/mypycov/ + - cp -r reports/htmlcov/* public/cov/ + - cp -r reports/mypycov/* public/mypycov/ artifacts: paths: - public diff --git a/makefile.bootstrapit.make b/makefile.bootstrapit.make index a3eadb6..d8f791f 100644 --- a/makefile.bootstrapit.make +++ b/makefile.bootstrapit.make @@ -300,9 +300,9 @@ git_hooks: ## -- Integration -- -## Run flake8 linter and check for fmt -.PHONY: lint -lint: +## Run isort with --check-only +.PHONY: lint_isort +lint_isort: @printf "isort ..\n" @$(DEV_ENV)/bin/isort \ --check-only \ @@ -314,6 +314,10 @@ lint: src/ test/ @printf "\e[1F\e[9C ok\n" + +## Run sjfmt with --check +.PHONY: lint_sjfmt +lint_sjfmt: @printf "sjfmt ..\n" @$(DEV_ENV)/bin/sjfmt \ --target-version=py36 \ @@ -323,19 +327,36 @@ lint: src/ test/ 2>&1 | sed "/All done/d" | sed "/left unchanged/d" @printf "\e[1F\e[9C ok\n" + +## Run flake8 +.PHONY: lint_flake8 +lint_flake8: + @rm -f reports/flake8*; + @mkdir -p "reports/"; + @printf "flake8 ..\n" - @$(DEV_ENV)/bin/flake8 src/ + @$(DEV_ENV)/bin/flake8 src/ --tee --output-file reports/flake8.txt || exit 0; + @$(DEV_ENV)/bin/flake8_junit reports/flake8.txt reports/flake8.xml >> /dev/null; + @$(DEV_ENV_PY) scripts/exit_0_if_empty.py reports/flake8.txt; + @printf "\e[1F\e[9C ok\n" +## Run flake8 linter and check for fmt +.PHONY: lint +lint: lint_isort lint_sjfmt lint_flake8 + + ## Run mypy type checker .PHONY: mypy mypy: @rm -rf ".mypy_cache"; + @rm -rf "reports/mypycov"; + @mkdir -p "reports/"; @printf "mypy ....\n" @MYPYPATH=stubs/:vendor/ $(DEV_ENV_PY) -m mypy \ - --html-report mypycov \ + --html-report reports/mypycov \ --no-error-summary \ src/ | sed "/Generated HTML report/d" @printf "\e[1F\e[9C ok\n" @@ -344,16 +365,10 @@ mypy: ## Run pylint. Should not break the build yet .PHONY: pylint pylint: - @printf "pylint ..\n"; - @$(DEV_ENV)/bin/pylint --jobs=4 --output-format=colorized --score=no \ - --disable=C0103,C0301,C0330,C0326,C0330,C0411,R0903,W1619,W1618,W1203 \ - --extension-pkg-whitelist=ujson,lxml,PIL,numpy,pandas,sklearn,pyblake2 \ - src/ - @$(DEV_ENV)/bin/pylint --jobs=4 --output-format=colorized --score=no \ - --disable=C0103,C0111,C0301,C0330,C0326,C0330,C0411,R0903,W1619,W1618,W1203 \ - --extension-pkg-whitelist=ujson,lxml,PIL,numpy,pandas,sklearn,pyblake2 \ - test/ + @mkdir -p "reports/"; + @printf "pylint ..\n"; + @$(DEV_ENV)/bin/pylint --rcfile=setup.cfg src/ test/ @printf "\e[1F\e[9C ok\n" @@ -363,6 +378,9 @@ test: @rm -rf ".pytest_cache"; @rm -rf "src/__pycache__"; @rm -rf "test/__pycache__"; + @rm -rf "reports/testcov/"; + @rm -f "reports/pytest*"; + @mkdir -p "reports/"; # First we test the local source tree using the dev environment ENV=$${ENV-dev} \ @@ -371,8 +389,10 @@ test: $(DEV_ENV_PY) -m pytest -v \ --doctest-modules \ --verbose \ - --cov-report html \ + --cov-report "html:reports/testcov/" \ --cov-report term \ + --html=reports/pytest/index.html \ + --junitxml reports/pytest.xml \ -k "$${PYTEST_FILTER}" \ $(shell cd src/ && ls -1 */__init__.py | awk '{ sub(/\/__init__.py/, "", $$1); print "--cov "$$1 }') \ test/ src/; @@ -394,9 +414,9 @@ test: ## -- Helpers -- -## Run code formatter on src/ and test/ -.PHONY: fmt -fmt: +## Run import sorting on src/ and test/ +.PHONY: fmt_isort +fmt_isort: @$(DEV_ENV)/bin/isort \ --force-single-line-imports \ --length-sort \ @@ -405,6 +425,10 @@ fmt: --project $(PKG_NAME) \ src/ test/; + +## Run code formatter on src/ and test/ +.PHONY: fmt_sjfmt +fmt_sjfmt: @$(DEV_ENV)/bin/sjfmt \ --target-version=py36 \ --skip-string-normalization \ @@ -412,10 +436,14 @@ fmt: src/ test/; +## Run code formatters +.PHONY: fmt +fmt: fmt_isort fmt_sjfmt -## Shortcut for make fmt lint mypy test + +## Shortcut for make fmt lint mypy devtest test .PHONY: check -check: fmt lint mypy test +check: fmt lint mypy devtest test ## Start subshell with environ variables set. diff --git a/requirements/integration.txt b/requirements/integration.txt index bbc5ed2..1b360d3 100644 --- a/requirements/integration.txt +++ b/requirements/integration.txt @@ -20,7 +20,9 @@ flake8-comprehensions flake8-junit-report pylint mypy -isort +# pylint doesn't support isort>=5 for now +# https://github.com/PyCQA/pylint/issues/3722 +isort<5 # http://doc.pytest.org/en/latest/py27-py34-deprecation.html # The pytest 4.6 series will be the last to support Python 2.7 @@ -28,7 +30,9 @@ isort # pytest 5.0 and onwards will support only Python 3.5+. pytest<5.0 pytest-cov -pytest-html +# https://github.com/pytest-dev/pytest-html/blob/master/CHANGES.rst +# pytest-html 2.0+ doesn't support python2.7 +pytest-html<2.0 readme_renderer[md] twine diff --git a/setup.cfg b/setup.cfg index 7da0807..1bd5bf7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -12,6 +12,13 @@ strict_optional = True ignore_missing_imports = True show_error_codes = True +[tool:isort] +known_third_party = pathlib2 +force_single_line = True +length_sort = True +line_length = 100 + + [flake8] max-line-length = 100 max-complexity = 10 @@ -48,6 +55,8 @@ ignore = D102 # Missing docstring in public function D103 + # Missing docstring in public package + # D104 # Missing docstring in magic method # D105 # Missing docstring on __init__ @@ -97,3 +106,39 @@ README.md = img.shields.io/static/v1.svg?label=PyCalVer&message={version}&color=blue Successfully installed pycalver-{pep440_version} pycalver, version {version} + + +[tool:pylint] +score = yes + +# pylint-ignore only works with jobs = 1 +jobs = 1 + +# Set the output format. Available formats are text, parseable, colorized, +# msvs (visual studio) and html. You can also give a reporter class, eg +# mypackage.mymodule.MyReporterClass. +output-format = colorized + +# Maximum number of locals for function / method body +max-locals = 20 + +# Maximum number of arguments for function / method +max-args = 8 + +good-names = logger,i,ex + +# These are packages that are implemented as c extensions and +# which pylint cannot do introspection on. +extension-pkg-whitelist = numpy,pandas,lxml,PIL,sklearn,pyblake2 + +notes=TODO,FIXME,XXX,SLOW,BUG + +# https://pylint.pycqa.org/en/stable/technical_reference/features.html +disable = + bad-continuation, + bad-whitespace, + logging-fstring-interpolation, + missing-docstring, + missing-module-docstring, + missing-class-docstring, + missing-function-docstring,