From ae35ae3c1ce6e65679fcd8b81d9fcba6e975c837 Mon Sep 17 00:00:00 2001 From: Manuel Barkhau Date: Wed, 26 Aug 2020 21:49:01 +0000 Subject: [PATCH] project maintenance --- .github/workflows/ci.yml | 73 ++++++++++++++++++++++++++++++ .gitlab-ci.yml | 23 ++++++++-- CONTRIBUTING.md | 87 ++++++++++++++++++++++++------------ Dockerfile | 3 +- README.md | 10 +++-- docker_base.Dockerfile | 13 +++++- makefile.bootstrapit.make | 46 +++++++++---------- requirements/development.txt | 8 ++-- requirements/integration.txt | 5 +-- setup.cfg | 14 ++++-- src/pycalver/cli.py | 15 +++---- 11 files changed, 213 insertions(+), 84 deletions(-) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..0d06d38 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,73 @@ +name: CI + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + + build-ubuntu: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - name: Cache Conda Envs + uses: actions/cache@v2 + with: + path: | + ~/miniconda3 + build/*.txt + key: ${{ runner.OS }}-conda-cache-${{ hashFiles('requirements/*.txt', 'setup.py', 'makefile*') }} + restore-keys: | + ${{ runner.OS }}-conda-cache-${{ hashFiles('requirements/*.txt', 'setup.py', 'makefile*') }} + + - name: make conda + run: + if [[ -e build/envs.txt ]]; then touch build/envs.txt; fi; + if [[ -e build/deps.txt ]]; then touch build/deps.txt; fi; + make conda + + - name: make lint + run: make lint + + - name: make mypy + run: make mypy + + - name: make test + run: make test + + build-macos: + + runs-on: macos-latest + + steps: + - uses: actions/checkout@v2 + + - name: Cache Conda Envs + uses: actions/cache@v2 + with: + path: | + ~/miniconda3 + build/*.txt + key: ${{ runner.OS }}-conda-cache-${{ hashFiles('requirements/*.txt', 'setup.py', 'makefile*') }} + restore-keys: | + ${{ runner.OS }}-conda-cache-${{ hashFiles('requirements/*.txt', 'setup.py', 'makefile*') }} + + - name: make conda + run: + if [[ -e build/envs.txt ]]; then touch build/envs.txt; fi; + if [[ -e build/deps.txt ]]; then touch build/deps.txt; fi; + make conda + + - name: make lint + run: make lint + + - name: make mypy + run: make mypy + + - name: make test + run: make test diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index eca86c3..0977023 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,23 +3,40 @@ stages: - build -unit: +lint: stage: test image: registry.gitlab.com/mbarkhau/pycalver/base script: - make lint - make mypy + artifacts: + reports: + junit: + - reports/flake8.xml + paths: + - reports/mypycov/ + allow_failure: false + + +unit: + # NOTE: Resource_group is conservative and can be disabled + # for simple tests. It should be enabled if the tests + # need exclusive access to some resource common external + # resource. This will prevent multiple pipelines from + # running concurrently. + # resource_group: test-unit + stage: test + image: registry.gitlab.com/mbarkhau/pycalver/base + script: - make test - make test_compat coverage: '/^(TOTAL|src).*?(\d+\%)$/' artifacts: reports: junit: - - reports/flake8.xml - reports/pytest.xml paths: - reports/testcov/ - - reports/mypycov/ allow_failure: false diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index af1a8f3..08dee2e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -18,7 +18,7 @@ - [These are not used on production, or staging, only](#these-are-not-used-on-production-or-staging-only) - [on development machines and the CI environment.](#on-development-machines-and-the-ci-environment) - [These are the requirements produced for specific builds. They can be](#these-are-the-requirements-produced-for-specific-builds-they-can-be) -- [used to debug version compatatbility issues . They are generated](#used-to-debug-version-compatatbility-issues-they-are-generated) +- [used to debug version compatibility issues . They are generated](#used-to-debug-version-compatibility-issues-they-are-generated) - [using pip freeze](#using-pip-freeze) - [Vendoring](#vendoring) - [Development](#development) @@ -41,11 +41,9 @@ steps and not encounter any errors: 1. `git clone ` 2. `cd ` - 3. `make install` + 3. `make conda` 4. `# get some coffee` - 5. `make lint` - 6. `make test` - 7. `make serve` + 5. `make fmt lint mypy test` If you as a new contributor encounter any errors, then please create an issue report and you will already have made a great contribution! @@ -65,7 +63,7 @@ connect to remote servers. If this is the case, you should make sure that your ssh keys are available in `${HOME}/.ssh`, or you will have to do `ssh-keygen` and install the generated public key to host system. If this is not done, `pip install` will fail -to install these dependencies from your private repositiories with +to install these dependencies from your private repositories with an error like this ```shell @@ -96,7 +94,7 @@ Cloning Git repository git@../group/project.git to project $ cd project dev@host:~/project -$ make install +$ make conda Solving environment: ... ``` @@ -183,7 +181,7 @@ Out[2]: '/home/dev/myproject/src/myproject/__init__.py' ## Project Types These guidelines written for different kinds of projects, each of -which is ideally: small, focosued and reusable. These projects can be: +which is ideally: small, focused and reusable. These projects can be: 1. Services: Projects which are deployed and run continuously. 2. Libraries: Projects which are not deployed by themselves but @@ -192,7 +190,7 @@ which is ideally: small, focosued and reusable. These projects can be: developers and admins. The choices made here are intended to make it easy to start new -projects by reducing the burdon of project setup to a minimum. +projects by reducing the burden of project setup to a minimum. ## Project Layout @@ -201,7 +199,7 @@ projects by reducing the burdon of project setup to a minimum. vendor/ # vendored dependencies stubs/ # mypy .pyi stub files test/ # pytest test files (files begin with test_) - scripts/ # miscalenious scripts used deployment and ops + scripts/ # miscellaneous scripts used deployment and ops requirements/ # dependency metadata files docs/ # documentation source files @@ -237,11 +235,11 @@ requirements/vendor.txt # installed via pip from pypi to vendor/ # These are not used on production, or staging, only # on development machines and the CI environment. -requirements/development.txt # useful packgages for development/debugging +requirements/development.txt # useful packages for development/debugging requirements/integration.txt # used for linting/testing/packaging # These are the requirements produced for specific builds. They can be -# used to debug version compatatbility issues. They are generated +# used to debug version compatibility issues. They are generated # using make freeze requirements/20190214t212403_freeze.txt ``` @@ -256,7 +254,7 @@ When adding a new dependency please consider: name without a version specifier. With this as the default, the project remains up to date in terms of security fixes and other library improvements. -- Some packages consider some of their dependancies to be optional, +- Some packages consider some of their dependencies to be optional, in which case you will have to specify their transitive dependencies. - Only specify/pin/freeze a specific (older) version if there are @@ -267,11 +265,11 @@ When adding a new dependency please consider: One argument against this approach is the issue of rogue package maintainers. A package maintainer might release a new version which -you automatically install using `make update`, and this new code opens +you automatically install using `make conda`, and this new code opens a back door or proceeds to send data from your production system to a random server on the internet. -The only prodection pypi or conda-forge have against this is to remove +The only protection pypi or conda-forge have against this is to remove packages that are reported to them. If you are paranoid, you could start pinning dependencies to older versions, for which you feel comfortable that any issues would have been noticed. This is only a @@ -305,16 +303,16 @@ Choose a file: or cannot be downloaded from pypi (such as openjdk or node). - `pypi.txt` is for dependencies on python packages, be they from pypi or git repositories. -- `vendor.txt` is appropriate for pure python libaries which are +- `vendor.txt` is appropriate for pure python libraries which are written using mypy. This allows the mypy type checker to work with types defined in other packages -After adding a new dependency, you can run `make update` +After adding a new dependency, you can run `make conda` ```shell (myproject_py36) dev@host:~/myproject -$ make update +$ make conda Solving environment: done Downloading and Extracting Packages @@ -322,10 +320,10 @@ requests-2.19.1 | 94 KB conda-forge ... ``` -Normally make update only does something if you update one of the -`requirements/*.txt` files has changed. If you know a dependency -was updated, and `make update` is not having an effect, you can -force the update using `make force update`. +Normally `make conda` only does something if you update one of the +`requirements/*.txt` files. If you know a dependency was updated, and +`make conda` is not having an effect, you can force the update using +`make force conda`. ### Vendoring @@ -336,7 +334,7 @@ then it's not required. There are a few reasons to vendor a dependency: -1. You want the source to be easilly accessible in your development +1. You want the source to be easily accessible in your development tools. For example mypy can access the types of vendored projects. 2. You don't trust the maintainer of a dependency, and want to review any updates using git diff. @@ -352,7 +350,7 @@ contribute to the upstream project when possible. The typical commands used during development are: -- `make install`: Setup virtual environment +- `make conda`: Setup virtual environment - `source activate`: Activate virtual environment - `make help`: Overview of tasks - `make fmt`: Format code @@ -370,6 +368,37 @@ Slightly less common but good to run before doing `git push`. want to trigger dozens of CI builds to debug a tricky issue. +### Packaging/Distribution + +Publishing a package is done using twine, for which you will need to somehow supply your pypi authentication. I haven't tried [keyring-support](https://twine.readthedocs.io/en/latest/#keyring-support), but your welcome to give that a shot. Another way is to add an entry in your `~/.pypirc`: + + +``` +[distutils] +index-servers = + pypi + pypi-legacy + +[pypi] +repository = https://pypi.org +username = Your.Username +password = secret + +[pypi-legacy] +repository = https://upload.pypi.org/legacy/ +username = Your.Username +password = secret +``` + +Creating a new package and uploading it to pypi will typically involve these steps: + +- `make lint mypy test`: Run CI locally, in case you don't trust the CI setup. +- `make bump_version`: Increment project wide version numbers and tag commit. +- `git push`: Push the bumped version. +- `make dist_build`: Create the .whl and .tar.gz distributions. +- `make dist_upload`: Publish to pypi. + + ### Docker The base image of the project is `docker_base.Dockerfile` which is @@ -406,7 +435,7 @@ to trim this down. ### Documentation -Documentation is written in Github Flavoured Markdown. Typora is +Documentation is written in Github Flavored Markdown. Typora is decent cross platform editor. TODO: `make doc` @@ -426,7 +455,7 @@ contradictory to each other in places), reading them will give you a good overview of how different people think about structuring their code in order to minimize common pitfalls. -Please read, view at your leasure: +Please read, view at your leisure: - Talks: - [Stop Writing Classes by Jack Diederich](https://www.youtube.com/watch?v=o9pEzgHorH0) @@ -443,14 +472,14 @@ Please read, view at your leasure: - https://github.com/google/styleguide/blob/gh-pages/pyguide.md Keep in mind, that all of this is about the form of your code, and -catching common pitfalls or gotchas. None of this releives you of the -burdon of thinking about your code. The reason to use linters and type +catching common pitfalls or gotchas. None of this relieves you of the +burden of thinking about your code. The reason to use linters and type checking is not to have a tool to make your code correct, but to support you to make your code correct. For now I won't go into the effort of writing yet another style guide. Instead, if your code passes `make fmt lint`, then it's acceptable. -Every time you encounter a linting error, consider it as an opportinity +Every time you encounter a linting error, consider it as an opportunity to learn a best practice and look up the error code. [^1]: Linux, MacOS and [WSL](https://docs.microsoft.com/en-us/windows/wsl/install-win10) diff --git a/Dockerfile b/Dockerfile index 1c4fe7f..b0119f4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,8 @@ ADD CHANGELOG.md CHANGELOG.md ADD LICENSE LICENSE ADD makefile makefile ADD makefile.bootstrapit.make makefile.bootstrapit.make +ADD scripts/exit_0_if_empty.py scripts/exit_0_if_empty.py ENV PYTHONPATH="src/:vendor/" -CMD make lint test_compat +CMD make lint mypy test_compat diff --git a/README.md b/README.md index 4bf7197..9b22567 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,8 @@ Project/Repo: Code Quality/CI: -[![Build Status][build_img]][build_ref] +[![GitHub Build Status][github_build_img]][github_build_ref] +[![GitLab Build Status][gitlab_build_img]][gitlab_build_ref] [![Type Checked with mypy][mypy_img]][mypy_ref] [![Code Coverage][codecov_img]][codecov_ref] [![Code Style: sjfmt][style_img]][style_ref] @@ -974,8 +975,11 @@ artifact of a package, eg. a `.whl` file. [cookiecutter_ref]: https://cookiecutter.readthedocs.io -[build_img]: https://gitlab.com/mbarkhau/pycalver/badges/master/pipeline.svg -[build_ref]: https://gitlab.com/mbarkhau/pycalver/pipelines +[github_build_img]: https://github.com/mbarkhau/pycalver/workflows/CI/badge.svg +[github_build_ref]: https://github.com/mbarkhau/pycalver/actions?query=workflow%3ACI + +[gitlab_build_img]: https://gitlab.com/mbarkhau/pycalver/badges/master/pipeline.svg +[gitlab_build_ref]: https://gitlab.com/mbarkhau/pycalver/pipelines [codecov_img]: https://gitlab.com/mbarkhau/pycalver/badges/master/coverage.svg [codecov_ref]: https://mbarkhau.gitlab.io/pycalver/cov diff --git a/docker_base.Dockerfile b/docker_base.Dockerfile index aa54ae8..bff005d 100644 --- a/docker_base.Dockerfile +++ b/docker_base.Dockerfile @@ -9,6 +9,11 @@ FROM registry.gitlab.com/mbarkhau/bootstrapit/env_builder AS builder +# gcc required for cmarkgfm on python3.8 +# https://github.com/theacodes/cmarkgfm/issues/22 +RUN apt-get update +RUN apt-get install -y gcc + RUN mkdir /root/.ssh/ && \ ssh-keyscan gitlab.com >> /root/.ssh/known_hosts && \ ssh-keyscan registry.gitlab.com >> /root/.ssh/known_hosts @@ -35,7 +40,13 @@ ADD scripts/ scripts/ ADD makefile.bootstrapit.make makefile.bootstrapit.make ADD makefile makefile -RUN make install +# install envs (relatively stable) +ADD requirements/conda.txt requirements/conda.txt +RUN make build/envs.txt + +# install python package dependencies (change more often) +ADD requirements/ requirements/ +RUN make conda RUN rm -f /root/.ssh/id_rsa diff --git a/makefile.bootstrapit.make b/makefile.bootstrapit.make index 1b8b0b3..fada680 100644 --- a/makefile.bootstrapit.make +++ b/makefile.bootstrapit.make @@ -13,7 +13,7 @@ SHELL := /bin/bash PROJECT_DIR := $(notdir $(abspath .)) ifndef DEVELOPMENT_PYTHON_VERSION - DEVELOPMENT_PYTHON_VERSION := python=3.6 + DEVELOPMENT_PYTHON_VERSION := python=3.8 endif ifndef SUPPORTED_PYTHON_VERSIONS @@ -186,18 +186,18 @@ help: @if [[ ! -f $(DEV_ENV_PY) ]]; then \ echo "Missing python interpreter at $(DEV_ENV_PY) !"; \ - echo "You problably want to install first:"; \ + echo "You problably want to first setup the virtual environments:"; \ echo ""; \ - echo " make install"; \ + echo " make conda"; \ echo ""; \ exit 0; \ fi @if [[ ! -f $(CONDA_BIN) ]]; then \ echo "No conda installation found!"; \ - echo "You problably want to install first:"; \ + echo "You problably want to first setup the virtual environments:"; \ echo ""; \ - echo " make install"; \ + echo " make conda"; \ echo ""; \ exit 0; \ fi @@ -281,14 +281,9 @@ force: rm -rf vendor/__pycache__/ -## Setup python virtual environments -.PHONY: install -install: build/deps.txt - - -## Update dependencies (pip install -U ...) -.PHONY: update -update: build/deps.txt +## Create/Update python virtual environments +.PHONY: conda +conda: build/deps.txt ## Install git pre-push hooks @@ -306,10 +301,8 @@ git_hooks: lint_isort: @printf "isort ...\n" @$(DEV_ENV)/bin/isort \ - --check-only \ - --force-single-line-imports \ - --length-sort \ --recursive \ + --check-only \ --line-width=$(MAX_LINE_LEN) \ --project $(MODULE_NAME) \ src/ test/ @@ -317,8 +310,8 @@ lint_isort: ## Run sjfmt with --check -.PHONY: lint_sjfmt -lint_sjfmt: +.PHONY: lint_fmt +lint_fmt: @printf "sjfmt ...\n" @$(DEV_ENV)/bin/sjfmt \ --target-version=py36 \ @@ -355,15 +348,15 @@ lint_pylint: ## Run pylint-ignore --update-ignorefile. -.PHONY: pylint_update_ignorefile -pylint_update_ignorefile: +.PHONY: pylint_ignore +pylint_ignore: $(DEV_ENV)/bin/pylint-ignore --rcfile=setup.cfg \ src/ test/ --update-ignorefile ## Run flake8 linter and check for fmt .PHONY: lint -lint: lint_isort lint_sjfmt lint_flake8 lint_pylint +lint: lint_isort lint_fmt lint_flake8 lint_pylint ## Run mypy type checker @@ -420,15 +413,10 @@ test: @rm -rf "test/__pycache__"; -## -- Helpers -- - - ## Run import sorting on src/ and test/ .PHONY: fmt_isort fmt_isort: @$(DEV_ENV)/bin/isort \ - --force-single-line-imports \ - --length-sort \ --recursive \ --line-width=$(MAX_LINE_LEN) \ --project $(MODULE_NAME) \ @@ -450,6 +438,9 @@ fmt_sjfmt: fmt: fmt_isort fmt_sjfmt +## -- Helpers -- + + ## Shortcut for make fmt lint mypy devtest test .PHONY: check check: fmt lint mypy devtest test @@ -480,6 +471,7 @@ activate: @echo 'export ENV=$${ENV-dev};' @echo 'export PYTHONPATH="src/:vendor/:$$PYTHONPATH";' @echo 'conda activate $(DEV_ENV_NAME);' + @echo 'function deactivate {' @echo ' if [[ -z $${_env_before_activate} ]]; then' @echo ' export ENV=$${_env_before_activate}; ' @@ -491,6 +483,8 @@ activate: @echo ' else' @echo ' unset PYTHONPATH;' @echo ' fi' + @echo ' unset _env_before_activate;' + @echo ' unset _pythonpath_before_activate;' @echo ' conda deactivate;' @echo '};' diff --git a/requirements/development.txt b/requirements/development.txt index 43661f8..d2d0505 100644 --- a/requirements/development.txt +++ b/requirements/development.txt @@ -14,10 +14,10 @@ pudb # https://pypi.org/project/q/ q -# rich is used for more readable tracebacks during development -# https://github.com/nir0s/backtrace -# https://rich.readthedocs.io/en/latest/traceback.html -rich +# pretty-traceback manipulates Python tracebacks to make +# them more readable. +# https://pypi.org/project/pretty-traceback/ +pretty-traceback # Py-Spy: A sampling profiler for Python programs. # https://github.com/benfred/py-spy diff --git a/requirements/integration.txt b/requirements/integration.txt index 6139963..1395383 100644 --- a/requirements/integration.txt +++ b/requirements/integration.txt @@ -18,8 +18,7 @@ flake8-docstrings flake8-builtins flake8-comprehensions flake8-junit-report -pylint -pylint-ignore +pylint-ignore>=2020.1013 mypy # pylint doesn't support isort>=5 for now # https://github.com/PyCQA/pylint/issues/3722 @@ -39,7 +38,7 @@ readme_renderer[md] twine md-toc -straitjacket +straitjacket>=v202008.1016 pycalver lib3to6 diff --git a/setup.cfg b/setup.cfg index d4e6d69..a914932 100644 --- a/setup.cfg +++ b/setup.cfg @@ -9,6 +9,7 @@ check_untyped_defs = True disallow_untyped_calls = True follow_imports = silent strict_optional = True +error_summary = False ignore_missing_imports = True show_error_codes = True warn_unreachable = True @@ -18,10 +19,9 @@ warn_redundant_casts = True [tool:isort] -known_third_party = pathlib2 +known_third_party = click,pathlib2 force_single_line = True length_sort = True -line_length = 100 [flake8] @@ -117,9 +117,9 @@ README.md = [tool:pylint] score = no +reports = no -# pylint-ignore only works with jobs = 1 -jobs = 1 +jobs = 4 # Set the output format. Available formats are text, parseable, colorized, # msvs (visual studio) and html. You can also give a reporter class, eg @@ -140,6 +140,12 @@ extension-pkg-whitelist = numpy,pandas,lxml,PIL,sklearn,pyblake2 notes=TODO,FIXME,XXX,SLOW,BUG +# similarities/duplicaition checker +min-similarity-lines=4 +ignore-comments=yes +ignore-docstrings=yes +ignore-imports=yes + # https://pylint.pycqa.org/en/stable/technical_reference/features.html disable = bad-continuation, diff --git a/src/pycalver/cli.py b/src/pycalver/cli.py index c9b5731..0316382 100755 --- a/src/pycalver/cli.py +++ b/src/pycalver/cli.py @@ -9,7 +9,6 @@ CLI module for PyCalVer. Provided subcommands: show, test, init, bump """ -import os import sys import typing as typ import logging @@ -25,16 +24,12 @@ from . import version _VERBOSE = 0 -# To enable pretty tracebacks: -# echo "export ENABLE_RICH_TB=1;" >> ~/.bashrc -if os.environ.get('ENABLE_RICH_TB') == '1': - try: - import rich.traceback +try: + import pretty_traceback - rich.traceback.install() - except ImportError: - # don't fail just because of missing dev library - pass + pretty_traceback.install() +except ImportError: + pass # no need to fail because of missing dev dependency click.disable_unicode_literals_warning = True