diff --git a/.gitignore b/.gitignore index 88a9cc1..f78cf18 100644 --- a/.gitignore +++ b/.gitignore @@ -29,9 +29,6 @@ var/ .pytest_cache/ .ipynb_checkpoints/ -# source dirs have to be explicitly added -src/ - # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..8451f06 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,33 @@ +stages: + - test + - build + + +unit: + stage: test + image: registry.gitlab.com/mbarkhau/pycalver/base:latest + script: + - make lint + - make mypy + - make test + coverage: '/TOTAL.*?(\d+\%)/' + artifacts: + paths: + - htmlcov/ + tags: + - docker + allow_failure: false + + +pages: + stage: build + script: + - mkdir -p public/cov + - cp htmlcov/* public/cov/ + artifacts: + paths: + - public + tags: + - docker + only: + - master diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..c6b05e3 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,5 @@ +# Changelog for https://gitlab.com/mbarkhau/pycalver + +## v201809.0001-alpha + + - Initial release diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e69de29..9b2aee8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -0,0 +1,413 @@ +# Contributing + + + +[](TOC) + +- [Contributing](#contributing) + - [Introduction](#introduction) + - [Setup](#setup) + - [Setup SSH keys](#setup-ssh-keys) + - [Setup Virtual Environments](#setup-virtual-environments) + - [Project Types](#project-types) + - [Project Layout](#project-layout) + - [Dependency Management](#dependency-management) +- [These are not used on production, or staging, only](#these-are-not-used-on-production-or-staging-only) +- [on development machines and the CI environment.](#on-development-machines-and-the-ci-environment) +- [These are the requirements produced for specific builds. They can be](#these-are-the-requirements-produced-for-specific-builds-they-can-be) +- [used to debug version compatatbility issues . They are generated](#used-to-debug-version-compatatbility-issues-they-are-generated) +- [using pip freeze](#using-pip-freeze) + - [Vendoring](#vendoring) + - [Development](#development) + - [Linting](#linting) + - [Type Checking](#type-checking) + - [Documentation](#documentation) + - [Setup to run docker](#setup-to-run-docker) + - [PyCharm](#pycharm) + - [Sublime Text](#sublime-text) + - [Best Practices](#best-practices) + +[](TOC) + + +## Introduction + +Friction for new contributors should be as low as possible. Ideally a +new contributor, starting any unix[^1] system can go through these +steps and not encounter any errors: + + 1. `git clone ` + 2. `cd ` + 3. `make install` + 4. `# get some coffee` + 5. `make lint` + 6. `make test` + 7. `make serve` + +If you as a new contributor encounter any errors, then please create +an issue report and you will already have made a great contribution! + + +## Setup + +The development workflow described here is documented based on a Unix +environment. Hopefully this will reduce discrepancies between +development and production systems. + + +### Setup SSH keys + +Projects which depend on private repositories require ssh to +connect to remote servers. If this is the case, you should make +sure that your ssh keys are available in `${HOME}/.ssh`, or you +will have to do `ssh-keygen` and install the generated public +key to host system. If this is not done, `pip install` will fail +to install these dependencies from your private repositiories with +an error like this + +```shell +Downloading/unpacking git+git://...git +Cloning Git repository git:// + +Permission denied (publickey). + +fatal: The remote end hung up unexpectedly +---------------------------------------- +Command /usr/local/bin/git clone ... failed with error code 128 +``` + + +### Setup Virtual Environments + +The first setup can take a while, since it will install miniconda and +download lots of dependencies for the first time. If you would like to +know more about conda, there is a good article written by Gergely +Szerovay: https://medium.freecodecamp.org/85f155f4353c + +```shell +dev@host:~ +$ git clone git@../group/project.git +Cloning Git repository git@../group/project.git to project +... + +$ cd project + +dev@host:~/project +$ make install +Solving environment: +... +``` + +This will do quite a few things. + +1. Install miniconda3, if it isn't already installed. It checks + the path `$HOME/miniconda3` for an existing installation +2. Creates python virtual environments for all supported python + versions of this project. +3. Installs application and development dependencies to the + environments. +4. Installs vendored dependencies into `vendor/` + +If installation was successful, you should be able to at least +run the linter (assuming previous developers have a bare minimum +of diligence). + +```console +$ make lint +flake8 .. ok +mypy .... ok +doc ..... ok +``` + +If this is the first time conda has been installed on your +system, you'll probably want to enable the `conda` command: + +``` +$ echo ". ${HOME}/miniconda3/etc/profile.d/conda.sh" >> ~/.bashrc +$ conda --version +conda 4.5.11 +``` + +You can also activate the default virtual environment as follows. + + +```shell +(myproject_py36) dev@host:~/myproject +$ conda env activate myproject_py36 +/home/dev/miniconda3/envs/myproject_py36/bin/python + +$ ipython +Python 3.6.6 | packaged by conda-forge | (default, Jul 26 2018, 09:53:17) +t Type 'copyright', 'credits' or 'license' for more information +IPython 6.5.0 -- An enhanced Interactive Python. Type '?' for help. + +In [1]: +``` + + +Note however, that this invocation does not have the correct +`PYTHONPATH` set up to import modules of the project. You can +review the definition for ``make ipy`` to see how to set up +`PYTHONPATH` correctly. + + +```shell +$ make ipy +Python 3.6.6 |Anaconda, Inc.| (default, Jun 28 2018, 17:14:51) +Type 'copyright', 'credits' or 'license' for more information +IPython 6.5.0 -- An enhanced Interactive Python. Type '?' for help. + +In [1]: import myproject + +In [2]: myproject.__file__ +Out[2]: '/mnt/c/Users/ManuelBarkhau/myproject/src/myproject/__init__.py' +``` + + +## Project Types + +These guidelines written for different kinds of projects, each of +which is ideally: small, focosued and reusable. These projects can be: + + 1. Services: Projects which are deployed and run continuously. + 2. Libraries: Projects which are not deployed by themselves but + installed and used by others. + 3. CLI Tools: Projects which are installed and mainly used by + developers and admins. + +The choices made here are intended to make it easy to start new +projects by reducing the burdon of project setup to a minimum. + + +## Project Layout + + src/ # source code of project + vendor/ # vendored dependencies + stubs/ # mypy .pyi stub files + test/ # pytest test files (files begin with test_) + scripts/ # miscalenious scripts used deployment and ops + + requirements/ # dependency metadata files + docs/ # documentation source files + data/ # fixtures for unit tests and db initialization + + setup.py # main python package metadata + setup.cfg # misc python tooling configuration + + README.md # project overview and status + CONTRIBUTING.md # guide for developers + CHANGELOG.md # for public libraries + LICENSE # for public libraries (MIT preferred) + + makefile # main project and environment management file + + +### Dependency Management + + +Dependencies are managed using a set of requirements/\*.txt files. You +only need to know about this if you want to add or change a dependency. + + +```shell +requirements/conda.txt # installed via conda from main or conda-forge +requirements/pypi.txt # installed via pip from pypi to virutal environments +requirements/vendor.txt # installed via pip from pypi to vendor/ + +# These are not used on production, or staging, only +# on development machines and the CI environment. +requirements/development.txt # useful packgages for development/debugging +requirements/integration.txt # used for linting/testing/packaging + +# These are the requirements produced for specific builds. They can be +# used to debug version compatatbility issues . They are generated +# using pip freeze +requirements/build-0123.freeze +``` + + +When adding a new dependency please consider: + +- Only specify direct dependencies of the project, not transitive + dependencies of other projects. These are installed via their + dependency declarations. +- The default specifier for a package should be only its name without + a version specifier. With this as the default, the project remains + up to date in terms of security fixes and other library + improvements. +- Some packages consider some of their dependancies to be optional, in + which case you will have to specify their transitive dependencies + +- Only specify/pin/freeze a specific (older) version if there are + known issues, or your project requires features from an unstable + (alpha/beta) version of the package. Each pinned version should + document why it was pinned, so that it can later be determined if + the issue has been resolved in the meantime. + +One argument against this approach is the issue of rogue package +maintainers. A package maintainer might release a new version which +you automatically install using `make update`, and this new code opens +a back door or proceeds to send data from your production system to a +random server on the internet. + +The only prodection pypi or conda-forge have against this is to remove +packages that are reported to them. If you are paranoid, you could +start pinning dependencies to older versions, for which you feel +comfortable that any issues would have been noticed. This is only a +half measure however, since the issues may not be noticed even after +months. + +Ultimately, if data breaches are a concern you should talk to your +network admin about firewall rules and if data loss is a concern you +should review your backup policy. + +Further Reading: + https://hackernoon.com/building-a-botnet-on-pypi-be1ad280b8d6 + https://python-security.readthedocs.io/packages.html + +Dependencies are installed in this order: + + - ``conda.txt`` + - ``pypi.txt`` + - ``vendor.txt`` + - ``development.txt`` + - ``integration.txt`` + +Please review the documentation header at the beginning of each file +to determine which file is appropriate for the dependency you want to +add. + +Choose a file: + +- ``conda.txt`` is appropriate for non python packages and packages + which would require compilation if they were downloaded from pypi. +- ``pypi.txt`` is for dependencies on python packages, be they from + pypi or git repositories. +- ``vendor.txt`` is appropriate for pure python libaries which are + written using mypy. This allows the mypy type checker to work with + types defined in other packages + +After adding a new dependency, you can run ``make update`` + + +```shell +(myproject_py36) dev@host:~/myproject +$ make update +Solving environment: done + +Downloading and Extracting Packages +requests-2.19.1 | 94 KB conda-forge +... +``` + + +### Vendoring + +Vendored dependencies are usually committed to git, but if you +trust the package maintainer and the installation via vendor.txt, +then it's not required. + +There are a few reasons to vendor a dependency: + +1. You want the source to be easilly accessible in your development + tools. For example mypy can access the types of vendored projects. +2. You don't trust the maintainer of a dependency, and want to review + any updates using git diff. +3. There is no maintainer or downloadable package, so your only option + is to download it into a local directory. For example you may want to + use some of the modules from https://github.com/TheAlgorithms/Python + +If you do vendor a dependency, avoid local modifications, instead +contribute to the upstream project when possible. + + +## Development + +TODO: document development tasks like lint, type checking in a +platform independent way, ideally they work with PyCharm. Until +then, these are platform agnostic commands that have to be +entered manually. + + +### Linting + + +```shell +flake8 src/ +sjfmt --py36 src/ +``` + + +### Type Checking + + +TODO: This is left open, until the mypy setup is complete + +```shell +mypy src/ +pytest test/ +``` + + +### Documentation + + +Documentation is written in Github Flavoured Markdown. Typora is +decent cross platform editor. + +TODO: `make doc` + +### Setup to run docker + + +TODO: + + +### PyCharm + + +TODO: Expand how to set editor, possibly by sharing editor config files? + +Recoomended plugins: + +https://plugins.jetbrains.com/plugin/10563-black-pycharm +https://plugins.jetbrains.com/plugin/7642-save-actions + + +### Sublime Text + + +https://github.com/jgirardet/sublack + + +## Best Practices + +While not all practices linked here are followed (they are +contradictory to each other in places), reading them will give you a +good overview of how different people think about structuring their +code in order to minimize common pitfalls. + +Please read, view at your leasure: + + - https://www.python.org/dev/peps/pep-0008/ + - https://github.com/amontalenti/elements-of-python-style + - https://github.com/google/styleguide/blob/gh-pages/pyguide.md + - https://www.youtube.com/watch?v=o9pEzgHorH0 + - https://www.youtube.com/watch?v=OSGv2VnC0go + - https://www.youtube.com/watch?v=wf-BqAjZb8M + +Keep in mind, that all of this is about the form of your code, and +catching common pitfalls or gotchas. None of this releives you of the +burdon of thinking about your code. The reason to use linters and type +checking is not to make the code correct, but to help you make your +code correct. + +For now I won't go into the effort of writing yet another style guide. +Instead, if your code passes `make lint`, then it's acceptable. Every +time you encounter a linting error, consider it as an opportinity to +learn a best practice. + +[^1]: Linux, MacOS and [WSL](https://docs.microsoft.com/en-us/windows/wsl/install-win10) diff --git a/LICENSE b/LICENSE index ee7dd80..2a943c6 100644 --- a/LICENSE +++ b/LICENSE @@ -6,16 +6,15 @@ Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: +copies of the Software, and to permit persons to whom the Software is furnished +to do so, subject to the following conditions: -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS +OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF +OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/docker_base.Dockerfile b/docker_base.Dockerfile new file mode 100644 index 0000000..a0e2fdb --- /dev/null +++ b/docker_base.Dockerfile @@ -0,0 +1,94 @@ +# Stages: +# alpine_base : Common base image, both for the builder and for the final image. +# This contains only minimal dependencies required in both cases +# for miniconda and the makefile. +# builder : stage in which the conda envrionment is created +# and dependencies are installed +# final : the final image containing only the required environment files, +# and none of the infrastructure required to generate them. + +FROM frolvlad/alpine-glibc AS alpine_base + +ENV LC_ALL=C.UTF-8 +ENV LANG=C.UTF-8 +ENV LANGUAGE en_US.UTF-8 + +ENV CONDA_DIR /opt/conda +ENV PATH $CONDA_DIR/bin:$PATH +ENV SHELL /bin/bash + +RUN apk add --no-cache bash make sed grep gawk curl git bzip2 unzip + +CMD [ "/bin/bash" ] + +FROM alpine_base AS builder + +RUN apk add --no-cache ca-certificates openssh-client openssh-keygen + +ENV MINICONDA_VER latest +ENV MINICONDA Miniconda3-$MINICONDA_VER-Linux-x86_64.sh +ENV MINICONDA_URL https://repo.continuum.io/miniconda/$MINICONDA + +RUN curl -L ${MINICONDA_URL} --silent -o miniconda3.sh && \ + /bin/bash miniconda3.sh -f -b -p $CONDA_DIR && \ + rm miniconda3.sh && \ + /opt/conda/bin/conda clean -tipsy && \ + ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \ + echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc && \ + echo "conda activate base" >> ~/.bashrc && \ + conda update --all --yes && \ + conda config --set auto_update_conda False + +# Project specific files only from here on forward + +RUN mkdir /root/.ssh/ && \ + ssh-keyscan gitlab.com >> /root/.ssh/known_hosts && \ + ssh-keyscan registry.gitlab.com >> /root/.ssh/known_hosts + +ARG SSH_PRIVATE_RSA_KEY +ENV ENV_SSH_PRIVATE_RSA_KEY=${SSH_PRIVATE_RSA_KEY} + +# Write private key and generate public key +RUN if [[ $ENV_SSH_PRIVATE_RSA_KEY ]]; then \ + echo -n "-----BEGIN RSA PRIVATE KEY-----" >> /root/.ssh/id_rsa && \ + echo -n ${ENV_SSH_PRIVATE_RSA_KEY} \ + | sed 's/-----BEGIN RSA PRIVATE KEY-----//' \ + | sed 's/-----END RSA PRIVATE KEY-----//' \ + | sed 's/ /\n/g' \ + >> /root/.ssh/id_rsa && \ + echo -n "-----END RSA PRIVATE KEY-----" >> /root/.ssh/id_rsa && \ + chmod 600 /root/.ssh/* && \ + ssh-keygen -y -f /root/.ssh/id_rsa > /root/.ssh/id_rsa.pub; \ + fi + +ADD requirements/ requirements/ +ADD scripts/ scripts/ + +ADD makefile.extra.make makefile.extra.make +ADD makefile.config.make makefile.config.make +ADD makefile makefile + +RUN make install + +RUN rm /root/.ssh/id_rsa +# Deleting pkgs implies that `conda install` +# will at have to pull all packages again. +RUN conda clean --all --yes +# Conda docs say that it is not safe to delete pkgs +# because there may be symbolic links, so we verify +# first that there are no such links. +RUN find -L /opt/conda/envs/ -type l | grep "/opt/conda/pkgs" || exit 0 + +# The conda install is not usable after this RUN command. Since +# we only need /opt/conda/envs/ anyway, this shouldn't be an issue. +RUN conda clean --all --yes && \ + ls -d /opt/conda/* | grep -v envs | xargs rm -rf && \ + find /opt/conda/ -name "*.exe" | xargs rm -rf && \ + find /opt/conda/ -name "__pycache__" | xargs rm -rf && \ + rm -rf /opt/conda/pkgs/ + + +FROM alpine_base + +COPY --from=builder /opt/conda/ /opt/conda/ +COPY --from=builder /vendor/ /vendor diff --git a/license.header b/license.header index 2628705..dce09d9 100644 --- a/license.header +++ b/license.header @@ -1,9 +1,9 @@ Individual files contain the following tag instead of the full license text. This file is part of the pycalver project - https://github.com/mbarkhau/pycalver + https://gitlab.com/mbarkhau/pycalver - (C) 2018 Manuel Barkhau (@mbarkhau) + Copyright (c) 2018 Manuel Barkhau (@mbarkhau) - MIT License SPDX-License-Identifier: MIT This enables machine processing of license information based on the SPDX diff --git a/makefile b/makefile new file mode 100644 index 0000000..5b473b2 --- /dev/null +++ b/makefile @@ -0,0 +1,425 @@ +# Helpful Links + +# http://clarkgrubb.com/makefile-style-guide +# https://explainshell.com +# https://stackoverflow.com/questions/448910 +# https://shiroyasha.svbtle.com/escape-sequences-a-quick-guide-1 + +MAKEFLAGS += --warn-undefined-variables +SHELL := /bin/bash +.SHELLFLAGS := -O extglob -eo pipefail -c +.DEFAULT_GOAL := help +.SUFFIXES: + +-include makefile.config.make + +PROJECT_DIR := $(notdir $(abspath .)) + +ifndef MODULE_SRC_PATH + MODULE_SRC_PATH := $(notdir $(abspath .)) +endif + +ifndef DEVELOPMENT_PYTHON_VERSION + DEVELOPMENT_PYTHON_VERSION := python=3.6 +endif + +ifndef SUPPORTED_PYTHON_VERSIONS + SUPPORTED_PYTHON_VERSIONS := $(DEVELOPMENT_PYTHON_VERSION) +endif + +PKG_NAME := $(PACKAGE_NAME) +MODULE_SRC_PATH = src/$(PKG_NAME)/ + +# TODO (mb 2018-09-23): Support for bash on windows +# perhaps we need to install conda using this +# https://repo.continuum.io/miniconda/Miniconda3-latest-Windows-x86_64.exe +PLATFORM = $(shell uname -s) + +# miniconda is shared between projects +CONDA_ROOT := $(shell if [[ -d /opt/conda ]]; then echo "/opt/conda"; else echo "$$HOME/miniconda3"; fi;) +CONDA_BIN := $(CONDA_ROOT)/bin/conda + +ENV_PREFIX := $(CONDA_ROOT)/envs + +DEV_ENV_NAME := \ + $(subst py,$(PKG_NAME)_py,$(subst .,,$(subst =,,$(subst thon,,$(DEVELOPMENT_PYTHON_VERSION))))) + +CONDA_ENV_NAMES := \ + $(subst py,$(PKG_NAME)_py,$(subst .,,$(subst =,,$(subst thon,,$(SUPPORTED_PYTHON_VERSIONS))))) + +CONDA_ENV_PATHS := \ + $(subst py,${ENV_PREFIX}/$(PKG_NAME)_py,$(subst .,,$(subst =,,$(subst thon,,$(SUPPORTED_PYTHON_VERSIONS))))) + + +# default version for development +DEV_ENV := $(ENV_PREFIX)/$(DEV_ENV_NAME) +DEV_ENV_PY := $(DEV_ENV)/bin/python + + +build/envs.txt: requirements/conda.txt + @mkdir -p build/ + + @if [[ ! -f $(CONDA_BIN) ]]; then \ + if [[ $(PLATFORM) == "Linux" ]]; then \ + echo "installing miniconda ..."; \ + curl "https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh" \ + -O build/miniconda3.sh; \ + fi + if [[ $(PLATFORM) == "MINGW64_NT-10.0" ]]; then \ + curl "https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh" \ + -O build/miniconda3.sh; \ + fi + if [[ $(PLATFORM) == "Darwin" ]]; then \ + curl "https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh" \ + -O build/miniconda3.sh; \ + fi + bash build/miniconda3.sh -b -p $(CONDA_ROOT); \ + rm build/miniconda3.sh; \ + fi + + rm -f build/envs.txt.tmp; + + @SUPPORTED_PYTHON_VERSIONS="$(SUPPORTED_PYTHON_VERSIONS)" \ + CONDA_ENV_NAMES="$(CONDA_ENV_NAMES)" \ + CONDA_ENV_PATHS="$(CONDA_ENV_PATHS)" \ + CONDA_BIN="$(CONDA_BIN)" \ + bash scripts/setup_conda_envs.sh; + + $(CONDA_BIN) env list \ + | grep $(PKG_NAME) \ + | rev | cut -d " " -f1 \ + | rev | sort >> build/envs.txt.tmp; + + mv build/envs.txt.tmp build/envs.txt; + + +build/deps.txt: build/envs.txt requirements/*.txt + @mkdir -p build/ + + @SUPPORTED_PYTHON_VERSIONS="$(SUPPORTED_PYTHON_VERSIONS)" \ + CONDA_ENV_NAMES="$(CONDA_ENV_NAMES)" \ + CONDA_ENV_PATHS="$(CONDA_ENV_PATHS)" \ + CONDA_BIN="$(CONDA_BIN)" \ + bash scripts/update_conda_env_deps.sh; + + @echo "updating $(DEV_ENV_NAME) development deps ..."; + + @$(DEV_ENV_PY) -m pip install \ + --disable-pip-version-check --upgrade \ + --requirement=requirements/integration.txt; + + @$(DEV_ENV_PY) -m pip install \ + --disable-pip-version-check --upgrade \ + --requirement=requirements/development.txt; + + @echo "updating local vendor dep copies ..."; + + @$(DEV_ENV_PY) -m pip install \ + --upgrade --disable-pip-version-check \ + --no-deps --target=./vendor \ + --requirement=requirements/vendor.txt; + + @rm -f build/deps.txt.tmp; + + @for env_name in $(CONDA_ENV_NAMES); do \ + env_py="${ENV_PREFIX}/$${env_name}/bin/python"; \ + printf "\npip freeze for $${env_name}:\n" >> build/deps.txt.tmp; \ + $${env_py} -m pip freeze >> build/deps.txt.tmp; \ + printf "\n\n" >> build/deps.txt.tmp; \ + done + + @mv build/deps.txt.tmp build/deps.txt + + +# Add the following 'help' target to your Makefile +# And add help text after each target name starting with '\#\#' +# A category can be added with @category + +## This help message +.PHONY: help +help: + @printf "Available make targets for \033[97m$(PKG_NAME)\033[0m:\n"; + + @awk '{ \ + if ($$0 ~ /^.PHONY: [a-zA-Z\-\_0-9]+$$/) { \ + helpCommand = substr($$0, index($$0, ":") + 2); \ + if (helpMessage) { \ + printf "\033[36m%-20s\033[0m %s\n", \ + helpCommand, helpMessage; \ + helpMessage = ""; \ + } \ + } else if ($$0 ~ /^##/) { \ + if (helpMessage) { \ + helpMessage = helpMessage"\n "substr($$0, 3); \ + } else { \ + helpMessage = substr($$0, 3); \ + } \ + } else { \ + if (helpMessage) { \ + print "\n "helpMessage"\n" \ + } \ + helpMessage = ""; \ + } \ + }' \ + $(MAKEFILE_LIST) + + @if [[ ! -f $(DEV_ENV_PY) ]]; then \ + echo "Missing python interpreter at $(DEV_ENV_PY) !"; \ + echo "You problably want to install first:"; \ + echo ""; \ + echo " make install"; \ + echo ""; \ + exit 0; \ + fi + + @if [[ ! -f $(CONDA_BIN) ]]; then \ + echo "No conda installation found!"; \ + echo "You problably want to install first:"; \ + echo ""; \ + echo " make install"; \ + echo ""; \ + exit 0; \ + fi + + +## -- Project Setup -- + + +## Delete conda envs and cache 💩 +.PHONY: clean +clean: + @for env_name in $(CONDA_ENV_NAMES); do \ + env_py="${ENV_PREFIX}/$${env_name}/bin/python"; \ + if [[ -f $${env_py} ]]; then \ + $(CONDA_BIN) env remove --name $${env_name} --yes; \ + fi; \ + done + + rm -f build/envs.txt + rm -f build/deps.txt + rm -rf vendor/ + rm -rf .mypy_cache/ + rm -rf .pytest_cache/ + rm -rf __pycache__/ + rm -rf src/__pycache__/ + rm -rf vendor/__pycache__/ + @printf "\n setup/update completed ✨ 🍰 ✨ \n\n" + + +## Force update of dependencies +## (this removes makefile markers) +.PHONY: force +force: + rm -f build/envs.txt + rm -f build/deps.txt + rm -rf vendor/ + rm -rf .mypy_cache/ + rm -rf .pytest_cache/ + rm -rf __pycache__/ + rm -rf src/__pycache__/ + rm -rf vendor/__pycache__/ + + +## Setup python virtual environments +.PHONY: install +install: build/deps.txt + + +## Update dependencies (pip install -U ...) +.PHONY: update +update: build/deps.txt + + +## Install git pre-push hooks +.PHONY: git_hooks +git_hooks: + @rm -f "${PWD}/.git/hooks/pre-push" + ln -s "${PWD}/scripts/pre-push-hook.sh" "${PWD}/.git/hooks/pre-push" + + + +# TODO make target to publish on pypi +# .PHONY: publish +# publish: +# echo "Not Implemented" + + +## -- Development -- + + +## Run code formatter on src/ and test/ +.PHONY: fmt +fmt: + @$(DEV_ENV)/bin/sjfmt --py36 --skip-string-normalization --line-length=100 \ + src/ test/ + + +## Run flake8 linter +.PHONY: lint +lint: + @printf "flake8 ..\n" + @$(DEV_ENV)/bin/flake8 src/ + @printf "\e[1F\e[9C ok\n" + + +## Run mypy type checker +.PHONY: mypy +mypy: + @rm -rf ".mypy_cache"; + + @printf "mypy ....\n" + @MYPYPATH=stubs/:vendor/ $(DEV_ENV_PY) -m mypy src/ + @printf "\e[1F\e[9C ok\n" + + +## Run pylint. Should not break the build yet +.PHONY: pylint +pylint: + @printf "pylint ..\n"; + @$(DEV_ENV)/bin/pylint --jobs=4 --output-format=colorized --score=no \ + --disable=C0103,C0301,C0330,C0326,C0330,C0411,R0903,W1619,W1618,W1203 \ + --extension-pkg-whitelist=ujson,lxml,PIL,numpy,pandas,sklearn,pyblake2 \ + src/ + @$(DEV_ENV)/bin/pylint --jobs=4 --output-format=colorized --score=no \ + --disable=C0103,C0111,C0301,C0330,C0326,C0330,C0411,R0903,W1619,W1618,W1203 \ + --extension-pkg-whitelist=ujson,lxml,PIL,numpy,pandas,sklearn,pyblake2 \ + test/ + + @printf "\e[1F\e[9C ok\n" + + +## Run pytest unit and integration tests +.PHONY: test +test: + @rm -rf ".pytest_cache"; + @rm -rf "src/__pycache__"; + @rm -rf "test/__pycache__"; + + ENV=dev PYTHONPATH=src/:vendor/:$$PYTHONPATH \ + $(DEV_ENV_PY) -m pytest -v \ + --doctest-modules \ + --cov-report html \ + --cov-report term \ + --cov=$(PKG_NAME) \ + test/ src/; + + @rm -rf ".pytest_cache"; + @rm -rf "src/__pycache__"; + @rm -rf "test/__pycache__"; + + +## -- Helpers -- + + +## Shortcut for make fmt lint pylint test +.PHONY: check +check: fmt lint mypy test + + +## Start shell with environ variables set. +.PHONY: env +env: + @bash -c '\ + ENV=dev \ + PYTHONPATH=\"src/:vendor/:$$PYTHONPATH\" \ + PATH=\"$(DEV_ENV)/bin/:$$PATH\"; \ + $$SHELL ' + + +## Drop into an ipython shell with correct env variables set +.PHONY: ipy +ipy: + @PYTHONPATH=src/:vendor/:$$PYTHONPATH \ + $(DEV_ENV)/bin/ipython + + +## Like `make test`, but with debug parameters +.PHONY: devtest +devtest: + @rm -rf ".pytest_cache"; + @rm -rf "src/__pycache__"; + @rm -rf "test/__pycache__"; + + +ifndef FILTER + ENV=dev PYTHONPATH=src/:vendor/:$$PYTHONPATH \ + $(DEV_ENV_PY) -m pytest -v \ + --doctest-modules \ + --no-cov \ + --verbose \ + --capture=no \ + --exitfirst \ + test/ src/; +else + ENV=dev PYTHONPATH=src/:vendor/:$$PYTHONPATH \ + $(DEV_ENV_PY) -m pytest -v \ + --doctest-modules \ + --no-cov \ + --verbose \ + --capture=no \ + --exitfirst \ + -k $(FILTER) \ + test/ src/; +endif + + @rm -rf ".pytest_cache"; + @rm -rf "src/__pycache__"; + @rm -rf "test/__pycache__"; + + +## -- Build/Deploy -- + + +## Generate Documentation +.PHONY: doc +doc: + echo "Not Implemented" + + +## Bump Version number in all files +.PHONY: bump_version +bump_version: + echo "Not Implemented" + + +## Freeze dependencies of the current development env +## These dependencies are used for the docker image +.PHONY: freeze +freeze: + echo "Not Implemented" + + +## Create python sdist and bdist_wheel distributions +.PHONY: build_dist +build_dist: + $(DEV_ENV_PY) setup.py sdist bdist_wheel + twine check dist/* + echo "To a PUBLIC release on pypi run:\n\t\$(DEV_ENV_PY) setup.py upload" + + +## Build docker images. Must be run when dependencies are added +## or updated. The main reasons this can fail are: +## 1. No ssh key at $(HOME)/.ssh/${PKG_NAME}_gitlab_runner_id_rsa +## (which is needed to install packages from private repos +## and is copied into a temp container during the build). +## 2. Your docker daemon is not running or configured to +## expose on tcp://localhost:2375 +.PHONY: build_docker +build_docker: + @if [[ -f $$HOME/.ssh/${PKG_NAME}_gitlab_runner_id_rsa ]]; then \ + docker build \ + --build-arg SSH_PRIVATE_RSA_KEY="$$(cat ${HOME}/.ssh/${PKG_NAME}_gitlab_runner_id_rsa)" \ + --file docker_base.Dockerfile \ + --tag $(DOCKER_REGISTRY_URL)/base:latest \ + . + else + docker build \ + --file docker_base.Dockerfile \ + --tag $(DOCKER_REGISTRY_URL)/base:latest \ + . + fi + + docker push $(DOCKER_REGISTRY_URL)/base:latest + + +-include makefile.extra.make diff --git a/makefile.config.make b/makefile.config.make new file mode 100644 index 0000000..d196e34 --- /dev/null +++ b/makefile.config.make @@ -0,0 +1,21 @@ + +PACKAGE_NAME := pycalver +DOCKER_REGISTRY_URL := registry.gitlab.com/mbarkhau/pycalver + +# This is the python version that is used for: +# - `make fmt` +# - `make ipy` +# - `make lint` +# - `make devtest` +DEVELOPMENT_PYTHON_VERSION := python=3.6 + +# These must be valid conda package names. A separate +# conda environment will be created for each of these. +# Some valid options are: +# - python=2.7 +# - python=3.5 +# - python=3.6 +# - python=3.7 +# - pypy2.7 +# - pypy3.5 +SUPPORTED_PYTHON_VERSIONS := python=2.7 python=3.6 python=3.7 diff --git a/makefile.extra.make b/makefile.extra.make new file mode 100644 index 0000000..7f637e0 --- /dev/null +++ b/makefile.extra.make @@ -0,0 +1,9 @@ + + +## Start the development http server in debug mode +## This is just to illustrate how to add your +## extra targets outside of the main makefile. +.PHONY: serve +serve: + echo "Not Implemented" + diff --git a/requirements-dev.txt b/requirements-dev.txt deleted file mode 100644 index 5e63e42..0000000 --- a/requirements-dev.txt +++ /dev/null @@ -1,7 +0,0 @@ -wheel -pip -twine -ipython -pudb -py-spy -snakeviz diff --git a/requirements-test.txt b/requirements-test.txt deleted file mode 100644 index deccc16..0000000 --- a/requirements-test.txt +++ /dev/null @@ -1,8 +0,0 @@ -flake8 -flake8-bugbear -mypy -typing-extensions -rst2html5 -pytest -pytest-cov -codecov diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index adcf504..0000000 --- a/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -setuptools -pathlib2 -typing -click diff --git a/requirements/conda.txt b/requirements/conda.txt new file mode 100644 index 0000000..bdd0e41 --- /dev/null +++ b/requirements/conda.txt @@ -0,0 +1,34 @@ +# These dependencies are installed using: +# +# conda install --channel conda-forge --name +# +# Conda should be used for +# +# 1. Binary python packages (numpy, pandas, pillow). +# The pypi may not always have binary packages for all platforms +# and architectures you want to support. For example, pyblake2 only +# has binary wheels for windows on pypi, whereas there are binary +# packages on conda-forge (as of Sep 2018). +# Binary wheels are becomming more common on the pypi this is +# becomming, so this is less and less of an issue. Most of the time +# it should be fine to add the dependency to pypi.txt instead. +# +# 2. Non python packages (nodejs, typescript). +# Using conda for these kinds of dependencies minimizes +# installation overhead for developers. + +# https://pypi.org/project/ujson/ +# UltraJSON is an ultra fast JSON encoder and decoder written +# in pure C with bindings for Python 2.5+ and 3. +ujson + +# The hot new pkdf on the block is argon2, winner of +# the https://password-hashing.net/ competition. +argon2_cffi + +# https://blake2.net/ +# BLAKE2 is a cryptographic hash function faster than MD5, SHA-1, +# SHA-2, and SHA-3, yet is at least as secure as the latest standard +# SHA-3. BLAKE2 has been adopted by many projects due to its high +# speed, security, and simplicity. +pyblake2 diff --git a/requirements/development.txt b/requirements/development.txt new file mode 100644 index 0000000..bd94e7d --- /dev/null +++ b/requirements/development.txt @@ -0,0 +1,26 @@ +# These dependencies are installed using: +# +# pip install --upgrade +# +# This list should only contain packages related to +# local development and debugging. It should not contain +# any packages required for production, building or packaging + +# PuDB is a full-screen, console-based visual debugger for Python. +# https://documen.tician.de/pudb/ +pudb + +# Py-Spy: A sampling profiler for Python programs. +# https://github.com/benfred/py-spy +# This is good for coarse grained profiling (even on production) +py-spy + +# SNAKEVIZ : A browser based viewer for the output of Python’s cProfile. +# https://jiffyclub.github.io/snakeviz/ +# This is good for fine grained profiling (function level/micro optimizations) +snakeviz + +# I've yet to find a decent memory profiler for python, feel free to +# add one after you've tested it and found it to be actually useful. + +ipython # nuff said diff --git a/requirements/integration.txt b/requirements/integration.txt new file mode 100644 index 0000000..10b9d8d --- /dev/null +++ b/requirements/integration.txt @@ -0,0 +1,24 @@ +# These dependencies are installed using: +# +# pip install --upgrade +# +# This file should only declare dependencies related to code +# formatting, linting, testing and packaging. +# +# No dependencies required for production should be listed here. + +flake8 +flake8-bugbear +flake8-docstrings +flake8-builtins +flake8-comprehensions +pylint +mypy +pytest +pytest-cov +pylint + +twine + +straitjacket +pycalver diff --git a/requirements/pypi.txt b/requirements/pypi.txt new file mode 100644 index 0000000..bc7d91a --- /dev/null +++ b/requirements/pypi.txt @@ -0,0 +1,12 @@ +# These dependencies are installed using: +# +# pip install --upgrade +# +# This list is the default package list. All pure python packages +# for the production environment at runtime should be listed here. +# Binary (non-pure) packages may also be listed here, but you +# should see if there is a conda package that suits your needs. + +pathlib2 +typing +click diff --git a/requirements/vendor.txt b/requirements/vendor.txt new file mode 100644 index 0000000..985e534 --- /dev/null +++ b/requirements/vendor.txt @@ -0,0 +1,22 @@ +# These dependencies are installed using: +# +# pip install --upgrade +# pip install --upgrade --no-deps --target vendor/ +# +# Vendored dependencies are installed both in the virtual +# environment as well as in the vendor/ directory. This way: +# +# 1. All transitive dependencies of a package are installed in +# the virtualenv (in the first installation step) +# 2. If there is a binary version of the package available, it +# will be installed into the virtualenv +# 3. In the third step only (--no-deps) the source version of +# the (--no-binary) package is installed to vendor/ +# +# This allows us to: +# +# 1. Easily navigate to the source of a vendored dependency +# 2. Use binary versions packages instead of source versions of +# packages, simply by not including the vendor/ directory in +# the PYTHONPATH. The version from the virtualenv will then +# be loaded instead. diff --git a/scripts/pre-push-hook.sh b/scripts/pre-push-hook.sh new file mode 100644 index 0000000..12c42ff --- /dev/null +++ b/scripts/pre-push-hook.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail; + +make fmt; + +git diff --exit-code --stat src/; +git diff --exit-code --stat test/; +git diff --exit-code --stat scripts/; +git diff --exit-code --stat requirements/; + +make lint; +make test; diff --git a/scripts/setup_conda_envs.sh b/scripts/setup_conda_envs.sh new file mode 100644 index 0000000..c16e36a --- /dev/null +++ b/scripts/setup_conda_envs.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +read -r -a env_paths <<< "${CONDA_ENV_PATHS//, /$IFS}"; +read -r -a env_names <<< "${CONDA_ENV_NAMES//, /$IFS}"; +read -r -a py_versions <<< "${SUPPORTED_PYTHON_VERSIONS//, /$IFS}"; + +for i in ${!env_paths[@]}; do + env_path=${env_paths[i]}; + env_path_python=${env_path}/bin/python; + env_name=${env_names[i]}; + py_version=${py_versions[i]}; + + if [[ ! -f ${env_path_python} ]]; then + echo "conda create --name ${env_name} ${py_version} ..."; + ${CONDA_BIN} create --name ${env_name} ${py_version} --yes --quiet; + fi; + + echo "updating ${env_name} conda deps ..."; + ${CONDA_BIN} install --name ${env_name} --channel conda-forge --yes --quiet \ + $(grep -o '^[^#][^ ]*' requirements/conda.txt) + + ${env_path_python} --version >> build/envs.txt.tmp \ + 2>>build/envs.txt.tmp \ + 1>>build/envs.txt.tmp; + +done; diff --git a/scripts/update_conda_env_deps.sh b/scripts/update_conda_env_deps.sh new file mode 100644 index 0000000..c578f3d --- /dev/null +++ b/scripts/update_conda_env_deps.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +read -r -a env_paths <<< "${CONDA_ENV_PATHS//, /$IFS}"; +read -r -a env_names <<< "${CONDA_ENV_NAMES//, /$IFS}"; + +for i in ${!env_paths[@]}; do + env_path=${env_paths[i]}; + env_path_python=${env_path}/bin/python; + env_name=${env_names[i]}; + + ${env_path_python} -m pip install --upgrade --quiet pip; + + echo "updating ${env_name} pypi deps ..."; + + ${env_path_python} -m pip install \ + --disable-pip-version-check --upgrade --quiet \ + --requirement=requirements/pypi.txt; + + echo "updating ${env_name} vendor deps ..."; + + ${env_path_python} -m pip install \ + --disable-pip-version-check --upgrade --quiet \ + --requirement=requirements/vendor.txt; +done; diff --git a/setup.cfg b/setup.cfg index 3a186ff..cb41352 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,42 +1,59 @@ -[flake8] -ignore = - # No whitespace after paren open "(" - E201, - # No whitespace before paren ")" - E202, - # No whitespace before ":" - E203, - # Multiple spaces before operator - E221 - # Multiple spaces before keyword - E272, - # Spaces around keyword/parameter equals - E251 - # Line too long (B950 is used instead) - E501, - # Line break before binary op - W503, - # Line break after binary op - W504 -select = C,E,F,W,B,B901,B950 -max-line-length = 100 -exclude = .git,__pycache__,.eggs/,dist/,.mypy_cache +[metadata] +license_file = LICENSE + +[bdist_wheel] +universal = 1 [mypy] check_untyped_defs = True disallow_untyped_calls = True follow_imports = silent strict_optional = True +ignore_missing_imports = True + +[flake8] +max-line-length = 100 +max-complexity = 10 +ignore = + # No whitespace after paren open "(" + E201 + # No whitespace before paren ")" + E202 + # No whitespace before ":" + E203 + # Multiple spaces before operator + E221 + # Multiple spaces before keyword + E272 + # Spaces around keyword/parameter equals + E251 + # Line too long (B950 is used instead) + E501 + # Line break before binary op + W503 + # Line break after binary op + W504 + # Missing docstring in public module + # D100 + # Missing docstring in public class + # D101 + # Missing docstring on __init__ + D107 +select = A,AAA,D,C,E,F,W,H,B,D212,D404,D405,D406,B901,B950 +exclude = + .git + __pycache__ + .eggs/ + dist/ + .mypy_cache + +# Hopefully this can be resolved, so D404, D405 start working +# https://github.com/PyCQA/pydocstyle/pull/188 -[aliases] -test=pytest [tool:pytest] -addopts = --verbose -python_files = test/*.py +addopts = --doctest-modules -[bdist_wheel] -universal = 1 [pycalver] current_version = v201809.0002-beta @@ -55,7 +72,7 @@ patterns = patterns = __version__ = "{version}" -[pycalver:file:README.rst] +[pycalver:file:README.md] patterns = - badge/CalVer-{calver}{build}-{release}-blue.svg - :alt: CalVer {version} \ No newline at end of file + [PyCalVer {calver}{build}-{release}] + img.shields.io/badge/PyCalVer-{calver}{build}--{release}-blue diff --git a/setup.py b/setup.py index 432c4d1..df43a45 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ # This file is part of the pycalver project -# https://github.com/mbarkhau/pycalver +# https://gitlab.com/mbarkhau/pycalver # -# (C) 2018 Manuel Barkhau (mbarkhau@gmail.com) +# (C) 2018 Manuel Barkhau (@mbarkhau) # SPDX-License-Identifier: MIT import os @@ -9,13 +9,13 @@ import sys import setuptools -def project_path(filename): - dirpath = os.path.abspath(os.path.dirname(__file__)) - return os.path.join(dirpath, filename) +def project_path(*sub_paths): + project_dirpath = os.path.abspath(os.path.dirname(__file__)) + return os.path.join(project_dirpath, *sub_paths) -def read(filename): - with open(project_path(filename), mode="rb") as fh: +def read(*sub_paths): + with open(project_path(*sub_paths), mode="rb") as fh: return fh.read().decode("utf-8") @@ -39,12 +39,12 @@ setuptools.setup( name="pycalver", license="MIT", author="Manuel Barkhau", - author_email="mbarkhau@gmail.com", - url="https://github.com/mbarkhau/pycalver", + author_email="@mbarkhau", + url="https://gitlab.com/mbarkhau/pycalver", version="201809.2b0", keywords="version versioning bumpversion calver", - description="CalVer versioning for python projects", + description="CalVer versioning for python libraries.", long_description=long_description, packages=packages, diff --git a/src/pycalver/__init__.py b/src/pycalver/__init__.py index 3a23c79..07521a5 100644 --- a/src/pycalver/__init__.py +++ b/src/pycalver/__init__.py @@ -1,7 +1,7 @@ # This file is part of the pycalver project -# https://github.com/mbarkhau/pycalver +# https://gitlab.com/mbarkhau/pycalver # -# (C) 2018 Manuel Barkhau (@mbarkhau) +# Copyright (c) 2018 Manuel Barkhau (@mbarkhau) - MIT License # SPDX-License-Identifier: MIT import os diff --git a/src/pycalver/__main__.py b/src/pycalver/__main__.py index 38cfc28..2f6712d 100644 --- a/src/pycalver/__main__.py +++ b/src/pycalver/__main__.py @@ -1,8 +1,8 @@ #!/usr/bin/env python # This file is part of the pycalver project -# https://github.com/mbarkhau/pycalver +# https://gitlab.com/mbarkhau/pycalver # -# (C) 2018 Manuel Barkhau (@mbarkhau) +# Copyright (c) 2018 Manuel Barkhau (@mbarkhau) - MIT License # SPDX-License-Identifier: MIT import io diff --git a/stubs/README.md b/stubs/README.md new file mode 100644 index 0000000..a78f521 --- /dev/null +++ b/stubs/README.md @@ -0,0 +1,6 @@ +# Stub files for mypy + +Before using stubs, check if the library you want to use +itself uses mypy. If it does, the better approach is to +add it to `requirements/vendor.txt`. This way mypy will +find the actual source instead of just stub files.