-
Notifications
You must be signed in to change notification settings - Fork 6
/
Makefile
400 lines (292 loc) · 12 KB
/
Makefile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
#!/usr/bin/make -f
ifndef VERBOSE
.SILENT:
endif
SHELL := bash
.ONESHELL:
.SHELLFLAGS := -eu -o pipefail -c
.DELETE_ON_ERROR:
MAKEFLAGS += --warn-undefined-variables
MAKEFLAGS += --no-builtin-rules
# For install target stability
export LC_ALL = en_US.UTF-8
export LANG = en_US.UTF-8
export PROJECT_NAME ?= clickhouse-tools
export PROJECT_NAME_UNDERSCORE ?= $(subst -,_,$(PROJECT_NAME))
export PYTHON ?= python3
PYTHON_MAJOR := $(shell $(PYTHON) -c 'import sys; print(sys.version_info[0])')
PYTHON_MINOR := $(shell $(PYTHON) -c 'import sys; print(sys.version_info[1])')
POETRY_DEFAULT_VERSION = 1.1.15
# The latest version supporting Python 3.6
POETRY_VERSION ?= $(POETRY_DEFAULT_VERSION)
POETRY_HOME ?= /opt/poetry
POETRY := $(POETRY_HOME)/bin/poetry
# The minimum officially supported version for Python 3.12 is 1.7.0 (https://python-poetry.org/history/#170---2023-11-03)
ifeq ($(shell test $(PYTHON_MAJOR) -ge 3 && test $(PYTHON_MINOR) -ge 12; echo $$?),0)
POETRY_VERSION="1.8.3"
endif
UPDATE_POETRY_LOCK =
ifneq ($(POETRY_VERSION), $(POETRY_DEFAULT_VERSION))
UPDATE_POETRY_LOCK="y"
endif
PREFIX ?= /opt/yandex/$(PROJECT_NAME)
export BUILD_PYTHON_OUTPUT_DIR ?= dist
export BUILD_DEB_OUTPUT_DIR ?= out
SRC_DIR ?= ch_tools
TESTS_DIR ?= tests
# It is used by DEB building tools to install a program to temporary
# directory before packaging
DESTDIR ?=
INSTALL_DIR = $(DESTDIR)$(PREFIX)
BIN_DIR = $(INSTALL_DIR)/bin
SYMLINK_BIN_DIR = $(DESTDIR)/usr/bin
WHL_FILE := $(PROJECT_NAME_UNDERSCORE)-*.whl
VENV_DIR := .venv
VERSION_FILE := version.txt
INSTALL_DEPS_STAMP := .install-deps
# Pass arguments for testing tools
BEHAVE_ARGS ?=
PYTEST_ARGS ?=
# Different ways of passing signing key for building debian package
export DEB_SIGN_KEY_ID ?=
export DEB_SIGN_KEY ?=
export DEB_SIGN_KEY_PATH ?=
# Platform of image for building debian package according to
# https://docs.docker.com/build/building/multi-platform/#building-multi-platform-images
# E.g. linux/amd64, linux/arm64, etc.
# If platform is not provided Docker uses platform of the host performing the build
export DEB_TARGET_PLATFORM ?=
# Name of image (with tag) for building deb package.
# E.g. ubuntu:22.04, ubuntu:jammy, ubuntu:bionic, etc.
# If it is not provided, default value in Dockerfile is used
export DEB_BUILD_DISTRIBUTION ?=
define ensure_poetry
if [ ! -e $(POETRY) ]; then
echo "Poetry could not be found. Please install it manually 'make install-poetry'";
exit 1;
fi
endef
# Should be default target, because "make" is just run
# by debhelper(dh_auto_build stage) for building the program during
# creation of a DEB package
.PHONY: build
build: build-python-packages ;
.PHONY: all
all: build lint test-unit test-integration
.PHONY: install
install: install-python-package install-symlinks install-bash-completions configure-logs ;
.PHONY: uninstall
uninstall: uninstall-python-package uninstall-symlinks uninstall-bash-completions uninstall-logrotate ;
.PHONY: install-deps
install-deps: $(INSTALL_DEPS_STAMP) ;
# Update dependencies in poetry.lock to their latest versions according to your pyproject.toml
.PHONY: update-deps
update-deps:
$(ensure_poetry)
$(POETRY) update
$(INSTALL_DEPS_STAMP): $(VENV_DIR) pyproject.toml poetry.lock
$(ensure_poetry)
@if [[ -n "${UPDATE_POETRY_LOCK}" ]]; then \
$(POETRY) update --lock; \
fi
$(POETRY) install --no-root
touch $(INSTALL_DEPS_STAMP)
.PHONY: install-poetry
install-poetry:
[ -e $(POETRY) ] && echo "Found installed poetry $$($(POETRY) --version)" && exit 0
echo "Installing poetry $(POETRY_VERSION)..."
curl -sSL https://install.python-poetry.org | POETRY_HOME=$(POETRY_HOME) $(PYTHON) - --version $(POETRY_VERSION)
# TODO: remove after getting rid of support Python 3.6
# Fix "cannot import name 'appengine' from 'urllib3.contrib'..." error
# while 'poetry publish' for version poetry 1.1.15 due to incompatibility with urllib3 >= 2.0.0
# https://urllib3.readthedocs.io/en/stable/v2-migration-guide.html#importerror-cannot-import-name-gaecontrib-from-requests-toolbelt-compat
$(POETRY_HOME)/venv/bin/python -m pip install "urllib3<2.0.0"
.PHONY: uninstall-poetry
uninstall-poetry:
[ ! -e $(POETRY_HOME) ] && exit 0;
rm -rf $(POETRY_HOME)
echo "Uninstalled poetry from $(POETRY_HOME)"
$(VENV_DIR):
$(ensure_poetry)
$(POETRY) config virtualenvs.in-project true
$(POETRY) env use $(PYTHON)
.PHONY: lint
lint: isort black codespell ruff pylint mypy
.PHONY: isort
isort: install-deps
$(POETRY) run isort --check --diff $(SRC_DIR) $(TESTS_DIR)
.PHONY: black
black: install-deps
$(POETRY) run black --check --diff $(SRC_DIR) $(TESTS_DIR)
.PHONY: codespell
codespell: install-deps
$(POETRY) run codespell
.PHONY: fix-codespell-errors
fix-codespell-errors: install-deps
$(POETRY) run codespell -w
.PHONY: ruff
ruff: install-deps
$(POETRY) run ruff check $(SRC_DIR) $(TESTS_DIR)
.PHONY: pylint
pylint: install-deps
$(POETRY) run pylint $(SRC_DIR)
.PHONY: mypy
mypy: install-deps
$(POETRY) run mypy --python-version=3.6 $(SRC_DIR) $(TESTS_DIR)
.PHONY: format
format: install-deps
$(POETRY) run isort $(SRC_DIR) $(TESTS_DIR)
$(POETRY) run black $(SRC_DIR) $(TESTS_DIR)
.PHONY: test-unit
test-unit: install-deps
$(POETRY) run $(PYTHON) -m pytest $(PYTEST_ARGS) $(TESTS_DIR)/unit
.PHONY: test-integration
test-integration: install-deps build-python-packages
cd $(TESTS_DIR)
export PYTHONPATH=$(CURDIR):$$PATH
$(POETRY) run behave --show-timings --stop --junit $(BEHAVE_ARGS)
.PHONY: publish
publish:
$(POETRY) publish
.PHONY: install-python-package
install-python-package: build-python-packages
echo 'Installing $(PROJECT_NAME)'
# Prepare new virtual environment
$(PYTHON) -m venv $(INSTALL_DIR)
rm -f $(BIN_DIR)/activate*
# Install python package
$(BIN_DIR)/pip install --upgrade pip
$(BIN_DIR)/pip install --no-compile $(BUILD_PYTHON_OUTPUT_DIR)/$(WHL_FILE)
# Clean python's artefacts
find $(INSTALL_DIR) -name __pycache__ -type d -exec rm -rf {} +
# Remove DESTDIR prefix from script's shebangs if it's present
test -n '$(DESTDIR)' \
&& grep -l -r -F '#!$(INSTALL_DIR)' $(INSTALL_DIR) \
| xargs sed -i -e 's|$(INSTALL_DIR)|$(PREFIX)|' \
|| true
.PHONY: build-python-packages
build-python-packages: prepare-version clean-dist
echo 'Building python packages...'
$(POETRY) build
.PHONY: clean-dist
clean-dist:
echo 'Cleaning up residuals from building of Python package'
rm -rf $(BUILD_PYTHON_OUTPUT_DIR)
.PHONY: uninstall-python-package
uninstall-python-package:
echo 'Uninstalling $(PROJECT_NAME)'
rm -rf $(INSTALL_DIR)
.PHONY: install-symlinks
install-symlinks:
echo 'Creating symlinks to $(SYMLINK_BIN_DIR)'
mkdir -p $(SYMLINK_BIN_DIR)
$(foreach bin, chadmin ch-monitoring keeper-monitoring ch-s3-credentials, \
ln -sf $(PREFIX)/bin/$(bin) $(SYMLINK_BIN_DIR);)
.PHONY: uninstall-symlinks
uninstall-symlinks:
echo 'Removing symlinks from $(SYMLINK_BIN_DIR)'
$(foreach bin, chadmin ch-monitoring keeper-monitoring ch-s3-credentials, \
rm -f $(SYMLINK_BIN_DIR)/$(bin);)
.PHONY: install-bash-completions
install-bash-completions:
echo 'Creating bash completions'
mkdir -p $(DESTDIR)/etc/bash_completion.d/
$(foreach bin, chadmin ch-monitoring keeper-monitoring, \
cp resources/completion/$(bin)-completion.bash $(DESTDIR)/etc/bash_completion.d/$(bin);)
.PHONY: uninstall-bash-completions
uninstall-bash-completions:
echo 'Removing bash completions'
$(foreach bin, chadmin ch-monitoring keeper-monitoring, \
rm -f $(DESTDIR)/etc/bash_completion.d/$(bin);)
.PHONY: configure-logs
configure-logs:
echo 'Configuring logging'
mkdir -p $(DESTDIR)/etc/logrotate.d/
$(foreach bin, chadmin clickhouse-monitoring keeper-monitoring, \
mkdir -p $(DESTDIR)/var/log/$(bin) ; \
chmod 775 $(DESTDIR)/var/log/$(bin) ; \
cp resources/logrotate/$(bin).logrotate $(DESTDIR)/etc/logrotate.d/$(bin);)
.PHONY: uninstall-logrotate
uninstall-logrotate:
echo 'Removing log rotation rules'
$(foreach bin, chadmin clickhouse-monitoring keeper-monitoring, \
rm -f $(DESTDIR)/etc/logrotate.d/$(bin);)
.PHONY: prepare-changelog
prepare-changelog: prepare-version
echo 'Bumping version into Debian package changelog'
DEBFULLNAME="Yandex LLC" DEBEMAIL="[email protected]" dch --force-bad-version --distribution stable -v $$(cat $(VERSION_FILE)) Autobuild
.PHONY: prepare-version
prepare-version: $(VERSION_FILE)
VERSION=$$(cat $(VERSION_FILE))
# Replace version in $(SRC_DIR)/__init__.py
sed -i "s/__version__ = \"[0-9\.]\+\"/__version__ = \"$${VERSION}\"/g" $(SRC_DIR)/__init__.py
# Replace version in pyproject.toml
$(POETRY) version $${VERSION}
echo "Version: $${VERSION}"
$(VERSION_FILE):
echo "2.$$(git rev-list HEAD --count).$$(git rev-parse --short HEAD | xargs -I {} printf '%d' 0x{})" > $@
.PHONY: prepare-build-deb
prepare-build-deb:
apt install python3-venv debhelper devscripts
.PHONY: build-deb-package
build-deb-package:
./build_deb_in_docker.sh
.PHONY: build-deb-package-local
build-deb-package-local: prepare-changelog
./build_deb.sh
.PHONY: clean_debuild
clean_debuild:
rm -rf debian/{files,.debhelper,$(PROJECT_NAME)*,*stamp}
rm -f ../$(PROJECT_NAME)_*{build,buildinfo,changes,deb,dsc,gz,xz}
.PHONY: clean
clean: clean_debuild
echo 'Cleaning up'
rm -rf $(BUILD_DEB_OUTPUT_DIR)
rm -rf $(BUILD_PYTHON_OUTPUT_DIR)
rm -rf $(VENV_DIR)
rm -rf $(INSTALL_DEPS_STAMP)
rm -rf .mypy_cache .ruff_cache tests/{.session_conf.sav,__pycache__,staging,reports}
.PHONY: help
help:
echo "Base targets:"
echo " install-poetry Install Poetry"
echo " uninstall-poetry Uninstall Poetry"
echo " install-deps Install Python dependencies to local environment $(VENV_DIR)"
echo " update-deps Update dependencies in poetry.lock to their latest versions"
echo " publish Publish python package to PYPI"
echo " lint Run linters. Alias for \"isort black codespell ruff pylint mypy\"."
echo " test-unit Run unit tests."
echo " test-integration Run integration tests."
echo " isort Perform isort checks."
echo " black Perform black checks."
echo " codespell Perform codespell checks."
echo " ruff Perform ruff checks."
echo " pylint Perform pylint checks."
echo " mypy Perform mypy checks."
echo " format Re-format source code to conform style settings enforced by"
echo " isort and black tools."
echo " prepare-changelog Add an autobuild version entity to changelog"
echo " prepare-version Update version based on latest commit"
echo " build-python-packages Build '$(PROJECT_NAME)' Python packages (sdist and wheel)"
echo " prepare-build-deb Install prerequisites for DEB packaging tool"
echo " build-deb-package Build '$(PROJECT_NAME)' debian package"
echo " clean Clean-up all produced/generated files inside tree"
echo ""
echo "--------------------------------------------------------------------------------"
echo ""
echo "Debian package build targets:"
echo " install Install '$(PROJECT_NAME)' debian package"
echo " uninstall Uninstall '$(PROJECT_NAME)' debian package"
echo ""
echo " install-python-package Install '$(PROJECT_NAME)' python package"
echo " uninstall-python-package Uninstall '$(PROJECT_NAME)' python package"
echo " install-symlinks Install symlinks to /usr/bin/"
echo " uninstall-symlinks Uninstall symlinks from /usr/bin/"
echo " install-bash-completions Install to /etc/bash_completion.d/"
echo " uninstall-bash-completions Uninstall from /etc/bash_completion.d/"
echo " configure-logs Install log rotation rules to /etc/logrotate.d/ and create log dirs"
echo " uninstall-logrotate Uninstall log rotation rules from /etc/logrotate.d/"
echo ""
echo "--------------------------------------------------------------------------------"
echo ""
echo " help Show this help message."