-
Notifications
You must be signed in to change notification settings - Fork 2
/
Makefile
448 lines (318 loc) · 14.8 KB
/
Makefile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
# ----------------------------------------
# Makefile for chr
# Generated using ontology-development-kit
# ODK Version: v1.2.29
# ----------------------------------------
# IMPORTANT: DO NOT EDIT THIS FILE. To override default make goals, use chr.Makefile instead
# ----------------------------------------
# More information: https://github.com/INCATools/ontology-development-kit/
# ----------------------------------------
# Standard Constants
# ----------------------------------------
# these can be overwritten on the command line
URIBASE= http://purl.obolibrary.org/obo
ONT= chr
ONTBASE= $(URIBASE)/$(ONT)
EDIT_FORMAT= owl
SRC = $(ONT)-edit.$(EDIT_FORMAT)
CATALOG= catalog-v001.xml
ROBOT= robot --catalog $(CATALOG)
RELEASEDIR= ../..
REPORTDIR= reports
TEMPLATEDIR= ../templates
TMPDIR= tmp
SCRIPTSDIR= ../scripts
SPARQLDIR = ../sparql
COMPONENTSDIR = components
REPORT_FAIL_ON = None
REPORT_LABEL = -l true
REPORT_PROFILE_OPTS =
OBO_FORMAT_OPTIONS =
SPARQL_VALIDATION_CHECKS = equivalent-classes owldef-self-reference
SPARQL_EXPORTS = basic-report class-count-by-prefix edges xrefs obsoletes synonyms
ODK_VERSION_MAKEFILE = v1.2.29
TODAY ?= $(shell date +%Y-%m-%d)
OBODATE ?= $(shell date +'%d:%m:%Y %H:%M')
VERSION= $(TODAY)
ANNOTATE_ONTOLOGY_VERSION = annotate -V $(ONTBASE)/releases/$(VERSION)/$@ --annotation owl:versionInfo $(VERSION)
OTHER_SRC = $(COMPONENTSDIR)/ucsc.owl
ONTOLOGYTERMS = $(TMPDIR)/ontologyterms.txt
FORMATS = $(sort owl obo json owl)
FORMATS_INCL_TSV = $(sort $(FORMATS) tsv)
RELEASE_ARTEFACTS = $(sort $(ONT)-full $(ONT)-base $(ONT)-base $(ONT)-full)
# ----------------------------------------
# Top-level targets
# ----------------------------------------
.PHONY: .FORCE
.PHONY: all
all: odkversion all_imports all_main all_subsets sparql_test all_reports all_assets
.PHONY: test
test: odkversion sparql_test all_reports
$(ROBOT) reason --input $(SRC) --reasoner ELK --equivalent-classes-allowed all --exclude-tautologies structural --output test.owl && rm test.owl && echo "Success"
.PHONY: odkversion
odkversion:
echo "ODK Makefile version: $(ODK_VERSION_MAKEFILE) (this is the version of the ODK with which this Makefile was generated, not the version of the ODK you are running)" &&\
echo "ROBOT version (ODK): " && $(ROBOT) --version
$(TMPDIR) $(REPORTDIR) :
mkdir -p $@
## -- main targets --
##
## By default this is the cross-product of {ont, ont-base} x FORMATS
MAIN_PRODUCTS = $(sort $(foreach r,$(RELEASE_ARTEFACTS), $(r)) $(ONT))
MAIN_GZIPPED =
MAIN_FILES = $(foreach n,$(MAIN_PRODUCTS), $(foreach f,$(FORMATS), $(n).$(f))) $(MAIN_GZIPPED)
.PHONY: all_main
all_main: $(MAIN_FILES)
## -- import targets --
##
## By default this is the cross-product of IMPORT_MODULES x FORMATS
IMPORTS = ro go ncbitaxon
IMPORT_ROOTS = $(patsubst %, imports/%_import, $(IMPORTS))
IMPORT_OWL_FILES = $(foreach n,$(IMPORT_ROOTS), $(n).owl)
IMPORT_FILES = $(IMPORT_OWL_FILES)
.PHONY: all_imports
all_imports: $(IMPORT_FILES)
## -- subset targets --
##
## By default this is the cross-product of SUBSETS x FORMATS
## Note we also include TSV as a format
SUBSETS =
SUBSET_ROOTS = $(patsubst %, subsets/%, $(SUBSETS))
SUBSET_FILES = $(foreach n,$(SUBSET_ROOTS), $(foreach f,$(FORMATS_INCL_TSV), $(n).$(f)))
.PHONY: all_subsets
all_subsets: $(SUBSET_FILES)
OBO_REPORT = $(SRC)-obo-report
REPORTS = $(OBO_REPORT)
REPORT_FILES = $(patsubst %, $(REPORTDIR)/%.tsv, $(REPORTS))
.PHONY: robot_reports
robot_reports: $(REPORT_FILES)
.PHONY: all_reports
all_reports: all_reports_onestep $(REPORT_FILES)
$(REPORTDIR)/validate_profile_owl2dl_%.txt: % | $(REPORTDIR)
$(ROBOT) validate-profile --profile DL -i $< -o $@
.PRECIOUS: $(REPORTDIR)/validate_profile_owl2dl_%.txt
.PHONY: validate_profile_%
validate_profile_%: $(REPORTDIR)/validate_profile_owl2dl_%.txt
## -- all files/assets --
ASSETS = \
$(IMPORT_FILES) \
$(MAIN_FILES) \
$(REPORT_FILES) \
$(SUBSET_FILES)
RELEASE_ASSETS = \
$(MAIN_FILES) \
$(SUBSET_FILES)
.PHONY: all_assets
all_assets: $(ASSETS)
.PHONY: show_assets
show_assets:
echo $(ASSETS)
du -sh $(ASSETS)
# ----------------------------------------
# Release Management
# ----------------------------------------
CLEANFILES=$(MAIN_FILES) $(SRCMERGED)
# This should be executed by the release manager whenever time comes to make a release.
# It will ensure that all assets/files are fresh, and will copy to release folder
.PHONY: prepare_release
prepare_release: $(ASSETS) $(PATTERN_RELEASE_FILES)
rsync -R $(RELEASE_ASSETS) $(RELEASEDIR) &&\
rm -f $(CLEANFILES) &&\
echo "Release files are now in $(RELEASEDIR) - now you should commit, push and make a release on your git hosting site such as GitHub or GitLab"
.PHONY: prepare_initial_release
prepare_initial_release: prepare_release
cd $(RELEASEDIR) && git add $(RELEASE_ASSETS)
# ----------------------------------------
# Import modules
# ----------------------------------------
# Most ontologies are modularly constructed using portions of other ontologies
# These live in the imports/ folder
# ------------------------
# Imports: Seeding system
# ------------------------
# seed.txt contains all referenced entities
IMPORTSEED=$(TMPDIR)/seed.txt
SRCMERGED=$(TMPDIR)/merged-$(SRC)
PRESEED=$(TMPDIR)/pre_seed.txt
$(SRCMERGED): $(SRC)
$(ROBOT) remove --input $< --select imports --trim false \
merge $(patsubst %, -i %, $(OTHER_SRC)) -o $@
$(PRESEED): $(SRCMERGED)
$(ROBOT) query -f csv -i $< --query ../sparql/terms.sparql [email protected] &&\
cat [email protected] | sort | uniq > $@
ALLSEED = $(PRESEED) \
$(IMPORTSEED): $(ALLSEED)
if [ $(IMP) = true ]; then cat $(ALLSEED) | sort | uniq > $@; fi
ANNOTATION_PROPERTIES=rdfs:label IAO:0000115
# -- Generate Import Modules --
#
# This pattern uses ROBOT to generate an import module
# Generate terms.txt for each import. (Assume OBO-style Possibly hacky step?)
# Should be able to drop this if robot can just take a big messy list of terms as input.
imports/%_terms_combined.txt: $(IMPORTSEED) imports/%_terms.txt
if [ $(IMP) = true ]; then cat $^ | grep -v ^# | sort | uniq > $@; fi
imports/%_import.owl: mirror/%.owl imports/%_terms_combined.txt
if [ $(IMP) = true ]; then $(ROBOT) query -i $< --update ../sparql/preprocess-module.ru \
extract -T imports/$*_terms_combined.txt --force true --copy-ontology-annotations true --individuals include --method BOT \
query --update ../sparql/inject-subset-declaration.ru --update ../sparql/postprocess-module.ru \
annotate --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) --output [email protected] && mv [email protected] $@; fi
.PRECIOUS: imports/%_import.owl
.PHONY: refresh-imports
refresh-imports:
make IMP=true MIR=true PAT=false IMP_LARGE=true all_imports -B
.PHONY: refresh-imports-excluding-large
refresh-imports-excluding-large:
make IMP=true MIR=true PAT=false IMP_LARGE=false all_imports -B
.PHONY: refresh-%
refresh-%:
make IMP=true IMP_LARGE=true MIR=true PAT=false imports/$*_import.owl -B
# ----------------------------------------
# Components
# ----------------------------------------
# Some ontologies contain external and internal components. A component is included in the ontology in its entirety.
$(COMPONENTSDIR)/%:
touch $@
.PRECIOUS: $(COMPONENTSDIR)/%
# ----------------------------------------
# Mirroring upstream ontologies
# ----------------------------------------
#
IMP=true # Global parameter to bypass import generation
MIR=true # Global parameter to bypass mirror generation
IMP_LARGE=true # Global parameter to bypass handling of large imports
## ONTOLOGY: ro
## Copy of ro is re-downloaded whenever source changes
mirror/ro.trigger: $(SRC)
mirror/ro.owl: mirror/ro.trigger
if [ $(MIR) = true ] && [ $(IMP) = true ]; then $(ROBOT) convert -I $(URIBASE)/ro.owl -o [email protected] && mv [email protected] $@; fi
.PRECIOUS: mirror/%.owl
## ONTOLOGY: so
## Copy of so is re-downloaded whenever source changes
mirror/so.trigger: $(SRC)
mirror/so.owl: mirror/so.trigger
if [ $(MIR) = true ] && [ $(IMP) = true ]; then $(ROBOT) convert -I $(URIBASE)/so.owl -o [email protected] && mv [email protected] $@; fi
.PRECIOUS: mirror/%.owl
## ONTOLOGY: go
## Copy of go is re-downloaded whenever source changes
mirror/go.trigger: $(SRC)
mirror/go.owl: mirror/go.trigger
if [ $(MIR) = true ] && [ $(IMP) = true ]; then $(ROBOT) convert -I $(URIBASE)/go.owl -o [email protected] && mv [email protected] $@; fi
.PRECIOUS: mirror/%.owl
## ONTOLOGY: geno
## Copy of geno is re-downloaded whenever source changes
mirror/geno.trigger: $(SRC)
mirror/geno.owl: mirror/geno.trigger
if [ $(MIR) = true ] && [ $(IMP) = true ]; then $(ROBOT) convert -I $(URIBASE)/geno.owl -o [email protected] && mv [email protected] $@; fi
.PRECIOUS: mirror/%.owl
## ONTOLOGY: ncbitaxon
## Copy of ncbitaxon is re-downloaded whenever source changes
mirror/ncbitaxon.trigger: $(SRC)
mirror/ncbitaxon.owl: mirror/ncbitaxon.trigger
if [ $(MIR) = true ] && [ $(IMP) = true ]; then $(ROBOT) convert -I http://purl.obolibrary.org/obo/ncbitaxon/subsets/taxslim.owl -o [email protected] && mv [email protected] $@; fi
.PRECIOUS: mirror/%.owl
# ----------------------------------------
# Subsets
# ----------------------------------------
subsets/%.tsv: subsets/%.owl
$(ROBOT) query -f tsv -i $< -s ../sparql/labels.sparql $@
.PRECIOUS: subsets/%.tsv
subsets/%.owl: $(ONT).owl
owltools --use-catalog $< --extract-ontology-subset --fill-gaps --subset $* -o [email protected] && mv [email protected] $@ &&\
$(ROBOT) annotate --input $@ --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) -o [email protected] && mv [email protected] $@
.PRECIOUS: subsets/%.owl
subsets/%.obo: subsets/%.owl
$(ROBOT) convert --input $< --check false -f obo $(OBO_FORMAT_OPTIONS) -o [email protected] && grep -v ^owl-axioms [email protected] > $@ && rm [email protected]
subsets/%.json: subsets/%.owl
$(ROBOT) convert --input $< --check false -f json -o [email protected] &&\
jq -S 'walk(if type == "array" then sort else . end)' [email protected] > $@ && rm [email protected]
# ----------------------------------------
# Release
# ----------------------------------------
# copy from staging area (this directory) to top-level
.PHONY: release
release: $(ONT).owl $(ONT).obo
cp $^ $(RELEASEDIR) && cp imports/* $(RELEASEDIR)/imports
# ----------------------------------------
# Sparql queries: Q/C
# ----------------------------------------
# these live in the ../sparql directory, and have suffix -violation.sparql
# adding the name here will make the violation check live.
# NOTE: these will soon be phased out and replaced by robot-report
# run all violation checks
SPARQL_VALIDATION_QUERIES = $(foreach V,$(SPARQL_VALIDATION_CHECKS),$(SPARQLDIR)/$(V)-violation.sparql)
sparql_test: $(SRC) catalog-v001.xml | $(REPORTDIR)
ifneq ($(SPARQL_VALIDATION_QUERIES),)
$(ROBOT) verify --catalog catalog-v001.xml -i $< --queries $(SPARQL_VALIDATION_QUERIES) -O $(REPORTDIR)
endif
# ----------------------------------------
# ROBOT report
# ----------------------------------------
$(REPORTDIR)/%-obo-report.tsv: % | $(REPORTDIR)
$(ROBOT) report -i $< $(REPORT_LABEL) $(REPORT_PROFILE_OPTS) --fail-on $(REPORT_FAIL_ON) --print 5 -o $@
# ----------------------------------------
# Sparql queries: Exports
# ----------------------------------------
SPARQL_EXPORTS_ARGS = $(foreach V,$(SPARQL_EXPORTS),-s $(SPARQLDIR)/$(V).sparql $(REPORTDIR)/$(V).tsv)
# This combines all into one single command
.PHONY: all_reports_onestep
all_reports_onestep: $(SRC)
ifneq ($(SPARQL_EXPORTS_ARGS),)
$(ROBOT) query -f tsv -i $< $(SPARQL_EXPORTS_ARGS)
endif
# ----------------------------------------
# Release artefacts: export formats
# ----------------------------------------
$(ONT)-full.obo: $(ONT)-full.owl
$(ROBOT) convert --input $< --check false -f obo $(OBO_FORMAT_OPTIONS) -o [email protected] && grep -v ^owl-axioms [email protected] > $@ && rm [email protected]
$(ONT)-full.json: $(ONT)-full.owl
$(ROBOT) annotate --input $< --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) \
convert --check false -f json -o [email protected] &&\
jq -S 'walk(if type == "array" then sort else . end)' [email protected] > $@ && rm [email protected]
$(ONT)-base.obo: $(ONT)-base.owl
$(ROBOT) convert --input $< --check false -f obo $(OBO_FORMAT_OPTIONS) -o [email protected] && grep -v ^owl-axioms [email protected] > $@ && rm [email protected]
$(ONT)-base.json: $(ONT)-base.owl
$(ROBOT) annotate --input $< --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) \
convert --check false -f json -o [email protected] &&\
jq -S 'walk(if type == "array" then sort else . end)' [email protected] > $@ && rm [email protected]
# We always want a base - even if it is not explicitly configured..
# We always want a full release - even if it is not explicitly configured..
# ----------------------------------------
# Release artefacts: main release artefacts
# ----------------------------------------
$(ONT).owl: $(ONT)-full.owl
$(ROBOT) annotate --input $< --ontology-iri $(URIBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) \
convert -o [email protected] && mv [email protected] $@
$(ONT).obo: $(ONT).owl
$(ROBOT) convert --input $< --check false -f obo $(OBO_FORMAT_OPTIONS) -o [email protected] && grep -v ^owl-axioms [email protected] > $@ && rm [email protected]
$(ONT).json: $(ONT)-full.owl
$(ROBOT) annotate --input $< --ontology-iri $(URIBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) \
convert --check false -f json -o [email protected] &&\
jq -S 'walk(if type == "array" then sort else . end)' [email protected] > $@ && rm [email protected]
# -----------------------------------------------------
# Release artefacts: variants (base, full, simple, etc)
# -----------------------------------------------------
SHARED_ROBOT_COMMANDS = remove --term owl:Nothing
$(ONTOLOGYTERMS): $(SRC) $(OTHER_SRC)
touch $(ONTOLOGYTERMS) && \
$(ROBOT) query --use-graphs true -f csv -i $< --query ../sparql/chr_terms.sparql $@
# base: OTHER sources of interest, such as definitions owl
$(ONT)-base.owl: $(SRC) $(OTHER_SRC)
$(ROBOT) remove --input $< --select imports --trim false \
merge $(patsubst %, -i %, $(OTHER_SRC)) \
$(SHARED_ROBOT_COMMANDS) annotate --link-annotation http://purl.org/dc/elements/1.1/type http://purl.obolibrary.org/obo/IAO_8000001 \
--ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) \
--output [email protected] && mv [email protected] $@
# Full: The full artefacts with imports merged, reasoned
$(ONT)-full.owl: $(SRC) $(OTHER_SRC)
$(ROBOT) merge --input $< \
reason --reasoner ELK --equivalent-classes-allowed all --exclude-tautologies structural \
relax \
reduce -r ELK \
$(SHARED_ROBOT_COMMANDS) annotate --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) --output [email protected] && mv [email protected] $@
.PHONY: validate_idranges
validate_idranges:
amm $(SCRIPTSDIR)/validate_id_ranges.sc chr-idranges.owl
.PHONY: update_repo
update_repo:
sh $(SCRIPTSDIR)/update_repo.sh
include chr.Makefile