forked from elastic/elasticsearch
-
Notifications
You must be signed in to change notification settings - Fork 0
/
build.gradle
449 lines (395 loc) · 17.6 KB
/
build.gradle
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import de.thetaphi.forbiddenapis.gradle.ForbiddenApisPlugin
import com.avast.gradle.dockercompose.tasks.ComposePull
import com.fasterxml.jackson.databind.JsonNode
import com.fasterxml.jackson.databind.ObjectMapper
import org.elasticsearch.gradle.Version
import org.elasticsearch.gradle.internal.BaseInternalPluginBuildPlugin
import org.elasticsearch.gradle.internal.ResolveAllDependencies
import org.elasticsearch.gradle.internal.info.BuildParams
import org.elasticsearch.gradle.util.GradleUtils
import org.gradle.plugins.ide.eclipse.model.AccessRule
import org.gradle.plugins.ide.eclipse.model.ProjectDependency
import java.nio.file.Files
import static java.nio.file.StandardCopyOption.REPLACE_EXISTING
import static org.elasticsearch.gradle.util.GradleUtils.maybeConfigure
buildscript {
repositories {
maven {
url 'https://jitpack.io'
}
mavenCentral()
}
}
plugins {
id 'lifecycle-base'
id 'elasticsearch.docker-support'
id 'elasticsearch.global-build-info'
id 'elasticsearch.build-complete'
id 'elasticsearch.build-scan'
id 'elasticsearch.jdk-download'
id 'elasticsearch.internal-distribution-download'
id 'elasticsearch.runtime-jdk-provision'
id 'elasticsearch.ide'
id 'elasticsearch.forbidden-dependencies'
id 'elasticsearch.local-distribution'
id 'elasticsearch.fips'
id 'elasticsearch.internal-testclusters'
id 'elasticsearch.run'
id 'elasticsearch.run-ccs'
id 'elasticsearch.release-tools'
id 'elasticsearch.versions'
}
/**
* This is a convenient method for declaring test artifact dependencies provided by the internal
* test artifact plugin. It replaces basically the longer dependency notation with explicit capability
* declaration like this:
*
* testImplementation(project(xpackModule('repositories-metering-api'))) {
* capabilities {
* requireCapability("org.elasticsearch.gradle:repositories-metering-api-test-artifacts")
* }
* }
*
* */
ext.testArtifact = { p, String name = "test" ->
def projectDependency = p.dependencies.create(p)
projectDependency.capabilities {
requireCapabilities("org.elasticsearch.gradle:${projectDependency.name}-${name}-artifacts")
};
}
class StepExpansion {
String templatePath
List<Version> versions
String variable
}
class ListExpansion {
List<Version> versions
String variable
}
// Filters out intermediate patch releases to reduce the load of CI testing
def filterIntermediatePatches = { List<Version> versions ->
versions.groupBy {"${it.major}.${it.minor}"}.values().collect {it.max()}
}
tasks.register("updateCIBwcVersions") {
def writeVersions = { File file, List<Version> versions ->
file.text = ""
file << "BWC_VERSION:\n"
versions.each {
file << " - \"$it\"\n"
}
}
def writeBuildkitePipeline = { String outputFilePath, String pipelineTemplatePath, List<ListExpansion> listExpansions, List<StepExpansion> stepExpansions = [] ->
def outputFile = file(outputFilePath)
def pipelineTemplate = file(pipelineTemplatePath)
def pipeline = pipelineTemplate.text
listExpansions.each { expansion ->
def listString = "[" + expansion.versions.collect { "\"${it}\"" }.join(", ") + "]"
pipeline = pipeline.replaceAll('\\$' + expansion.variable, listString)
}
stepExpansions.each { expansion ->
def steps = ""
expansion.versions.each {
steps += "\n" + file(expansion.templatePath).text.replaceAll('\\$BWC_VERSION', it.toString())
}
pipeline = pipeline.replaceAll(' *\\$' + expansion.variable, steps)
}
outputFile.text = "# This file is auto-generated. See ${pipelineTemplatePath}\n" + pipeline
}
// Writes a Buildkite pipelime from a template, and replaces $BWC_LIST with an array of versions
// Useful for writing a list of versions in a matrix configuration
def expandBwcList = { String outputFilePath, String pipelineTemplatePath, List<Version> versions ->
writeBuildkitePipeline(outputFilePath, pipelineTemplatePath, [new ListExpansion(versions: versions, variable: "BWC_LIST")])
}
// Writes a Buildkite pipeline from a template, and replaces $BWC_STEPS with a list of steps, one for each version
// Useful when you need to configure more versions than are allowed in a matrix configuration
def expandBwcSteps = { String outputFilePath, String pipelineTemplatePath, String stepTemplatePath, List<Version> versions ->
writeBuildkitePipeline(outputFilePath, pipelineTemplatePath, [], [new StepExpansion(templatePath: stepTemplatePath, versions: versions, variable: "BWC_STEPS")])
}
doLast {
writeVersions(file(".ci/bwcVersions"), filterIntermediatePatches(BuildParams.bwcVersions.allIndexCompatible))
writeVersions(file(".ci/snapshotBwcVersions"), filterIntermediatePatches(BuildParams.bwcVersions.unreleasedIndexCompatible))
expandBwcList(
".buildkite/pipelines/intake.yml",
".buildkite/pipelines/intake.template.yml",
filterIntermediatePatches(BuildParams.bwcVersions.unreleasedIndexCompatible)
)
writeBuildkitePipeline(
".buildkite/pipelines/periodic.yml",
".buildkite/pipelines/periodic.template.yml",
[
new ListExpansion(versions: filterIntermediatePatches(BuildParams.bwcVersions.unreleasedIndexCompatible), variable: "BWC_LIST"),
],
[
new StepExpansion(templatePath: ".buildkite/pipelines/periodic.bwc.template.yml", versions: filterIntermediatePatches(BuildParams.bwcVersions.allIndexCompatible), variable: "BWC_STEPS"),
]
)
expandBwcSteps(
".buildkite/pipelines/periodic-packaging.yml",
".buildkite/pipelines/periodic-packaging.template.yml",
".buildkite/pipelines/periodic-packaging.bwc.template.yml",
filterIntermediatePatches(BuildParams.bwcVersions.allIndexCompatible)
)
}
}
tasks.register("verifyVersions") {
def verifyCiYaml = { File file, List<Version> versions ->
String ciYml = file.text
versions.each {
if (ciYml.contains("\"$it\"\n") == false) {
throw new Exception("${file} is outdated, run `./gradlew updateCIBwcVersions` and check in the results")
}
}
}
doLast {
if (gradle.startParameter.isOffline()) {
throw new GradleException("Must run in online mode to verify versions")
}
// Read the list from maven central.
// Fetch the metadata and parse the xml into Version instances because it's more straight forward here
// rather than bwcVersion ( VersionCollection ).
new URL('https://repo1.maven.org/maven2/org/elasticsearch/elasticsearch/maven-metadata.xml').openStream().withStream { s ->
BuildParams.bwcVersions.compareToAuthoritative(
new XmlParser().parse(s)
.versioning.versions.version
.collect { it.text() }.findAll { it ==~ /\d+\.\d+\.\d+/ }
.collect { Version.fromString(it) }
)
}
verifyCiYaml(file(".ci/bwcVersions"), filterIntermediatePatches(BuildParams.bwcVersions.allIndexCompatible))
verifyCiYaml(file(".ci/snapshotBwcVersions"), BuildParams.bwcVersions.unreleasedIndexCompatible)
// Make sure backport bot config file is up to date
JsonNode backportConfig = new ObjectMapper().readTree(file(".backportrc.json"))
BuildParams.bwcVersions.forPreviousUnreleased { unreleasedVersion ->
boolean valid = backportConfig.get("targetBranchChoices").elements().any { branchChoice ->
if (branchChoice.isObject()) {
return branchChoice.get("name").textValue() == unreleasedVersion.branch
} else {
return branchChoice.textValue() == unreleasedVersion.branch
}
}
if (valid == false) {
throw new GradleException("No branch choice exists for development branch ${unreleasedVersion.branch} in .backportrc.json.")
}
}
String versionMapping = backportConfig.get("branchLabelMapping").fields().find { it.value.textValue() == 'main' }.key
String expectedMapping = "^v${versions.elasticsearch.replaceAll('-SNAPSHOT', '')}\$"
if (versionMapping != expectedMapping) {
throw new GradleException(
"Backport label mapping for branch 'main' is '${versionMapping}' but should be " +
"'${expectedMapping}'. Update .backportrc.json."
)
}
}
}
/*
* When adding backcompat behavior that spans major versions, temporarily
* disabling the backcompat tests is necessary. This flag controls
* the enabled state of every bwc task. It should be set back to true
* after the backport of the backcompat code is complete.
*/
boolean bwc_tests_enabled = true
// place a PR link here when committing bwc changes:
String bwc_tests_disabled_issue = ""
if (bwc_tests_enabled == false) {
if (bwc_tests_disabled_issue.isEmpty()) {
throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false")
}
println "========================= WARNING ========================="
println " Backwards compatibility tests are disabled!"
println "See ${bwc_tests_disabled_issue}"
println "==========================================================="
}
if (project.gradle.startParameter.taskNames.any { it.startsWith("checkPart") || it == 'functionalTests' }) {
// Disable BWC tests for checkPart* tasks and platform support tests as it's expected that this will run on it's own check
bwc_tests_enabled = false
}
subprojects { proj ->
apply plugin: 'elasticsearch.base'
}
allprojects {
// We disable this plugin for now till we shaked out the issues we see
// e.g. see https://github.com/elastic/elasticsearch/issues/72169
// apply plugin:'elasticsearch.internal-test-rerun'
plugins.withType(BaseInternalPluginBuildPlugin).whenPluginAdded {
project.dependencies {
compileOnly project(":server")
testImplementation project(":test:framework")
}
}
// injecting groovy property variables into all projects
project.ext {
// for ide hacks...
isEclipse = providers.systemProperty("eclipse.launcher").isPresent() || // Detects gradle launched from Eclipse's IDE
providers.systemProperty("eclipse.application").isPresent() || // Detects gradle launched from the Eclipse compiler server
gradle.startParameter.taskNames.contains('eclipse') || // Detects gradle launched from the command line to do eclipse stuff
gradle.startParameter.taskNames.contains('cleanEclipse')
}
ext.bwc_tests_enabled = bwc_tests_enabled
// eclipse configuration
apply plugin: 'elasticsearch.eclipse'
/*
* Allow accessing com/sun/net/httpserver in projects that have
* configured forbidden apis to allow it.
*/
plugins.withType(ForbiddenApisPlugin) {
eclipse.classpath.file.whenMerged { classpath ->
if (false == forbiddenApisTest.bundledSignatures.contains('jdk-non-portable')) {
classpath.entries
.findAll { it.kind == "con" && it.toString().contains("org.eclipse.jdt.launching.JRE_CONTAINER") }
.each {
it.accessRules.add(new AccessRule("accessible", "com/sun/net/httpserver/*"))
}
}
}
}
tasks.register('resolveAllDependencies', ResolveAllDependencies) {
configs = project.configurations
resolveJavaToolChain = true
if (project.path.contains("fixture")) {
dependsOn tasks.withType(ComposePull)
}
}
plugins.withId('lifecycle-base') {
if (project.path.startsWith(":x-pack:")) {
if (project.path.contains("security") || project.path.contains(":ml")) {
tasks.register('checkPart4') { dependsOn 'check' }
} else if (project.path == ":x-pack:plugin" || project.path.contains("ql") || project.path.contains("smoke-test")) {
tasks.register('checkPart3') { dependsOn 'check' }
} else if (project.path.contains("multi-node")) {
tasks.register('checkPart5') { dependsOn 'check' }
} else {
tasks.register('checkPart2') { dependsOn 'check' }
}
} else {
tasks.register('checkPart1') { dependsOn 'check' }
}
tasks.register('functionalTests') { dependsOn 'check' }
}
/*
* Remove assemble/dependenciesInfo on all qa projects because we don't
* need to publish artifacts for them.
*/
if (project.name.equals('qa') || project.path.contains(':qa:')) {
maybeConfigure(project.tasks, 'assemble') {
it.enabled = false
}
maybeConfigure(project.tasks, 'dependenciesInfo') {
it.enabled = false
}
}
project.afterEvaluate {
// Ensure similar tasks in dependent projects run first. The projectsEvaluated here is
// important because, while dependencies.all will pickup future dependencies,
// it is not necessarily true that the task exists in both projects at the time
// the dependency is added.
if (project.path == ':test:framework') {
// :test:framework:test cannot run before and after :server:test
return
}
tasks.matching { it.name.equals('integTest') }.configureEach { integTestTask ->
integTestTask.mustRunAfter tasks.matching { it.name.equals("test") }
}
/* configurations.matching { it.canBeResolved }.all { Configuration configuration ->
dependencies.matching { it instanceof ProjectDependency }.all { ProjectDependency dep ->
Project upstreamProject = dep.dependencyProject
if (project.path != upstreamProject?.path) {
for (String taskName : ['test', 'integTest']) {
project.tasks.matching { it.name == taskName }.configureEach { task ->
task.shouldRunAfter(upstreamProject.tasks.matching { upStreamTask -> upStreamTask.name == taskName })
}
}
}
}
}*/
}
apply plugin: 'elasticsearch.formatting'
}
tasks.register("verifyBwcTestsEnabled") {
doLast {
if (bwc_tests_enabled == false) {
throw new GradleException('Bwc tests are disabled. They must be re-enabled after completing backcompat behavior backporting.')
}
}
}
tasks.register("branchConsistency") {
description 'Ensures this branch is internally consistent. For example, that versions constants match released versions.'
group 'Verification'
dependsOn ":verifyVersions", ":verifyBwcTestsEnabled"
}
tasks.named("wrapper").configure {
distributionType = 'ALL'
doLast {
// copy wrapper properties file to build-tools-internal to allow seamless idea integration
def file = new File("build-tools-internal/gradle/wrapper/gradle-wrapper.properties")
Files.copy(wrapper.getPropertiesFile().toPath(), file.toPath(), REPLACE_EXISTING)
// copy wrapper properties file to plugins/examples to allow seamless idea integration
def examplePluginsWrapperProperties = new File("plugins/examples/gradle/wrapper/gradle-wrapper.properties")
Files.copy(wrapper.getPropertiesFile().toPath(), examplePluginsWrapperProperties.toPath(), REPLACE_EXISTING)
// Update build-tools to reflect the Gradle upgrade
// TODO: we can remove this once we have tests to make sure older versions work.
project.file('build-tools-internal/src/main/resources/minimumGradleVersion').text = gradleVersion
println "Updated minimum Gradle Version"
}
}
gradle.projectsEvaluated {
// Having the same group and name for distinct projects causes Gradle to consider them equal when resolving
// dependencies leading to hard to debug failures. Run a check across all project to prevent this from happening.
// see: https://github.com/gradle/gradle/issues/847
Map coordsToProject = [:]
project.allprojects.forEach { p ->
String coords = "${p.group}:${p.name}"
if (false == coordsToProject.putIfAbsent(coords, p)) {
throw new GradleException(
"Detected that two projects: ${p.path} and ${coordsToProject[coords].path} " +
"have the same name and group: ${coords}. " +
"This doesn't currently work correctly in Gradle, see: " +
"https://github.com/gradle/gradle/issues/847"
)
}
}
}
tasks.named("precommit") {
dependsOn gradle.includedBuild('build-tools').task(':precommit')
dependsOn gradle.includedBuild('build-tools-internal').task(':precommit')
}
tasks.named("checkPart1").configure {
dependsOn gradle.includedBuild('build-tools').task(':check')
dependsOn gradle.includedBuild('build-tools-internal').task(':check')
}
tasks.named("assemble").configure {
dependsOn gradle.includedBuild('build-tools').task(':assemble')
}
tasks.named("cleanEclipse").configure {
dependsOn gradle.includedBuild('build-conventions').task(':cleanEclipse')
dependsOn gradle.includedBuild('build-tools').task(':cleanEclipse')
dependsOn gradle.includedBuild('build-tools-internal').task(':cleanEclipse')
}
tasks.named("eclipse").configure {
dependsOn gradle.includedBuild('build-conventions').task(':eclipse')
dependsOn gradle.includedBuild('build-tools').task(':eclipse')
dependsOn gradle.includedBuild('build-tools-internal').task(':eclipse')
}
tasks.register("buildReleaseArtifacts").configure {
group = 'build'
description = 'Builds all artifacts required for release manager'
dependsOn allprojects.findAll {
it.path.startsWith(':distribution:docker') == false
&& it.path.startsWith(':ml-cpp') == false
&& it.path.startsWith(':distribution:bwc') == false
&& it.path.startsWith(':test:fixture') == false
}
.collect { GradleUtils.findByName(it.tasks, 'assemble') }
.findAll { it != null }
}
tasks.register("spotlessApply").configure {
dependsOn gradle.includedBuild('build-tools').task(':spotlessApply')
dependsOn gradle.includedBuild('build-tools').task(':reaper:spotlessApply')
dependsOn gradle.includedBuild('build-tools-internal').task(':spotlessApply')
}