You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

build.gradle 12KB

6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
SONAR-12686 upgrade es client to 7.9.3 and move to HTTP - add should minimum match eq 1 to user index queries ES 7.X changed behaviour in case filter query with bool it defaults to '0' https://www.elastic.co/guide/en/elasticsearch/reference/7.x/breaking-changes-7.0.html#_the_filter_context_has_been_removed - fix issue index routing param ES 7.X helped discover this bug as new setting has been auto configured which is 'index.number_of_routing_shards'. This has changed how documents are distributed across shards depending on how many shards the index has. Without that change issues docs has been incorrectly routed to the same shard hash as projects and it worked no matter what routing key you used projectUuid or auth_projectUuid. - update ngram and edge_ngram names to match with es 7.x nGram and edgeNgram has been deprecated in favour of ngram and edge_ngram https://www.elastic.co/guide/en/elasticsearch/reference/current/breaking-changes-7.0.html#deprecated-ngram-edgengram-token-filter-cannot-be-used - remove `_all : enabled` usage from UT This field was already deprecated in 6.X, now it has been removed. https://www.elastic.co/guide/en/elasticsearch/reference/current/breaking-changes-7.0.html#all-meta-field-removed - add Elasticsearch High Level REST client dependency - use sonar.search.port for ES HTTP - main process use ES Rest client to check ES status - sonar.cluster.search.hosts has HTTP ports on APP nodes also sonar.search.port and sonar.search.host MUST be configured on each Search node with the host and HTTP port of the current node - use Elasticsearch high level rest client - use in EsTester - use as primary es client - use indices api to get all indices name instead of cluster api - use cluster health api to check cluster state - support raw requests for 'nodes/_stats' and '_cluster/stats' - support raw requests for 'indices/_stats' - leave netty4plugin as testCompile dependency it is used in UTs - all ES non-test calls go through EsClient class - add rest client ES profiling
4 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
6 years ago
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358
  1. import groovy.json.JsonOutput
  2. import groovy.json.JsonSlurper
  3. import org.apache.tools.ant.filters.ReplaceTokens
  4. plugins {
  5. id "com.github.hierynomus.license-report"
  6. id "com.github.johnrengelman.shadow"
  7. id "de.undercouch.download"
  8. id "org.cyclonedx.bom"
  9. }
  10. sonarqube {
  11. properties {
  12. property 'sonar.projectName', "${projectTitle} :: Application"
  13. }
  14. }
  15. configurations {
  16. zip
  17. scanner
  18. web
  19. shutdowner
  20. jdbc_mssql {
  21. transitive = false
  22. }
  23. jdbc_postgresql {
  24. transitive = false
  25. }
  26. jdbc_h2 {
  27. transitive = false
  28. }
  29. bundledPlugin {
  30. transitive = false
  31. }
  32. bundledPlugin_deps {
  33. extendsFrom bundledPlugin
  34. transitive = true
  35. }
  36. appLicenses.extendsFrom(compile, web, scanner, jdbc_mssql, jdbc_postgresql, jdbc_h2)
  37. cyclonedx
  38. }
  39. jar.enabled = false
  40. shadowJar {
  41. archiveBaseName = 'sonar-application'
  42. archiveClassifier = null
  43. mergeServiceFiles()
  44. manifest {
  45. attributes('Main-Class': 'org.sonar.application.App')
  46. }
  47. }
  48. dependencies {
  49. // please keep list ordered
  50. compile 'org.slf4j:slf4j-api'
  51. compile 'org.elasticsearch.client:elasticsearch-rest-high-level-client'
  52. compile 'org.sonarsource.api.plugin:sonar-plugin-api'
  53. compile project(':server:sonar-ce')
  54. compile project(':server:sonar-main')
  55. compile project(':server:sonar-process')
  56. compile project(':server:sonar-webserver')
  57. compile project(':sonar-core')
  58. compile project(':sonar-plugin-api-impl')
  59. compileOnly 'com.google.code.findbugs:jsr305'
  60. scanner project(path: ':sonar-scanner-engine-shaded', configuration: 'shadow')
  61. cyclonedx project(path: ':sonar-scanner-engine-shaded')
  62. web project(':server:sonar-web')
  63. shutdowner project(':sonar-shutdowner')
  64. jdbc_h2 'com.h2database:h2'
  65. jdbc_mssql 'com.microsoft.sqlserver:mssql-jdbc'
  66. jdbc_postgresql 'org.postgresql:postgresql'
  67. }
  68. // declare dependencies in configuration bundledPlugin to be packaged in lib/extensions
  69. apply from: 'bundled_plugins.gradle'
  70. //verify if sonar.properties files does not have any external input
  71. task verifySonarProperties(type: Verify) {
  72. def propertiesFile = file('src/main/assembly/conf/sonar.properties')
  73. propertiesFile.withReader { reader ->
  74. def line
  75. while ((line = reader.readLine()) != null) {
  76. if (!line.startsWith('#') && !line.isEmpty()) {
  77. throw new GradleException('sonar.properties file by default must not provide any user configuration.')
  78. }
  79. }
  80. }
  81. }
  82. task verifyElasticSearchDownload(type: Verify) {
  83. src new File(buildDir, "$elasticsearchDownloadUrlFile")
  84. algorithm 'SHA-512'
  85. checksum elasticsearchDownloadSha512
  86. }
  87. task downloadElasticSearch(type: Download) {
  88. def artifactoryUsername = System.env.'ARTIFACTORY_PRIVATE_USERNAME' ?: (project.hasProperty('artifactoryUsername') ? project.getProperty('artifactoryUsername') : '')
  89. def artifactoryPassword = System.env.'ARTIFACTORY_PRIVATE_PASSWORD' ?: (project.hasProperty('artifactoryPassword') ? project.getProperty('artifactoryPassword') : '')
  90. if (artifactoryUsername && artifactoryPassword) {
  91. src "$elasticsearchDownloadRepoxUrlPath$elasticsearchDownloadUrlFile"
  92. username artifactoryUsername
  93. password artifactoryPassword
  94. } else {
  95. src "$elasticsearchDownloadUrlPath$elasticsearchDownloadUrlFile"
  96. }
  97. dest "$buildDir/$elasticsearchDownloadUrlFile"
  98. onlyIfModified true
  99. finalizedBy verifyElasticSearchDownload
  100. }
  101. downloadLicenses {
  102. dependencyConfiguration = 'appLicenses'
  103. }
  104. task zip(type: Zip, dependsOn: [configurations.compileClasspath, tasks.downloadLicenses, downloadElasticSearch, verifyElasticSearchDownload]) {
  105. duplicatesStrategy DuplicatesStrategy.EXCLUDE
  106. def archiveDir = "sonarqube-$project.version"
  107. into("${archiveDir}/") {
  108. from(tasks.downloadLicenses.outputs) {
  109. include 'dependency-license.json'
  110. eachFile { jsonFile ->
  111. def json = new JsonSlurper().parse(jsonFile.file)
  112. json.dependencies.each { dependency ->
  113. if (dependency.licenses.size() > 1) {
  114. def idx = dependency.licenses.findIndexOf { it.name == "Elastic License 2.0" }
  115. if (idx >= 0) {
  116. dependency.licenses = [dependency.licenses[idx]]
  117. }
  118. }
  119. }
  120. json.dependencies.sort { it.name }
  121. def jsonText = JsonOutput.toJson(json)
  122. jsonFile.file.text = JsonOutput.prettyPrint(jsonText)
  123. }
  124. }
  125. from(file('src/main/assembly')) {
  126. exclude 'conf/sonar.properties'
  127. exclude 'bin/windows-x86-64/lib/SonarServiceWrapper.xml'
  128. exclude 'bin/windows-x86-64/StartSonar.bat'
  129. exclude 'elasticsearch-patch'
  130. exclude 'bin/linux-x86-64/sonar.sh'
  131. exclude 'bin/macosx-universal-64/sonar.sh'
  132. }
  133. }
  134. from(tarTree(downloadElasticSearch.dest)) {
  135. eachFile { fcd ->
  136. def path = fcd.relativePath.segments - fcd.relativeSourcePath.segments + fcd.relativeSourcePath.segments.drop(1)
  137. fcd.relativePath = new RelativePath(true, *path)
  138. }
  139. into("${archiveDir}/elasticsearch")
  140. filesMatching('**/bin/elasticsearch-env') {
  141. // to avoid warning logs
  142. filter { line -> line.replaceAll('echo "warning: no-jdk distributions.*', ':') }
  143. }
  144. // elasticsearch script will be replaced by patched version below
  145. exclude '**/bin/elasticsearch'
  146. exclude '**/bin/elasticsearch-certgen'
  147. exclude '**/bin/elasticsearch-certutil'
  148. exclude '**/bin/elasticsearch-cli'
  149. exclude '**/bin/elasticsearch-croneval'
  150. exclude '**/bin/elasticsearch-geoip'
  151. exclude '**/bin/elasticsearch-keystore'
  152. exclude '**/bin/elasticsearch-migrate'
  153. exclude '**/bin/elasticsearch-node'
  154. exclude '**/bin/elasticsearch-plugin'
  155. exclude '**/bin/elasticsearch-saml-metadata'
  156. exclude '**/bin/elasticsearch-service-tokens'
  157. exclude '**/bin/elasticsearch-setup-passwords'
  158. exclude '**/bin/elasticsearch-shard'
  159. exclude '**/bin/elasticsearch-sql-cli*'
  160. exclude '**/bin/elasticsearch-syskeygen'
  161. exclude '**/bin/elasticsearch-users'
  162. exclude '**/bin/x-pack-env'
  163. exclude '**/bin/x-pack-security-env'
  164. exclude '**/bin/x-pack-watcher-env'
  165. exclude '**/jdk/**'
  166. exclude '**/lib/tools/**'
  167. exclude '**/modules/aggs-matrix-stats/**'
  168. exclude '**/modules/constant-keyword/**'
  169. exclude '**/modules/frozen-indices/**'
  170. exclude '**/modules/ingest-common/**'
  171. exclude '**/modules/ingest-geoip/**'
  172. exclude '**/modules/ingest-user-agent/**'
  173. exclude '**/modules/kibana/**'
  174. exclude '**/modules/lang-expression/**'
  175. exclude '**/modules/lang-mustache/**'
  176. exclude '**/modules/legacy-geo/**'
  177. exclude '**/modules/mapper-extras/**'
  178. exclude '**/modules/mapper-version/**'
  179. exclude '**/modules/percolator/**'
  180. exclude '**/modules/rank-eval/**'
  181. exclude '**/modules/repositories-metering-api/**'
  182. exclude '**/modules/repository-encrypted/**'
  183. exclude '**/modules/repository-url/**'
  184. exclude '**/modules/runtime-fields-common/**'
  185. exclude '**/modules/search-business-rules/**'
  186. exclude '**/modules/searchable-snapshots/**'
  187. exclude '**/modules/snapshot-repo-test-kit/**'
  188. exclude '**/modules/spatial/**'
  189. exclude '**/modules/transform/**'
  190. exclude '**/modules/unsigned-long/**'
  191. exclude '**/modules/vector-tile/**'
  192. exclude '**/modules/vectors/**'
  193. exclude '**/modules/wildcard/**'
  194. exclude '**/modules/x-pack-*/**'
  195. includeEmptyDirs = false
  196. }
  197. into("${archiveDir}/conf/") {
  198. from file('src/main/assembly/conf/sonar.properties')
  199. filter(ReplaceTokens, tokens: [
  200. 'searchDefaultHeapSize': '512MB',
  201. 'searchJavaOpts' : '-Xmx512m -Xms512m -XX:MaxDirectMemorySize=256m -XX:+HeapDumpOnOutOfMemoryError',
  202. 'ceDefaultHeapSize' : '512MB',
  203. 'ceJavaOpts' : '-Xmx512m -Xms128m -XX:+HeapDumpOnOutOfMemoryError',
  204. 'webDefaultHeapSize' : '512MB',
  205. 'webJavaOpts' : '-Xmx512m -Xms128m -XX:+HeapDumpOnOutOfMemoryError'
  206. ])
  207. }
  208. into("${archiveDir}/bin/linux-x86-64/") {
  209. from file('src/main/assembly/bin/linux-x86-64/sonar.sh')
  210. filter(ReplaceTokens, tokens: [
  211. 'sqversion': version
  212. ])
  213. }
  214. into("${archiveDir}/bin/macosx-universal-64/") {
  215. from file('src/main/assembly/bin/macosx-universal-64/sonar.sh')
  216. filter(ReplaceTokens, tokens: [
  217. 'sqversion': version
  218. ])
  219. }
  220. into("${archiveDir}/bin/windows-x86-64/") {
  221. from file('src/main/assembly/bin/windows-x86-64/StartSonar.bat')
  222. filter(ReplaceTokens, tokens: [
  223. 'sqversion': version
  224. ])
  225. }
  226. into("${archiveDir}/bin/windows-x86-64/lib/") {
  227. from file('src/main/assembly/bin/windows-x86-64/lib/SonarServiceWrapper.xml')
  228. filter(ReplaceTokens, tokens: [
  229. 'sqversion': version
  230. ])
  231. }
  232. into("${archiveDir}/elasticsearch/") {
  233. from file('src/main/assembly/elasticsearch-patch')
  234. include 'bin/elasticsearch'
  235. }
  236. // Create the empty dir (plugins) required by elasticsearch
  237. into("${archiveDir}/elasticsearch/") {
  238. from "$buildDir/elasticsearch"
  239. }
  240. into("${archiveDir}/lib/extensions/") {
  241. from configurations.bundledPlugin
  242. }
  243. into("${archiveDir}/lib/scanner/") {
  244. from configurations.scanner
  245. }
  246. into("${archiveDir}/lib/") {
  247. from shadowJar
  248. }
  249. into("${archiveDir}/web/") {
  250. duplicatesStrategy DuplicatesStrategy.FAIL
  251. // FIXME use configurations.web with correct artifacts
  252. from(tasks.getByPath(':server:sonar-web:yarn_run').outputs) { a ->
  253. if (official) {
  254. project(':private:branding').fileTree('src').visit { b ->
  255. if (!b.isDirectory()) {
  256. a.exclude b.relativePath.toString()
  257. }
  258. }
  259. }
  260. }
  261. if (official) {
  262. from project(':private:branding').file('src')
  263. }
  264. }
  265. into("${archiveDir}/lib/jdbc/mssql/") {
  266. from configurations.jdbc_mssql
  267. }
  268. into("${archiveDir}/lib/jdbc/postgresql/") {
  269. from configurations.jdbc_postgresql
  270. }
  271. into("${archiveDir}/lib/jdbc/h2/") {
  272. from configurations.jdbc_h2
  273. }
  274. into("${archiveDir}/lib/") {
  275. from configurations.shutdowner
  276. }
  277. }
  278. // Create the empty dir required by elasticsearch
  279. zip.doFirst {
  280. new File(buildDir, 'elasticsearch/plugins').mkdirs()
  281. }
  282. // Check the size of the archive
  283. zip.doLast {
  284. def minLength = 272000000
  285. def maxLength = 297000000
  286. def length = archiveFile.get().asFile.length()
  287. if (length < minLength)
  288. throw new GradleException("${archiveFileName.get()} size ($length) too small. Min is $minLength")
  289. if (length > maxLength)
  290. throw new GradleException("${destinationDirectory.get()}/${archiveFileName.get()} size ($length) too large. Max is $maxLength")
  291. }
  292. assemble.dependsOn zip
  293. // the script start.sh unpacks OSS distribution into $buildDir/distributions/sonarqube-oss.
  294. // This directory should be deleted when the zip is changed.
  295. task cleanLocalUnzippedDir(dependsOn: zip) {
  296. def unzippedDir = file("$buildDir/distributions/sonarqube-$version")
  297. inputs.files(file("$buildDir/distributions/sonar-application-${version}.zip"))
  298. outputs.upToDateWhen { true }
  299. doLast {
  300. println("delete directory ${unzippedDir}")
  301. project.delete(unzippedDir)
  302. }
  303. }
  304. assemble.dependsOn cleanLocalUnzippedDir
  305. artifacts { zip zip }
  306. artifactoryPublish.skip = false
  307. def bomFile = layout.buildDirectory.file('reports/bom.json')
  308. cyclonedxBom {
  309. includeConfigs += ["runtimeClasspath", "web", "shutdowner", "jdbc_mssql", "jdbc_postgresql", "jdbc_h2", "bundledPlugin_deps",
  310. "cyclonedx"]
  311. outputs.file bomFile
  312. outputs.upToDateWhen { false }
  313. }
  314. def bomArtifact = artifacts.add('archives', bomFile.get().asFile) {
  315. type 'json'
  316. classifier 'cyclonedx'
  317. builtBy 'cyclonedxBom'
  318. }
  319. publishing {
  320. publications {
  321. mavenJava(MavenPublication) {
  322. artifact zip
  323. }
  324. mavenJava(MavenPublication) {
  325. artifact bomArtifact
  326. }
  327. }
  328. }