3 * Copyright (C) 2009-2020 SonarSource SA
4 * mailto:info AT sonarsource DOT com
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 3 of the License, or (at your option) any later version.
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public License
17 * along with this program; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
20 package org.sonar.server.platform.db.migration.version.v86;
22 import java.io.IOException;
23 import java.io.InputStream;
24 import java.io.InputStreamReader;
25 import java.io.StringReader;
26 import java.io.StringWriter;
27 import java.io.Writer;
28 import java.sql.SQLException;
29 import java.util.AbstractMap;
30 import java.util.ArrayList;
31 import java.util.Collection;
32 import java.util.LinkedHashMap;
33 import java.util.List;
35 import java.util.Objects;
37 import java.util.TreeSet;
38 import java.util.regex.Pattern;
39 import java.util.stream.Collectors;
40 import javax.annotation.CheckForNull;
41 import javax.annotation.Nullable;
42 import javax.xml.XMLConstants;
43 import javax.xml.parsers.ParserConfigurationException;
44 import javax.xml.parsers.SAXParser;
45 import javax.xml.parsers.SAXParserFactory;
46 import javax.xml.stream.XMLInputFactory;
47 import javax.xml.stream.XMLStreamException;
48 import javax.xml.transform.sax.SAXSource;
49 import javax.xml.validation.SchemaFactory;
50 import org.apache.commons.lang.StringUtils;
51 import org.codehaus.staxmate.SMInputFactory;
52 import org.codehaus.staxmate.in.SMHierarchicCursor;
53 import org.codehaus.staxmate.in.SMInputCursor;
54 import org.sonar.api.resources.Qualifiers;
55 import org.sonar.api.utils.System2;
56 import org.sonar.core.util.UuidFactory;
57 import org.sonar.db.Database;
58 import org.sonar.server.platform.db.migration.step.DataChange;
59 import org.sonar.server.platform.db.migration.step.Select;
60 import org.sonar.server.platform.db.migration.step.Upsert;
61 import org.xml.sax.InputSource;
62 import org.xml.sax.SAXException;
63 import org.xml.sax.SAXParseException;
64 import org.xml.sax.helpers.DefaultHandler;
66 import static java.lang.String.format;
67 import static java.nio.charset.StandardCharsets.UTF_8;
68 import static javax.xml.XMLConstants.FEATURE_SECURE_PROCESSING;
69 import static org.apache.commons.io.IOUtils.toInputStream;
70 import static org.apache.commons.lang.StringEscapeUtils.escapeXml;
71 import static org.apache.commons.lang.StringUtils.trim;
73 public class MigrateApplicationDefinitionsFromXmlToDb extends DataChange {
74 static final int TEXT_VALUE_MAX_LENGTH = 4000;
75 private static final String SELECT_APPLICATION_UUID_BY_KEY = "select uuid from projects where kee = ? and qualifier = 'APP'";
76 private static final String SELECT_PROJECTS_BY_KEYS = "select kee,uuid from projects where kee in (%s) and qualifier = 'TRK'";
77 private static final String UPDATE_INTERNAL_PROP_TEXT_VALUE = "update internal_properties set text_value = ?, clob_value = NULL where kee = ?";
78 private static final String UPDATE_INTERNAL_PROP_CLOB_VALUE = "update internal_properties set clob_value = ?, text_value = NULL where kee = ?";
79 private static final String VIEWS_DEF_KEY = "views.def";
81 private final UuidFactory uuidFactory;
82 private final System2 system;
84 public MigrateApplicationDefinitionsFromXmlToDb(Database db, UuidFactory uuidFactory, System2 system) {
87 this.uuidFactory = uuidFactory;
92 protected void execute(Context context) throws SQLException {
93 String xml = getViewsDefinition(context);
94 // skip migration if `views.def` does not exist in the db
100 Map<String, ViewXml.ViewDef> defs = ViewXml.parse(xml);
101 List<ViewXml.ViewDef> applications = defs.values()
103 .filter(v -> Qualifiers.APP.equals(v.getQualifier()))
104 .collect(Collectors.toList());
105 for (ViewXml.ViewDef app : applications) {
106 insertApplication(context, app);
108 cleanUpViewsDefinitionsXml(context, defs.values());
109 } catch (Exception e) {
110 throw new IllegalStateException("Failed to migrate views definitions property.", e);
115 private void insertApplication(Context context, ViewXml.ViewDef app) throws SQLException {
116 long now = system.now();
117 String applicationUuid = context.prepareSelect(SELECT_APPLICATION_UUID_BY_KEY)
118 .setString(1, app.getKey())
119 .get(r -> r.getString(1));
121 // ignore if application only exists in xml and not in the db. It will be removed from the xml at later stage of the migration.
122 if (applicationUuid == null) {
126 String queryParam = app.getProjects().stream().map(uuid -> "'" + uuid + "'").collect(Collectors.joining(","));
127 Map<String, String> projectUuidsByKeys = context.prepareSelect(format(SELECT_PROJECTS_BY_KEYS, queryParam))
128 .list(r -> new AbstractMap.SimpleEntry<>(r.getString(1), r.getString(2)))
130 .collect(Collectors.toMap(AbstractMap.SimpleEntry::getKey, AbstractMap.SimpleEntry::getValue));
132 insertApplicationProjects(context, app, applicationUuid, projectUuidsByKeys, now);
133 if (!app.getApplicationBranches().isEmpty()) {
134 insertApplicationBranchesProjects(context, app, applicationUuid, projectUuidsByKeys, now);
138 private void insertApplicationProjects(Context context, ViewXml.ViewDef app, String applicationUuid,
139 Map<String, String> projectUuidsByKeys, long createdTime) throws SQLException {
140 Upsert insertApplicationProjectsQuery = context.prepareUpsert("insert into " +
141 "app_projects(uuid, application_uuid, project_uuid, created_at) " +
142 "values (?, ?, ?, ?)");
143 for (String projectKey : app.getProjects()) {
144 String applicationProjectUuid = uuidFactory.create();
145 String projectUuid = projectUuidsByKeys.get(projectKey);
147 // ignore project if it does not exist anymore
148 if (projectUuid == null) {
152 insertApplicationProjectsQuery
153 .setString(1, applicationProjectUuid)
154 .setString(2, applicationUuid)
155 .setString(3, projectUuid)
156 .setLong(4, createdTime)
160 insertApplicationProjectsQuery.execute().commit();
163 private void insertApplicationBranchesProjects(Context context, ViewXml.ViewDef app, String applicationUuid,
164 Map<String, String> projectUuidsByKeys, long createdTime) throws SQLException {
165 Map<String, String> appBranchUuidByKey = context.prepareSelect("select kee,uuid from project_branches where project_uuid = ?")
166 .setString(1, applicationUuid)
167 .list(r -> new AbstractMap.SimpleEntry<>(r.getString(1), r.getString(2)))
169 .collect(Collectors.toMap(AbstractMap.SimpleEntry::getKey, AbstractMap.SimpleEntry::getValue));
171 Upsert insertApplicationsBranchProjectsQuery = context.prepareUpsert("insert into " +
172 "app_branch_project_branch(uuid, application_uuid, application_branch_uuid, project_uuid, project_branch_uuid, created_at) " +
173 "values (?, ?, ?, ?, ?, ?)");
174 boolean insert = false;
175 for (ViewXml.ApplicationBranchDef branch : app.getApplicationBranches()) {
176 String applicationBranchUuid = appBranchUuidByKey.get(branch.getKey());
177 // ignore application branch if it does not exist in the DB anymore
178 if (applicationBranchUuid == null) {
182 if (insertApplicationBranchProjects(context, branch, applicationUuid, applicationBranchUuid, projectUuidsByKeys, createdTime,
183 insertApplicationsBranchProjectsQuery)) {
189 insertApplicationsBranchProjectsQuery.execute().commit();
193 private boolean insertApplicationBranchProjects(Context context, ViewXml.ApplicationBranchDef branch, String applicationUuid,
194 String applicationBranchUuid, Map<String, String> projectUuidsByKeys, long createdTime,
195 Upsert insertApplicationsBranchProjectsQuery) throws SQLException {
197 boolean insert = false;
198 for (ViewXml.ApplicationProjectDef appProjDef : branch.getProjects()) {
199 String projectUuid = projectUuidsByKeys.get(appProjDef.getKey());
201 // ignore projects that do not exist in the DB anymore
202 if (projectUuid != null) {
203 String projectBranchUuid = context.prepareSelect("select uuid from project_branches where project_uuid = ? and kee = ?")
204 .setString(1, projectUuid)
205 .setString(2, appProjDef.getBranch())
206 .get(r -> r.getString(1));
208 // ignore project branches that do not exist in the DB anymore
209 if (projectBranchUuid != null) {
210 String applicationBranchProjectUuid = uuidFactory.create();
211 insertApplicationsBranchProjectsQuery
212 .setString(1, applicationBranchProjectUuid)
213 .setString(2, applicationUuid)
214 .setString(3, applicationBranchUuid)
215 .setString(4, projectUuid)
216 .setString(5, projectBranchUuid)
217 .setLong(6, createdTime)
228 private static String getViewsDefinition(DataChange.Context context) throws SQLException {
229 Select select = context.prepareSelect("select text_value,clob_value from internal_properties where kee=?");
230 select.setString(1, VIEWS_DEF_KEY);
231 return select.get(row -> {
232 String v = row.getString(1);
236 return row.getString(2);
241 private static void cleanUpViewsDefinitionsXml(Context context, Collection<ViewXml.ViewDef> definitions) throws SQLException, IOException {
242 definitions = definitions.stream()
243 .filter(d -> !"APP".equals(d.getQualifier()))
244 .collect(Collectors.toList());
246 StringWriter output = new StringWriter();
247 new ViewXml.ViewDefinitionsSerializer().write(definitions, output);
248 String value = output.toString();
249 String statement = UPDATE_INTERNAL_PROP_TEXT_VALUE;
250 if (mustBeStoredInClob(value)) {
251 statement = UPDATE_INTERNAL_PROP_CLOB_VALUE;
254 context.prepareUpsert(statement)
255 .setString(1, output.toString())
256 .setString(2, VIEWS_DEF_KEY)
261 private static boolean mustBeStoredInClob(String value) {
262 return value.length() > TEXT_VALUE_MAX_LENGTH;
265 private static class ViewXml {
266 static final String SCHEMA_VIEWS = "/static/views.xsd";
267 static final String VIEWS_HEADER_BARE = "<views>";
268 static final Pattern VIEWS_HEADER_BARE_PATTERN = Pattern.compile(VIEWS_HEADER_BARE);
269 static final String VIEWS_HEADER_FQ = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
270 + "<views xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xmlns=\"http://sonarsource.com/schema/views\">";
273 // nothing to do here
276 private static Map<String, ViewDef> parse(String xml) throws ParserConfigurationException, SAXException, IOException, XMLStreamException {
277 if (StringUtils.isEmpty(xml)) {
278 return new LinkedHashMap<>(0);
283 SMInputFactory inputFactory = initStax();
284 SMHierarchicCursor rootC = inputFactory.rootElementCursor(new StringReader(xml));
285 rootC.advance(); // <views>
286 SMInputCursor cursor = rootC.childElementCursor();
287 views = parseViewDefinitions(cursor);
289 Map<String, ViewDef> result = new LinkedHashMap<>(views.size());
290 for (ViewDef def : views) {
291 result.put(def.getKey(), def);
297 private static void validate(String xml) throws IOException, SAXException, ParserConfigurationException {
298 // Replace bare, namespace unaware header with fully qualified header (with schema declaration)
299 String fullyQualifiedXml = VIEWS_HEADER_BARE_PATTERN.matcher(xml).replaceFirst(VIEWS_HEADER_FQ);
300 try (InputStream xsd = MigrateApplicationDefinitionsFromXmlToDb.class.getResourceAsStream(SCHEMA_VIEWS)) {
301 InputSource viewsDefinition = new InputSource(new InputStreamReader(toInputStream(fullyQualifiedXml, UTF_8), UTF_8));
303 SchemaFactory saxSchemaFactory = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI);
304 saxSchemaFactory.setProperty(XMLConstants.ACCESS_EXTERNAL_SCHEMA, "");
305 saxSchemaFactory.setProperty(XMLConstants.ACCESS_EXTERNAL_DTD, "");
307 SAXParserFactory parserFactory = SAXParserFactory.newInstance();
308 parserFactory.setFeature(FEATURE_SECURE_PROCESSING, true);
309 parserFactory.setNamespaceAware(true);
310 parserFactory.setSchema(saxSchemaFactory.newSchema(new SAXSource(new InputSource(xsd))));
312 SAXParser saxParser = parserFactory.newSAXParser();
313 saxParser.setProperty(XMLConstants.ACCESS_EXTERNAL_DTD, "");
314 saxParser.setProperty(XMLConstants.ACCESS_EXTERNAL_SCHEMA, "");
315 saxParser.parse(viewsDefinition, new ViewsValidator());
319 private static List<ViewDef> parseViewDefinitions(SMInputCursor viewsCursor) throws XMLStreamException {
320 List<ViewDef> views = new ArrayList<>();
321 while (viewsCursor.getNext() != null) {
322 ViewDef viewDef = new ViewDef();
323 viewDef.setKey(viewsCursor.getAttrValue("key"));
324 viewDef.setDef(Boolean.parseBoolean(viewsCursor.getAttrValue("def")));
325 viewDef.setParent(viewsCursor.getAttrValue("parent"));
326 viewDef.setRoot(viewsCursor.getAttrValue("root"));
327 SMInputCursor viewCursor = viewsCursor.childElementCursor();
328 while (viewCursor.getNext() != null) {
329 String nodeName = viewCursor.getLocalName();
330 parseChildElement(viewDef, viewCursor, nodeName);
337 private static void parseChildElement(ViewDef viewDef, SMInputCursor viewCursor, String nodeName) throws XMLStreamException {
338 if (StringUtils.equals(nodeName, "name")) {
339 viewDef.setName(trim(viewCursor.collectDescendantText()));
340 } else if (StringUtils.equals(nodeName, "desc")) {
341 viewDef.setDesc(trim(viewCursor.collectDescendantText()));
342 } else if (StringUtils.equals(nodeName, "regexp")) {
343 viewDef.setRegexp(trim(viewCursor.collectDescendantText()));
344 } else if (StringUtils.equals(nodeName, "language")) {
345 viewDef.setLanguage(trim(viewCursor.collectDescendantText()));
346 } else if (StringUtils.equals(nodeName, "tag_key")) {
347 viewDef.setTagKey(trim(viewCursor.collectDescendantText()));
348 } else if (StringUtils.equals(nodeName, "tag_value")) {
349 viewDef.setTagValue(trim(viewCursor.collectDescendantText()));
350 } else if (StringUtils.equals(nodeName, "p")) {
351 viewDef.addProject(trim(viewCursor.collectDescendantText()));
352 } else if (StringUtils.equals(nodeName, "vw-ref")) {
353 viewDef.addReference(trim(viewCursor.collectDescendantText()));
354 } else if (StringUtils.equals(nodeName, "qualifier")) {
355 viewDef.setQualifier(trim(viewCursor.collectDescendantText()));
356 } else if (StringUtils.equals(nodeName, "branch")) {
357 parseBranch(viewDef, viewCursor);
358 } else if (StringUtils.equals(nodeName, "tagsAssociation")) {
359 parseTagsAssociation(viewDef, viewCursor);
363 private static void parseBranch(ViewDef def, SMInputCursor viewCursor) throws XMLStreamException {
364 List<ApplicationProjectDef> projects = new ArrayList<>();
365 String key = viewCursor.getAttrValue("key");
366 SMInputCursor projectCursor = viewCursor.childElementCursor();
367 while (projectCursor.getNext() != null) {
368 if (Objects.equals(projectCursor.getLocalName(), "p")) {
369 String branch = projectCursor.getAttrValue("branch");
370 String projectKey = trim(projectCursor.collectDescendantText());
371 projects.add(new ApplicationProjectDef().setKey(projectKey).setBranch(branch));
374 def.getApplicationBranches().add(new ApplicationBranchDef()
376 .setProjects(projects));
379 private static void parseTagsAssociation(ViewDef def, SMInputCursor viewCursor) throws XMLStreamException {
380 SMInputCursor projectCursor = viewCursor.childElementCursor();
381 while (projectCursor.getNext() != null) {
382 def.addTagAssociation(trim(projectCursor.collectDescendantText()));
386 private static SMInputFactory initStax() {
387 XMLInputFactory xmlFactory = XMLInputFactory.newInstance();
388 xmlFactory.setProperty(XMLInputFactory.IS_COALESCING, Boolean.TRUE);
389 xmlFactory.setProperty(XMLInputFactory.IS_NAMESPACE_AWARE, Boolean.FALSE);
390 // just so it won't try to load DTD in if there's DOCTYPE
391 xmlFactory.setProperty(XMLInputFactory.SUPPORT_DTD, Boolean.FALSE);
392 xmlFactory.setProperty(XMLInputFactory.IS_VALIDATING, Boolean.FALSE);
393 return new SMInputFactory(xmlFactory);
396 private static class ViewDef {
399 String parent = null;
405 List<String> p = new ArrayList<>();
407 List<String> vwRef = new ArrayList<>();
413 String regexp = null;
415 String language = null;
417 String tagKey = null;
419 String tagValue = null;
421 String qualifier = null;
423 List<ApplicationBranchDef> applicationBranches = new ArrayList<>();
425 Set<String> tagsAssociation = new TreeSet<>();
427 public String getKey() {
431 public String getParent() {
436 public String getRoot() {
440 public boolean isDef() {
444 public List<String> getProjects() {
448 public List<String> getReferences() {
452 public String getName() {
456 public String getDesc() {
461 public String getRegexp() {
466 public String getLanguage() {
471 public String getTagKey() {
476 public String getTagValue() {
481 public String getQualifier() {
485 public List<ApplicationBranchDef> getApplicationBranches() {
486 return applicationBranches;
489 public Set<String> getTagsAssociation() {
490 return tagsAssociation;
493 public ViewDef setKey(String key) {
498 public ViewDef setParent(String parent) {
499 this.parent = parent;
503 public ViewDef setRoot(@Nullable String root) {
508 public ViewDef setDef(boolean def) {
513 public ViewDef setProjects(List<String> projects) {
518 public ViewDef addProject(String project) {
523 public ViewDef removeProject(String project) {
524 this.p.remove(project);
528 public ViewDef setName(String name) {
533 public ViewDef setDesc(@Nullable String desc) {
538 public ViewDef setRegexp(@Nullable String regexp) {
539 this.regexp = regexp;
543 public ViewDef setLanguage(@Nullable String language) {
544 this.language = language;
548 public ViewDef setTagKey(@Nullable String tagKey) {
549 this.tagKey = tagKey;
553 public ViewDef setTagValue(@Nullable String tagValue) {
554 this.tagValue = tagValue;
558 public ViewDef addReference(String reference) {
559 this.vwRef.add(reference);
563 public ViewDef removeReference(String reference) {
564 this.vwRef.remove(reference);
568 public ViewDef setReferences(List<String> vwRef) {
573 public ViewDef setQualifier(@Nullable String qualifier) {
574 this.qualifier = qualifier;
578 public ViewDef setApplicationBranches(List<ApplicationBranchDef> branches) {
579 this.applicationBranches = branches;
583 public ViewDef addTagAssociation(String tag) {
584 this.tagsAssociation.add(tag);
588 public ViewDef setTagsAssociation(Set<String> tagsAssociation) {
589 this.tagsAssociation = tagsAssociation;
594 private static class ApplicationProjectDef {
595 private String key = null;
596 private String branch = null;
598 public String getKey() {
602 public ApplicationProjectDef setKey(String key) {
608 public String getBranch() {
612 public ApplicationProjectDef setBranch(@Nullable String branch) {
613 this.branch = branch;
618 private static class ApplicationBranchDef {
620 private String key = null;
621 private List<ApplicationProjectDef> p = new ArrayList<>();
623 public String getKey() {
627 public ApplicationBranchDef setKey(String key) {
632 public List<ApplicationProjectDef> getProjects() {
636 public ApplicationBranchDef setProjects(List<ApplicationProjectDef> p) {
642 private static final class ViewsValidator extends DefaultHandler {
644 public void error(SAXParseException exception) throws SAXException {
649 public void warning(SAXParseException exception) throws SAXException {
654 public void fatalError(SAXParseException exception) throws SAXException {
659 static class ViewDefinitionsSerializer {
661 public void write(Collection<ViewDef> definitions, Writer writer) throws IOException {
662 writer.append(VIEWS_HEADER_BARE);
664 for (ViewDef def : definitions) {
665 writer.append("<vw");
666 writer.append(" key=\"").append(escapeXml(def.getKey())).append("\"");
667 writer.append(" def=\"").append(Boolean.toString(def.isDef())).append("\"");
668 String parent = def.getParent();
669 if (parent != null) {
670 writer.append(" root=\"").append(escapeXml(def.getRoot())).append("\"");
671 writer.append(" parent=\"").append(escapeXml(parent)).append("\"");
675 writer.append("<name><![CDATA[").append(def.getName()).append("]]></name>");
676 writeOptionalElements(writer, def);
678 for (String project : def.getProjects()) {
679 writer.append("<p>").append(project).append("</p>");
681 for (String ref : def.getReferences()) {
682 writer.append("<vw-ref><![CDATA[").append(ref).append("]]></vw-ref>");
684 writeTagsAssociation(writer, def);
685 writer.append("</vw>");
688 writer.append("</views>");
691 private static void writeOptionalElements(Writer writer, ViewDef def) throws IOException {
692 String description = def.getDesc();
693 if (description != null) {
694 writer.append("<desc><![CDATA[").append(description).append("]]></desc>");
696 String regexp = def.getRegexp();
697 if (regexp != null) {
698 writer.append("<regexp><![CDATA[").append(regexp).append("]]></regexp>");
700 String language = def.getLanguage();
701 if (language != null) {
702 writer.append("<language><![CDATA[").append(language).append("]]></language>");
704 String customMeasureKey = def.getTagKey();
705 if (customMeasureKey != null) {
706 writer.append("<tag_key><![CDATA[").append(customMeasureKey).append("]]></tag_key>");
708 String customMeasureValue = def.getTagValue();
709 if (customMeasureValue != null) {
710 writer.append("<tag_value><![CDATA[").append(customMeasureValue).append("]]></tag_value>");
712 String qualifier = def.getQualifier();
713 if (qualifier != null) {
714 writer.append("<qualifier><![CDATA[").append(qualifier).append("]]></qualifier>");
718 private static void writeTagsAssociation(Writer writer, ViewDef definition) throws IOException {
719 Set<String> tagsAssociation = definition.getTagsAssociation();
720 if (tagsAssociation.isEmpty()) {
723 writer.append("<tagsAssociation>");
724 for (String tag : tagsAssociation) {
725 writer.append("<tag>").append(tag).append("</tag>");
727 writer.append("</tagsAssociation>");