3 * Copyright (C) 2009-2021 SonarSource SA
4 * mailto:info AT sonarsource DOT com
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 3 of the License, or (at your option) any later version.
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public License
17 * along with this program; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
20 package org.sonar.server.platform.db.migration.version.v86;
22 import java.io.IOException;
23 import java.io.InputStream;
24 import java.io.InputStreamReader;
25 import java.io.StringReader;
26 import java.io.StringWriter;
27 import java.io.Writer;
28 import java.sql.SQLException;
29 import java.util.AbstractMap;
30 import java.util.ArrayList;
31 import java.util.Collection;
32 import java.util.LinkedHashMap;
33 import java.util.List;
35 import java.util.Objects;
37 import java.util.TreeSet;
38 import java.util.regex.Pattern;
39 import java.util.stream.Collectors;
40 import javax.annotation.CheckForNull;
41 import javax.annotation.Nullable;
42 import javax.xml.XMLConstants;
43 import javax.xml.parsers.ParserConfigurationException;
44 import javax.xml.parsers.SAXParser;
45 import javax.xml.parsers.SAXParserFactory;
46 import javax.xml.stream.XMLInputFactory;
47 import javax.xml.stream.XMLStreamException;
48 import javax.xml.transform.sax.SAXSource;
49 import javax.xml.validation.SchemaFactory;
50 import org.apache.commons.lang.StringUtils;
51 import org.codehaus.staxmate.SMInputFactory;
52 import org.codehaus.staxmate.in.SMHierarchicCursor;
53 import org.codehaus.staxmate.in.SMInputCursor;
54 import org.sonar.api.resources.Qualifiers;
55 import org.sonar.api.utils.System2;
56 import org.sonar.core.util.UuidFactory;
57 import org.sonar.db.Database;
58 import org.sonar.server.platform.db.migration.step.DataChange;
59 import org.sonar.server.platform.db.migration.step.Select;
60 import org.sonar.server.platform.db.migration.step.Upsert;
61 import org.xml.sax.InputSource;
62 import org.xml.sax.SAXException;
63 import org.xml.sax.SAXParseException;
64 import org.xml.sax.helpers.DefaultHandler;
66 import static java.lang.String.format;
67 import static java.nio.charset.StandardCharsets.UTF_8;
68 import static javax.xml.XMLConstants.FEATURE_SECURE_PROCESSING;
69 import static org.apache.commons.io.IOUtils.toInputStream;
70 import static org.apache.commons.lang.StringEscapeUtils.escapeXml;
71 import static org.apache.commons.lang.StringUtils.trim;
73 public class MigrateApplicationDefinitionsFromXmlToDb extends DataChange {
74 static final int TEXT_VALUE_MAX_LENGTH = 4000;
75 private static final String SELECT_APPLICATION_UUID_BY_KEY = "select uuid from projects where kee = ? and qualifier = 'APP'";
76 private static final String SELECT_PROJECTS_BY_KEYS = "select kee,uuid from projects where kee in (%s) and qualifier = 'TRK'";
77 private static final String SELECT_PROJECTS_BY_APP = "select project_uuid from app_projects where application_uuid = ?";
78 private static final String UPDATE_INTERNAL_PROP_TEXT_VALUE = "update internal_properties set text_value = ?, clob_value = NULL where kee = ?";
79 private static final String UPDATE_INTERNAL_PROP_CLOB_VALUE = "update internal_properties set clob_value = ?, text_value = NULL where kee = ?";
80 private static final String VIEWS_DEF_KEY = "views.def";
82 private final UuidFactory uuidFactory;
83 private final System2 system;
85 public MigrateApplicationDefinitionsFromXmlToDb(Database db, UuidFactory uuidFactory, System2 system) {
88 this.uuidFactory = uuidFactory;
93 protected void execute(Context context) throws SQLException {
94 String xml = getViewsDefinition(context);
95 // skip migration if `views.def` does not exist in the db
101 Map<String, ViewXml.ViewDef> defs = ViewXml.parse(xml);
102 List<ViewXml.ViewDef> applications = defs.values()
104 .filter(v -> Qualifiers.APP.equals(v.getQualifier()))
105 .collect(Collectors.toList());
106 for (ViewXml.ViewDef app : applications) {
107 insertApplication(context, app);
109 cleanUpViewsDefinitionsXml(context, defs.values());
110 } catch (Exception e) {
111 throw new IllegalStateException("Failed to migrate views definitions property.", e);
116 private void insertApplication(Context context, ViewXml.ViewDef app) throws SQLException {
117 long now = system.now();
118 String applicationUuid = context.prepareSelect(SELECT_APPLICATION_UUID_BY_KEY)
119 .setString(1, app.getKey())
120 .get(r -> r.getString(1));
122 // skip migration if:
123 // - application only exists in xml and not in the db. It will be removed from the xml at later stage of the migration.
124 // - application contains no projects- it's already in a valid db state
125 List<String> projects = app.getProjects();
126 if (applicationUuid == null || projects.isEmpty()) {
130 List<String> alreadyAddedProjects = context.prepareSelect(SELECT_PROJECTS_BY_APP).setString(1, applicationUuid)
131 .list(r -> r.getString(1));
133 String queryParam = projects.stream().map(uuid -> "'" + uuid + "'").collect(Collectors.joining(","));
134 Map<String, String> projectUuidsByKeys = context.prepareSelect(format(SELECT_PROJECTS_BY_KEYS, queryParam))
135 .list(r -> new AbstractMap.SimpleEntry<>(r.getString(1), r.getString(2)))
137 .filter(project -> !alreadyAddedProjects.contains(project.getValue()))
138 .collect(Collectors.toMap(AbstractMap.SimpleEntry::getKey, AbstractMap.SimpleEntry::getValue));
140 if (!projectUuidsByKeys.isEmpty()) {
141 insertApplicationProjects(context, app, applicationUuid, projectUuidsByKeys, now);
143 if (!app.getApplicationBranches().isEmpty()) {
144 insertApplicationBranchesProjects(context, app, applicationUuid, projectUuidsByKeys, now);
148 private void insertApplicationProjects(Context context, ViewXml.ViewDef app, String applicationUuid,
149 Map<String, String> projectUuidsByKeys, long createdTime) throws SQLException {
150 Upsert insertApplicationProjectsQuery = context.prepareUpsert("insert into " +
151 "app_projects(uuid, application_uuid, project_uuid, created_at) " +
152 "values (?, ?, ?, ?)");
153 for (String projectKey : app.getProjects()) {
154 String applicationProjectUuid = uuidFactory.create();
155 String projectUuid = projectUuidsByKeys.get(projectKey);
157 // ignore project if it does not exist anymore
158 if (projectUuid == null) {
162 insertApplicationProjectsQuery
163 .setString(1, applicationProjectUuid)
164 .setString(2, applicationUuid)
165 .setString(3, projectUuid)
166 .setLong(4, createdTime)
170 insertApplicationProjectsQuery.execute().commit();
173 private void insertApplicationBranchesProjects(Context context, ViewXml.ViewDef app, String applicationUuid,
174 Map<String, String> projectUuidsByKeys, long createdTime) throws SQLException {
175 Map<String, String> appBranchUuidByKey = context.prepareSelect("select kee,uuid from project_branches where project_uuid = ?")
176 .setString(1, applicationUuid)
177 .list(r -> new AbstractMap.SimpleEntry<>(r.getString(1), r.getString(2)))
179 .collect(Collectors.toMap(AbstractMap.SimpleEntry::getKey, AbstractMap.SimpleEntry::getValue));
181 Upsert insertApplicationsBranchProjectsQuery = context.prepareUpsert("insert into " +
182 "app_branch_project_branch(uuid, application_uuid, application_branch_uuid, project_uuid, project_branch_uuid, created_at) " +
183 "values (?, ?, ?, ?, ?, ?)");
184 boolean insert = false;
185 for (ViewXml.ApplicationBranchDef branch : app.getApplicationBranches()) {
186 String applicationBranchUuid = appBranchUuidByKey.get(branch.getKey());
187 // ignore application branch if it does not exist in the DB anymore
188 if (applicationBranchUuid == null) {
192 if (insertApplicationBranchProjects(context, branch, applicationUuid, applicationBranchUuid, projectUuidsByKeys, createdTime,
193 insertApplicationsBranchProjectsQuery)) {
199 insertApplicationsBranchProjectsQuery.execute().commit();
203 private boolean insertApplicationBranchProjects(Context context, ViewXml.ApplicationBranchDef branch, String applicationUuid,
204 String applicationBranchUuid, Map<String, String> projectUuidsByKeys, long createdTime,
205 Upsert insertApplicationsBranchProjectsQuery) throws SQLException {
207 boolean insert = false;
208 for (ViewXml.ApplicationProjectDef appProjDef : branch.getProjects()) {
209 String projectUuid = projectUuidsByKeys.get(appProjDef.getKey());
211 // ignore projects that do not exist in the DB anymore
212 if (projectUuid != null) {
213 String projectBranchUuid = context.prepareSelect("select uuid from project_branches where project_uuid = ? and kee = ?")
214 .setString(1, projectUuid)
215 .setString(2, appProjDef.getBranch())
216 .get(r -> r.getString(1));
218 // ignore project branches that do not exist in the DB anymore
219 if (projectBranchUuid != null) {
220 String applicationBranchProjectUuid = uuidFactory.create();
221 insertApplicationsBranchProjectsQuery
222 .setString(1, applicationBranchProjectUuid)
223 .setString(2, applicationUuid)
224 .setString(3, applicationBranchUuid)
225 .setString(4, projectUuid)
226 .setString(5, projectBranchUuid)
227 .setLong(6, createdTime)
238 private static String getViewsDefinition(DataChange.Context context) throws SQLException {
239 Select select = context.prepareSelect("select text_value,clob_value from internal_properties where kee=?");
240 select.setString(1, VIEWS_DEF_KEY);
241 return select.get(row -> {
242 String v = row.getString(1);
246 return row.getString(2);
251 private static void cleanUpViewsDefinitionsXml(Context context, Collection<ViewXml.ViewDef> definitions) throws SQLException, IOException {
252 definitions = definitions.stream()
253 .filter(d -> !"APP".equals(d.getQualifier()))
254 .collect(Collectors.toList());
256 StringWriter output = new StringWriter();
257 new ViewXml.ViewDefinitionsSerializer().write(definitions, output);
258 String value = output.toString();
259 String statement = UPDATE_INTERNAL_PROP_TEXT_VALUE;
260 if (mustBeStoredInClob(value)) {
261 statement = UPDATE_INTERNAL_PROP_CLOB_VALUE;
264 context.prepareUpsert(statement)
265 .setString(1, output.toString())
266 .setString(2, VIEWS_DEF_KEY)
271 private static boolean mustBeStoredInClob(String value) {
272 return value.length() > TEXT_VALUE_MAX_LENGTH;
275 private static class ViewXml {
276 static final String SCHEMA_VIEWS = "/static/views.xsd";
277 static final String VIEWS_HEADER_BARE = "<views>";
278 static final Pattern VIEWS_HEADER_BARE_PATTERN = Pattern.compile(VIEWS_HEADER_BARE);
279 static final String VIEWS_HEADER_FQ = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
280 + "<views xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xmlns=\"http://sonarsource.com/schema/views\">";
283 // nothing to do here
286 private static Map<String, ViewDef> parse(String xml) throws ParserConfigurationException, SAXException, IOException, XMLStreamException {
287 if (StringUtils.isEmpty(xml)) {
288 return new LinkedHashMap<>(0);
293 SMInputFactory inputFactory = initStax();
294 SMHierarchicCursor rootC = inputFactory.rootElementCursor(new StringReader(xml));
295 rootC.advance(); // <views>
296 SMInputCursor cursor = rootC.childElementCursor();
297 views = parseViewDefinitions(cursor);
299 Map<String, ViewDef> result = new LinkedHashMap<>(views.size());
300 for (ViewDef def : views) {
301 result.put(def.getKey(), def);
307 private static void validate(String xml) throws IOException, SAXException, ParserConfigurationException {
308 // Replace bare, namespace unaware header with fully qualified header (with schema declaration)
309 String fullyQualifiedXml = VIEWS_HEADER_BARE_PATTERN.matcher(xml).replaceFirst(VIEWS_HEADER_FQ);
310 try (InputStream xsd = MigrateApplicationDefinitionsFromXmlToDb.class.getResourceAsStream(SCHEMA_VIEWS)) {
311 InputSource viewsDefinition = new InputSource(new InputStreamReader(toInputStream(fullyQualifiedXml, UTF_8), UTF_8));
313 SchemaFactory saxSchemaFactory = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI);
314 saxSchemaFactory.setProperty(XMLConstants.ACCESS_EXTERNAL_SCHEMA, "");
315 saxSchemaFactory.setProperty(XMLConstants.ACCESS_EXTERNAL_DTD, "");
317 SAXParserFactory parserFactory = SAXParserFactory.newInstance();
318 parserFactory.setFeature(FEATURE_SECURE_PROCESSING, true);
319 parserFactory.setNamespaceAware(true);
320 parserFactory.setSchema(saxSchemaFactory.newSchema(new SAXSource(new InputSource(xsd))));
322 SAXParser saxParser = parserFactory.newSAXParser();
323 saxParser.setProperty(XMLConstants.ACCESS_EXTERNAL_DTD, "");
324 saxParser.setProperty(XMLConstants.ACCESS_EXTERNAL_SCHEMA, "");
325 saxParser.parse(viewsDefinition, new ViewsValidator());
329 private static List<ViewDef> parseViewDefinitions(SMInputCursor viewsCursor) throws XMLStreamException {
330 List<ViewDef> views = new ArrayList<>();
331 while (viewsCursor.getNext() != null) {
332 ViewDef viewDef = new ViewDef();
333 viewDef.setKey(viewsCursor.getAttrValue("key"));
334 viewDef.setDef(Boolean.parseBoolean(viewsCursor.getAttrValue("def")));
335 viewDef.setParent(viewsCursor.getAttrValue("parent"));
336 viewDef.setRoot(viewsCursor.getAttrValue("root"));
337 SMInputCursor viewCursor = viewsCursor.childElementCursor();
338 while (viewCursor.getNext() != null) {
339 String nodeName = viewCursor.getLocalName();
340 parseChildElement(viewDef, viewCursor, nodeName);
347 private static void parseChildElement(ViewDef viewDef, SMInputCursor viewCursor, String nodeName) throws XMLStreamException {
348 if (StringUtils.equals(nodeName, "name")) {
349 viewDef.setName(trim(viewCursor.collectDescendantText()));
350 } else if (StringUtils.equals(nodeName, "desc")) {
351 viewDef.setDesc(trim(viewCursor.collectDescendantText()));
352 } else if (StringUtils.equals(nodeName, "regexp")) {
353 viewDef.setRegexp(trim(viewCursor.collectDescendantText()));
354 } else if (StringUtils.equals(nodeName, "language")) {
355 viewDef.setLanguage(trim(viewCursor.collectDescendantText()));
356 } else if (StringUtils.equals(nodeName, "tag_key")) {
357 viewDef.setTagKey(trim(viewCursor.collectDescendantText()));
358 } else if (StringUtils.equals(nodeName, "tag_value")) {
359 viewDef.setTagValue(trim(viewCursor.collectDescendantText()));
360 } else if (StringUtils.equals(nodeName, "p")) {
361 viewDef.addProject(trim(viewCursor.collectDescendantText()));
362 } else if (StringUtils.equals(nodeName, "vw-ref")) {
363 viewDef.addReference(trim(viewCursor.collectDescendantText()));
364 } else if (StringUtils.equals(nodeName, "qualifier")) {
365 viewDef.setQualifier(trim(viewCursor.collectDescendantText()));
366 } else if (StringUtils.equals(nodeName, "branch")) {
367 parseBranch(viewDef, viewCursor);
368 } else if (StringUtils.equals(nodeName, "tagsAssociation")) {
369 parseTagsAssociation(viewDef, viewCursor);
373 private static void parseBranch(ViewDef def, SMInputCursor viewCursor) throws XMLStreamException {
374 List<ApplicationProjectDef> projects = new ArrayList<>();
375 String key = viewCursor.getAttrValue("key");
376 SMInputCursor projectCursor = viewCursor.childElementCursor();
377 while (projectCursor.getNext() != null) {
378 if (Objects.equals(projectCursor.getLocalName(), "p")) {
379 String branch = projectCursor.getAttrValue("branch");
380 String projectKey = trim(projectCursor.collectDescendantText());
381 projects.add(new ApplicationProjectDef().setKey(projectKey).setBranch(branch));
384 def.getApplicationBranches().add(new ApplicationBranchDef()
386 .setProjects(projects));
389 private static void parseTagsAssociation(ViewDef def, SMInputCursor viewCursor) throws XMLStreamException {
390 SMInputCursor projectCursor = viewCursor.childElementCursor();
391 while (projectCursor.getNext() != null) {
392 def.addTagAssociation(trim(projectCursor.collectDescendantText()));
396 private static SMInputFactory initStax() {
397 XMLInputFactory xmlFactory = XMLInputFactory.newInstance();
398 xmlFactory.setProperty(XMLInputFactory.IS_COALESCING, Boolean.TRUE);
399 xmlFactory.setProperty(XMLInputFactory.IS_NAMESPACE_AWARE, Boolean.FALSE);
400 // just so it won't try to load DTD in if there's DOCTYPE
401 xmlFactory.setProperty(XMLInputFactory.SUPPORT_DTD, Boolean.FALSE);
402 xmlFactory.setProperty(XMLInputFactory.IS_VALIDATING, Boolean.FALSE);
403 return new SMInputFactory(xmlFactory);
406 private static class ViewDef {
409 String parent = null;
415 List<String> p = new ArrayList<>();
417 List<String> vwRef = new ArrayList<>();
423 String regexp = null;
425 String language = null;
427 String tagKey = null;
429 String tagValue = null;
431 String qualifier = null;
433 List<ApplicationBranchDef> applicationBranches = new ArrayList<>();
435 Set<String> tagsAssociation = new TreeSet<>();
437 public String getKey() {
441 public String getParent() {
446 public String getRoot() {
450 public boolean isDef() {
454 public List<String> getProjects() {
458 public List<String> getReferences() {
462 public String getName() {
466 public String getDesc() {
471 public String getRegexp() {
476 public String getLanguage() {
481 public String getTagKey() {
486 public String getTagValue() {
491 public String getQualifier() {
495 public List<ApplicationBranchDef> getApplicationBranches() {
496 return applicationBranches;
499 public Set<String> getTagsAssociation() {
500 return tagsAssociation;
503 public ViewDef setKey(String key) {
508 public ViewDef setParent(String parent) {
509 this.parent = parent;
513 public ViewDef setRoot(@Nullable String root) {
518 public ViewDef setDef(boolean def) {
523 public ViewDef setProjects(List<String> projects) {
528 public ViewDef addProject(String project) {
533 public ViewDef removeProject(String project) {
534 this.p.remove(project);
538 public ViewDef setName(String name) {
543 public ViewDef setDesc(@Nullable String desc) {
548 public ViewDef setRegexp(@Nullable String regexp) {
549 this.regexp = regexp;
553 public ViewDef setLanguage(@Nullable String language) {
554 this.language = language;
558 public ViewDef setTagKey(@Nullable String tagKey) {
559 this.tagKey = tagKey;
563 public ViewDef setTagValue(@Nullable String tagValue) {
564 this.tagValue = tagValue;
568 public ViewDef addReference(String reference) {
569 this.vwRef.add(reference);
573 public ViewDef removeReference(String reference) {
574 this.vwRef.remove(reference);
578 public ViewDef setReferences(List<String> vwRef) {
583 public ViewDef setQualifier(@Nullable String qualifier) {
584 this.qualifier = qualifier;
588 public ViewDef setApplicationBranches(List<ApplicationBranchDef> branches) {
589 this.applicationBranches = branches;
593 public ViewDef addTagAssociation(String tag) {
594 this.tagsAssociation.add(tag);
598 public ViewDef setTagsAssociation(Set<String> tagsAssociation) {
599 this.tagsAssociation = tagsAssociation;
604 private static class ApplicationProjectDef {
605 private String key = null;
606 private String branch = null;
608 public String getKey() {
612 public ApplicationProjectDef setKey(String key) {
618 public String getBranch() {
622 public ApplicationProjectDef setBranch(@Nullable String branch) {
623 this.branch = branch;
628 private static class ApplicationBranchDef {
630 private String key = null;
631 private List<ApplicationProjectDef> p = new ArrayList<>();
633 public String getKey() {
637 public ApplicationBranchDef setKey(String key) {
642 public List<ApplicationProjectDef> getProjects() {
646 public ApplicationBranchDef setProjects(List<ApplicationProjectDef> p) {
652 private static final class ViewsValidator extends DefaultHandler {
654 public void error(SAXParseException exception) throws SAXException {
659 public void warning(SAXParseException exception) throws SAXException {
664 public void fatalError(SAXParseException exception) throws SAXException {
669 static class ViewDefinitionsSerializer {
671 public void write(Collection<ViewDef> definitions, Writer writer) throws IOException {
672 writer.append(VIEWS_HEADER_BARE);
674 for (ViewDef def : definitions) {
675 writer.append("<vw");
676 writer.append(" key=\"").append(escapeXml(def.getKey())).append("\"");
677 writer.append(" def=\"").append(Boolean.toString(def.isDef())).append("\"");
678 String parent = def.getParent();
679 if (parent != null) {
680 writer.append(" root=\"").append(escapeXml(def.getRoot())).append("\"");
681 writer.append(" parent=\"").append(escapeXml(parent)).append("\"");
685 writer.append("<name><![CDATA[").append(def.getName()).append("]]></name>");
686 writeOptionalElements(writer, def);
688 for (String project : def.getProjects()) {
689 writer.append("<p>").append(project).append("</p>");
691 for (String ref : def.getReferences()) {
692 writer.append("<vw-ref><![CDATA[").append(ref).append("]]></vw-ref>");
694 writeTagsAssociation(writer, def);
695 writer.append("</vw>");
698 writer.append("</views>");
701 private static void writeOptionalElements(Writer writer, ViewDef def) throws IOException {
702 String description = def.getDesc();
703 if (description != null) {
704 writer.append("<desc><![CDATA[").append(description).append("]]></desc>");
706 String regexp = def.getRegexp();
707 if (regexp != null) {
708 writer.append("<regexp><![CDATA[").append(regexp).append("]]></regexp>");
710 String language = def.getLanguage();
711 if (language != null) {
712 writer.append("<language><![CDATA[").append(language).append("]]></language>");
714 String customMeasureKey = def.getTagKey();
715 if (customMeasureKey != null) {
716 writer.append("<tag_key><![CDATA[").append(customMeasureKey).append("]]></tag_key>");
718 String customMeasureValue = def.getTagValue();
719 if (customMeasureValue != null) {
720 writer.append("<tag_value><![CDATA[").append(customMeasureValue).append("]]></tag_value>");
722 String qualifier = def.getQualifier();
723 if (qualifier != null) {
724 writer.append("<qualifier><![CDATA[").append(qualifier).append("]]></qualifier>");
728 private static void writeTagsAssociation(Writer writer, ViewDef definition) throws IOException {
729 Set<String> tagsAssociation = definition.getTagsAssociation();
730 if (tagsAssociation.isEmpty()) {
733 writer.append("<tagsAssociation>");
734 for (String tag : tagsAssociation) {
735 writer.append("<tag>").append(tag).append("</tag>");
737 writer.append("</tagsAssociation>");