# See BuildArchetypes for details on environment
# BuildDemos needs git in PATH and depends on gitpython library
# gitpython can be installed with python installer script "pip":
-# pip install gitpython
+# pip install gitpython
#
# Deployment dependency: requests
# pip install requests
# Deploy depends on .deployUrl and .deployCredentials files in home folder
-import sys, os
+import sys, os, pickle
from os.path import join, isfile
from fnmatch import fnmatch
from xml.etree.ElementTree import ElementTree
# "my-demo" : ("my_demo_url_or_path", "my-demo-dev-branch")
}
+status_dump = {"messages": []}
+
+def dump_status(error_occurred):
+ status_dump["error"] = error_occurred
+ pickle.dump(status_dump, open("result/demo_validation_status.pickle", "wb"))
+
+def log_status(log_string):
+ status_dump["messages"].add(log_string)
+ print(log_string)
+
def checkout(folder, url, repoBranch = "master"):
Repo.clone_from(url, join(resultPath, folder), branch = repoBranch)
if __name__ == "__main__":
- # Do imports.
+ # Do imports.
try:
from git import Repo
except:
- print("BuildDemos depends on gitpython. Install it with `pip install gitpython`")
+ log_status("BuildDemos depends on gitpython. Install it with `pip install gitpython`")
+ dump_status(True)
sys.exit(1)
from BuildHelpers import updateRepositories, mavenValidate, copyWarFiles, getLogFile, removeDir, getArgs, mavenInstall, resultPath, readPomFile, parser
from DeployHelpers import deployWar
args = getArgs()
demosFailed = False
ignoredDemos = args.ignore.split(",")
-
+
wars = []
for demo in demos:
updateRepositories(join(resultPath, demo), args.pluginRepo, postfix="plugin")
mavenValidate(demo, logFile=getLogFile(demo))
wars.extend(copyWarFiles(demo))
- print("%s demo validation succeeded!" % (demo))
+ log_status("%s demo validation succeeded!" % (demo))
except Exception as e:
- print("%s demo validation failed: %s" % (demo, e))
+ log_status("%s demo validation failed: %s" % (demo, e))
if demo not in ignoredDemos:
demosFailed = True
except EnvironmentError as e:
- print("%s demo validation failed: %s" % (demo, e))
+ log_status("%s demo validation failed: %s" % (demo, e))
if demo not in ignoredDemos:
demosFailed = True
try:
try:
deployWar(war)
except Exception as e:
- print("War %s failed to deploy: %s" % (war, e))
+ log_status("War %s failed to deploy: %s" % (war, e))
demosFailed = True
if demosFailed:
+ dump_status(True)
sys.exit(1)
+
+ dump_status(False)
--- /dev/null
+from BuildDemos import demos
+from BuildArchetypes import archetypes
+import argparse, requests, json, subprocess, re, pickle
+
+parser = argparse.ArgumentParser()
+parser.add_argument("version", type=str, help="Vaadin version that was just built")
+parser.add_argument("deployUrl", type=str, help="Base url of the deployment server")
+
+parser.add_argument("teamcityUser", type=str, help="Teamcity username to use")
+parser.add_argument("teamcityPassword", type=str, help="Password for given teamcity username")
+
+parser.add_argument("teamcityUrl", type=str, help="Address to the teamcity server")
+parser.add_argument("buildTypeId", type=str, help="The ID of this build step")
+parser.add_argument("buildId", type=str, help="ID of the build to generate this report for")
+
+parser.add_argument("frameworkRepoUrl", type=str, help="URL to the framework staging repository")
+parser.add_argument("archetypeRepoUrl", type=str, help="URL to the archetype staging repository")
+parser.add_argument("pluginRepoUrl", type=str, help="URL to the plugin staging repository")
+args = parser.parse_args()
+
+buildResultUrl = "http://{}/viewLog.html?buildId={}&tab=buildResultsDiv&buildTypeId={}".format(args.teamcityUrl, args.buildId, args.buildTypeId)
+
+def createTableRow(*columns):
+ html = "<tr>"
+ for column in columns:
+ html += "<td>" + column + "</td>"
+ return html + "</tr>"
+
+def getHtmlList(array):
+ html = "<ul>"
+ for item in array:
+ html += "<li>" + item + "</li>"
+ return html + "</ul>"
+
+def getBuildStatusHtml():
+ build_steps_request_string = "http://{}/app/rest/problemOccurrences?locator=build:{}".format(args.teamcityUrl, args.buildId)
+ build_steps_request = requests.get(build_steps_request_string, auth=(args.teamcityUser, args.teamcityPassword), headers={'Accept':'application/json'})
+ if build_steps_request.status_code != 200:
+ return createTableRow(traffic_light.format(color="black"), "Build status: unable to retrieve status of build")
+ else:
+ build_steps_json = build_steps_request.json()
+ if build_steps_json["count"] == 0:
+ return createTableRow(traffic_light.format(color="green"), "Build status: all build steps successful")
+ else:
+ return createTableRow(traffic_light.format(color="red"), "Build status: there are failing build steps, <a href={}>check the build report</a>".format(buildResultUrl))
+
+def getTestStatusHtml():
+ test_failures_request_string = "http://{}/app/rest/testOccurrences?locator=build:{},status:FAILURE".format(args.teamcityUrl, args.buildId)
+ test_failures_request = requests.get(test_failures_request_string, auth=(args.teamcityUser, args.teamcityPassword), headers={'Accept':'application/json'})
+ if test_failures_request.status_code != 200:
+ return createTableRow(traffic_light.format(color="black"), "Test status: unable to retrieve status of tests")
+ else:
+ test_failures_json = test_failures_request.json()
+ if test_failures_json["count"] == 0:
+ return createTableRow(traffic_light.format(color="green"), "Test status: all tests passing")
+ else:
+ return createTableRow(traffic_light.format(color="red"), "Test status: there are " + str(test_failures_json["count"]) + " failing tests, <a href={}>check the build report</a>".format(buildResultUrl))
+
+def getDemoValidationStatusHtml():
+ status = pickle.load(open("result/demo_validation_status.pickle", "rb"))
+ if status["error"]:
+ return createTableRow(traffic_light.format(color="red"), getHtmlList(status["messages"]))
+ else:
+ return createTableRow(traffic_light.format(color="green"), getHtmlList(status["messages"]))
+
+def getDemoLinksHtml():
+ demos_html = "Try demos"
+ link_list = list(map(lambda demo: "<a href='{url}/{demoName}-{version}'>{demoName}</a>".format(url=args.deployUrl, demoName=demo, version=args.version), demos))
+ return demos_html + getHtmlList(link_list)
+
+def getArchetypeLinksHtml():
+ archetypes_html = "Try archetypes"
+ link_list = list(map(lambda archetype: "<a href='{url}/{archetypeName}-{version}'>{demoName}</a>".format(url=args.deployUrl, archetypeName=archetype, version=args.version), archetypes))
+ return archetypes_html + getHtmlList(link_list)
+
+def getDirs(url):
+ page = requests.get(url)
+ files = re.findall('<a href=.*>(.*)</a>', page.text)
+ dirs = filter(lambda x: x.endswith('/'), files)
+ return list(map(lambda x: x.replace('/', ''), dirs))
+
+def dirTree(url):
+ dirs = getDirs(url)
+ result = []
+ for d in dirs:
+ result.append(d)
+ subDirs = list(map(lambda x: d + '/' + x, dirTree(url + '/' + d)))
+ result.extend(subDirs)
+ return result
+
+def getAllowedArtifactPaths(allowedArtifacts):
+ result = []
+ for artifact in allowedArtifacts:
+ parts = artifact.split('/', 1)
+ result.append(parts[0])
+ if len(parts) > 1:
+ subart = getAllowedArtifactPaths([ parts[1] ])
+ subArtifacts = list(map(lambda x: parts[0] + '/' + x, subart))
+ result.extend(subArtifacts)
+ return result
+
+def checkStagingContents(url, allowedArtifacts):
+ dirs = dirTree(url)
+ allowedDirs = getAllowedArtifactPaths(allowedArtifacts)
+ return set(dirs) == set(allowedDirs)
+
+def getStagingContentsHtml(repoUrl, allowedArtifacts, name):
+ if checkStagingContents(repoUrl, allowedArtifacts):
+ return createTableRow(traffic_light.format(color="green"), "No extra artifacts found in the {} staging repository. <a href=\"{}\">Link to the repository.</a>".format(name, repoUrl))
+ else:
+ return createTableRow(traffic_light.format(color="red"), "Extra artifacts found in the {} staging repository. <a href=\"{}\">Link to the repository.</a>".format(name, repoUrl))
+
+def completeArtifactName(artifactId, version):
+ return 'com/vaadin/' + artifactId + '/' + version
+
+def completeArtifactNames(artifactIds, version):
+ return list(map(lambda x: completeArtifactName(x, version), artifactIds))
+
+
+allowedPluginArtifacts = completeArtifactNames([ 'vaadin-maven-plugin' ], args.version)
+allowedArchetypeArtifacts = completeArtifactNames([ 'vaadin-archetype-application', 'vaadin-archetype-application-multimodule', 'vaadin-archetype-application-example', 'vaadin-archetype-widget', 'vaadin-archetype-liferay-portlet' ], args.version)
+allowedFrameworkArtifacts = completeArtifactNames([ 'vaadin-root', 'vaadin-bom', 'vaadin-shared', 'vaadin-server', 'vaadin-client', 'vaadin-client-compiler', 'vaadin-client-compiled', 'vaadin-push', 'vaadin-themes', 'vaadin-widgets' ], args.version)
+
+content = "<html><head></head><body><table>"
+traffic_light = "<svg width=\"20px\" height=\"20px\" style=\"padding-right:5px\"><circle cx=\"10\" cy=\"10\" r=\"10\" fill=\"{color}\"/></svg>"
+
+# Build step status
+content += getBuildStatusHtml()
+
+# Test failures
+content += getTestStatusHtml()
+
+# Missing @since tags
+try:
+ p1 = subprocess.Popen(['find', '.', '-name', '*.java'], stdout=subprocess.PIPE)
+ p2 = subprocess.Popen(['xargs', 'egrep', '-n', '@since ?$'], stdin=p1.stdout, stdout=subprocess.PIPE)
+ missing = subprocess.check_output(['egrep', '-v', '/(testbench|test|tests|target)/'], stdin=p2.stdout)
+ content += createTableRow(traffic_light.format(color="red"), "Empty @since:<br><pre>%s</pre>" % (missing))
+
+except subprocess.CalledProcessError as e:
+ if e.returncode == 1:
+ content += createTableRow(traffic_light.format(color="green"), "No empty @since")
+ else:
+ raise e
+
+# check staging repositories don't contain extra artifacts
+content += getStagingContentsHtml(args.frameworkRepoUrl, allowedFrameworkArtifacts, "framework")
+content += getStagingContentsHtml(args.archetypeRepoUrl, allowedArchetypeArtifacts, "archetype")
+content += getStagingContentsHtml(args.pluginRepoUrl, allowedPluginArtifacts, "plugin")
+
+content += createTableRow("", "<h2>Manual checks before publishing</h2>")
+# try demos
+content += createTableRow("", getDemoLinksHtml())
+content += createTableRow("", getArchetypeLinksHtml())
+
+# link to release notes
+content += createTableRow("", "<a href=\"http://{}/repository/download/{}/{}:id/release-notes/release-notes.html\">Check release notes</a>".format(args.teamcityUrl, args.buildTypeId, args.buildId))
+# link to api diff
+content += createTableRow("", "<a href=\"http://{}/repository/download/{}/{}:id/apidiff/changes.html\">API Diff</a>")
+
+# check that trac tickets are in the correct status
+content += createTableRow("", "<a href=\"https://dev.vaadin.com/query?status=closed&status=pending-release&component=Core+Framework&resolution=fixed&group=milestone&col=id&col=summary&col=component&col=status&col=type&col=priority&col=milestone&order=priority\">Check that trac tickets have correct status</a>")
+# pending release tickets without milestone
+content += createTableRow("", "<a href=\"https://dev.vaadin.com/query?status=pending-release&milestone=\">Pending-release tickets without milestone</a>")
+
+content += createTableRow("", "<h2>Preparations before publishing</h2>")
+# close trac milestone
+content += createTableRow("", "<a href=\"https://dev.vaadin.com/milestone/Vaadin {version}\">Close Trac Milestone (deselect \"retarget tickets\")</a>".format(version=args.version))
+# verify pending release tickets still have milestone
+content += createTableRow("", "<a href=\"https://dev.vaadin.com/query?status=pending-release&component=Core+Framework&resolution=fixed&col=id&col=summary&col=component&col=milestone&col=status&col=type\">Verify pending release tickets still have milestone {version}</a>".format(version=args.version))
+# link to build dependencies tab to initiate publish step
+content += createTableRow("", "<a href=\"http://{}/viewLog.html?buildId={}&buildTypeId={}&tab=dependencies\"><h2>Start Publish Release from dependencies tab</h2></a>".format(args.teamcityUrl, args.buildId, args.buildTypeId))
+
+content += "</table></body></html>"
+f = open("result/report.html", 'w')
+f.write(content)
--- /dev/null
+import argparse, requests
+
+parser = argparse.ArgumentParser(description="Post-publish report generator")
+parser.add_argument("version", type=str, help="Vaadin version that was just built")
+parser.add_argument("teamcityUrl", type=str, help="Address to the teamcity server")
+parser.add_argument("buildTypeId", type=str, help="The ID of this build step")
+parser.add_argument("buildId", type=str, help="ID of the build to generate this report for")
+parser.add_argument("projectId", type=str, help="The ID of this project")
+args = parser.parse_args()
+
+buildResultUrl = "http://{}/viewLog.html?buildId={}&tab=buildResultsDiv&buildTypeId={}".format(args.teamcityUrl, args.buildId, args.buildTypeId)
+
+(major, minor, maintenance) = args.version.split(".", 2)
+prerelease = "." in maintenance
+
+def createTableRow(*columns):
+ html = "<tr>"
+ for column in columns:
+ html += "<td>" + column + "</td>"
+ return html + "</tr>"
+
+traffic_light = "<svg width=\"20px\" height=\"20px\" style=\"padding-right:5px\"><circle cx=\"10\" cy=\"10\" r=\"10\" fill=\"{color}\"/></svg>"
+
+content = "<html><head></head><body><table>"
+
+# Batch update tickets in trac
+content += createTableRow("", "<a href=\"https://dev.vaadin.com/query?status=pending-release&component=Core+Framework&resolution=fixed&milestone=Vaadin {version}&col=id&col=summary&col=component&col=milestone&col=status&col=type\">Batch update tickets in Trac</a>")
+
+# Create milestone for next release
+content += createTableRow("", "<a href=\"https://dev.vaadin.com/milestone?action=new\">Create milestone for next release</a>")
+
+# Tag and pin build
+content += createTableRow("", "<a href=\"{url}\">Tag and pin build</a>".format(url=buildResultUrl))
+
+# Traffic light for archetype metadata
+archetypeMetadataUrl = ""
+if not prerelease:
+ archetypeMetadataUrl = "http://vaadin.com/download/maven-archetypes.xml"
+else:
+ archetypeMetadataUrl ="http://vaadin.com/download/maven-archetypes-prerelease.xml"
+
+archetype_metadata_request = requests.get(archetypeMetadataUrl)
+if archetype_metadata_request.status_code != 200:
+ content += createTableRow(traffic_light.format(color="black"), "<a href='{url}'>Check archetype metadata: unable to retrieve metadata</a>".format(url=archetypeMetadataUrl))
+else:
+ if "version=\"{version}\"".format(version=args.version) in archetype_metadata_request.content:
+ content += createTableRow(traffic_light.format(color="green"), "<a href='{url}'>Check archetype metadata: metadata is correct</a>".format(url=archetypeMetadataUrl))
+ else:
+ content += createTableRow(traffic_light.format(color="red"), "<a href='{url}'>Check archetype metadata: metadata is incorrect</a>".format(url=archetypeMetadataUrl))
+
+# TODO GitHub milestones
+
+# Inform marketing and PO
+content += createTableRow("", "Inform marketing and PO about the release")
+
+# Link to version update in teamcity
+content += createTableRow("", "<a href=\"http://{}/admin/editProject.html?projectId={}&tab=projectParams\">Update vaadin.version.latest and vaadin.version.next parameters in TeamCity</a>".format(args.teamcityUrl, args.projectId))
+
+# Link to GH release notes
+content += createTableRow("", "<a href=\"https://github.com/vaadin/vaadin/releases/new\">Write release notes in GH</a>")
+
+content += "</table></body></html>"
+
+with open("result/report.html", "wb") as f:
+ f.write(content)
--- /dev/null
+#coding=UTF-8
+
+try:
+ import requests
+except Exception as e:
+ print("GeneratePublishReportPart1 depends on requests library. Install it with `pip install requests`")
+ sys.exit(1)
+import argparse, cgi, re
+from os.path import exists, isdir
+from os import makedirs
+
+metadataChecks = {
+ 'https://vaadin.com/download/LATEST7': '^7\..*',
+ 'https://vaadin.com/download/VERSIONS_7': '^7\..*',
+ 'https://vaadin.com/download/release/7.7/LATEST': '^7\..*',
+ 'https://vaadin.com/download/LATEST': '^6\..*',
+ 'https://vaadin.com/download/PRERELEASES': '^{ver}'
+}
+
+parser = argparse.ArgumentParser(description="Post-publish report generator")
+parser.add_argument("version", type=str, help="Vaadin version that was just built")
+parser.add_argument("teamcityUrl", type=str, help="Address to the teamcity server")
+parser.add_argument("buildTypeId", type=str, help="The ID of this build step")
+parser.add_argument("buildId", type=str, help="ID of the build to generate this report for")
+args = parser.parse_args()
+
+traffic_light = "<svg width=\"20px\" height=\"20px\" style=\"padding-right:5px\"><circle cx=\"10\" cy=\"10\" r=\"10\" fill=\"{color}\"/></svg>"
+
+def getTrafficLight(b):
+ return traffic_light.format(color="green") if b else traffic_light.format(color="red")
+
+resultPath = "result"
+if not exists(resultPath):
+ makedirs(resultPath)
+elif not isdir(resultPath):
+ print("Result path is not a directory.")
+ sys.exit(1)
+
+(major, minor, maintenance) = args.version.split(".", 2)
+prerelease = "." in maintenance
+if prerelease:
+ maintenance = maintenance.split('.')[0]
+
+def checkUrlContents(url, regexp):
+ r = requests.get(url)
+ return re.match(regexp, r.text) != None
+
+def checkUrlStatus(url):
+ r = requests.get(url)
+ return r.status_code == 200
+
+metadataOk = True
+for url in metadataChecks:
+ metadataOk = metadataOk and checkUrlContents(url, metadataChecks[url].format(ver=args.version))
+
+tagOk = checkUrlStatus("https://github.com/vaadin/vaadin/releases/tag/{ver}".format(ver=args.version))
+
+if not prerelease:
+ downloadPageOk = checkUrlStatus("https://vaadin.com/download/release/{maj}.{min}/{ver}/".format(maj=major, min=minor, ver=args.version))
+else:
+ downloadPageOk = checkUrlStatus("https://vaadin.com/download/prerelease/{maj}.{min}/{maj}.{min}.{main}/{ver}".format(maj=major, min=minor, main=maintenance, ver=args.version))
+
+content = """<html>
+<head></head>
+<body>
+<table>
+<tr><td>{metadataOk}</td><td>Metadata ok on vaadin.com</td></tr>
+<tr><td>{tagOk}</td><td>Tag ok on github.com</td></tr>
+<tr><td>{downloadPageOk}</td><td>Download folder on vaadin.com contains the version</td></tr>
+""".format(metadataOk=getTrafficLight(metadataOk), tagOk=getTrafficLight(tagOk), downloadPageOk=getTrafficLight(downloadPageOk))
+
+mavenUrl = ""
+if not prerelease:
+ mavenUrl = "http://repo1.maven.org/maven2/com/vaadin/vaadin-server/{ver}".format(ver=args.version)
+ content += "<tr><td></td><td><a href='{mvnUrl}'>Check {ver} is published to maven.org (might take a while)</td></tr>".format(ver=args.version, mvnUrl=mavenUrl)
+else:
+ mavenUrl = "http://maven.vaadin.com/vaadin-prereleases/com/vaadin/vaadin-server/{ver}".format(ver=args.version)
+ content += "<tr><td></td><td><a href='{mvnUrl}'>Check {ver} is published as prerelease to maven.vaadin.com</td></tr>".format(ver=args.version, mvnUrl=mavenUrl)
+
+content += "<tr><td></td><td><a href=\"https://dev.vaadin.com/admin/ticket/versions\">Add version {version} to Trac</a></td></tr>".format(version=args.version)
+
+if not prerelease:
+ content += '<tr><td></td><td><a href="https://dev.vaadin.com/admin/ticket/versions">Set latest version to default</a></td></tr>'
+
+content += """
+<tr><td></td><td><a href="http://test.vaadin.com/{version}/run/LabelModes?restartApplication">Verify uploaded to test.vaadin.com</a></td></tr>
+""".format(version=args.version)
+
+if not prerelease:
+ content += '<tr><td></td><td><a href="http://vaadin.com/api">Verify API version list updated</a></td></tr>'
+
+content += """
+<tr><td></td><td><a href="http://{teamcityUrl}/viewLog.html?buildId={buildId}&buildTypeId={buildTypeId}&tab=dependencies"><h2>Start Post-Publish Release from dependencies tab</a></td></tr>
+</table>
+</body>
+</html>""".format(teamcityUrl=args.teamcityUrl, buildTypeId=args.buildTypeId, buildId=args.buildId, version=args.version)
+
+f = open("result/report.html", 'w')
+f.write(content)
projects / vaadin-framework.git / commitdiff