Combine Archetype and Demo build scripts (#9943)

scripts/BuildArchetypes.py

-#coding=UTF-8
-
-#
-# Windows users: 
-# You need to setup your environment so that you have mvn on your PATH.
-# Maven needs that JAVA_HOME environment is set and points to a JDK
-# Python3 is required as this script uses some Python3 specific features.
-# Might work with Python2, haven't tested.
-#
-# python BuildArchetypes.py version --repo staging-repo-url
-#
-
-import subprocess, sys
-from os.path import join
-
-## DEFAULT VARIABLES ##
-
-# ArchetypeGroupId
-archetypeGroup = "com.vaadin"
-
-# List of built archetypes
-archetypes = [
-	"vaadin-archetype-widget", 
-	"vaadin-archetype-application", 
-	"vaadin-archetype-application-example",
-	"vaadin-archetype-application-multimodule"
-]
-
-# Maven GroupID
-group = "testpkg"
-
-log = None
-args = None
-
-## BUILDING METHODS ##
-
-# Generates and modifies a maven pom file
-def generateArchetype(archetype, artifactId, repo):
-	# Generate the required command line for archetype generation
-	cmd = [mavenCmd, "archetype:generate"]
-	cmd.append("-DarchetypeGroupId=%s" % (archetypeGroup))
-	cmd.append("-DarchetypeArtifactId=%s" % (archetype))
-	cmd.append("-DarchetypeVersion=%s" % (args.version))
-	if repo is not None:
-		cmd.append("-DarchetypeRepository=%s" % repo)
-	cmd.append("-DgroupId=%s" % (group))
-	cmd.append("-DartifactId=%s" % (artifactId))
-	cmd.append("-Dversion=1.0-SNAPSHOT")
-	cmd.append("-DinteractiveMode=false")
-	if hasattr(args, "maven") and args.maven is not None:
-		cmd.extend(args.maven.strip('"').split(" "))
-	
-	# Generate pom.xml
-	print("Generating pom.xml for archetype %s" % (archetype))
-	subprocess.check_call(cmd, cwd=resultPath, stdout=log)
-
-def getDeploymentContext(archetype, version):
-	return "%s-%s" % (archetype.split("-", 2)[2], version)
-	
-## DO THIS IF RUN AS A SCRIPT (not import) ##
-if __name__ == "__main__":
-	from BuildHelpers import mavenValidate, copyWarFiles, getLogFile, mavenCmd, updateRepositories, getArgs, removeDir, parser, resultPath
-	from DeployHelpers import deployWar
-
-	archetypesFailed = False
-
-	# Parse the arguments
-	args = getArgs()
-
-	wars = {}
-
-	for archetype in archetypes:
-		artifactId = "test-%s-%s" % (archetype, args.version.replace(".", "-"))
-		try:
-			log = getLogFile(archetype)
-			generateArchetype(archetype, artifactId, args.pluginRepo)
-			if hasattr(args, "fwRepo") and args.fwRepo is not None:
-				updateRepositories(join(resultPath, artifactId), args.fwRepo)
-			if hasattr(args, "pluginRepo") and args.pluginRepo is not None:
-				updateRepositories(join(resultPath, artifactId), args.pluginRepo, postfix="plugin")
-			mavenValidate(artifactId, logFile=log)	
-			warFiles = copyWarFiles(artifactId, name=archetype)
-			for war in warFiles:
-				wars[war] = "%s.war" % (getDeploymentContext(archetype, args.version))
-			print("%s validation succeeded!" % (archetype))
-		except Exception as e:
-			print("Archetype %s build failed:" % (archetype), e)
-			archetypesFailed = True
-		try:
-			removeDir(artifactId)
-		except:
-			pass
-		print("")
-
-	for i in wars:
-		try:
-			deployWar(i, wars[i])
-		except Exception as e:
-			print("War %s failed to deploy: %s" % (war, e))
-			archetypesFailed = True
-
-	if archetypesFailed:
-		sys.exit(1)

scripts/BuildDemos.py

 #	"my-demo" : ("my_demo_url_or_path", "my-demo-dev-branch")
 }
 
+# List of built archetypes
+archetypes = [
+	"vaadin-archetype-widget",
+	"vaadin-archetype-application",
+	"vaadin-archetype-application-example",
+	"vaadin-archetype-application-multimodule"
+]
+
 status_dump = {"messages": []}
 
 def dump_status(error_occurred):
 def log_status(log_string):
 	status_dump["messages"].append(log_string)
 	print(log_string)
+	sys.stdout.flush()
 
 def checkout(folder, url, repoBranch = "master"):
 	Repo.clone_from(url, join(resultPath, folder), branch = repoBranch)
 
 if __name__ == "__main__":
-	# Do imports.	
+	# Do imports.
 	try:
 		from git import Repo
 	except:
 		log_status("BuildDemos depends on gitpython. Install it with `pip install gitpython`")
 		dump_status(True)
 		sys.exit(1)
-	from BuildHelpers import updateRepositories, mavenValidate, copyWarFiles, getLogFile, removeDir, getArgs, mavenInstall, resultPath, readPomFile, parser, dockerWrap
+	from BuildHelpers import mavenValidate, copyWarFiles, getLogFile, removeDir, getArgs, resultPath, parser, dockerWrap, generateArchetype
 	from DeployHelpers import deployWar
 	# Add command line agrument for ignoring failing demos
 	parser.add_argument("--ignore", type=str, help="Ignored demos", default="")
+
+	# Control to skip demos and archetypes
+	parser.add_argument("--skipDemos", action="store_true", help="Skip building demos")
+	parser.add_argument("--skipArchetypes", action="store_true", help="Skip building archetypes")
+
 	args = getArgs()
 	demosFailed = False
 	ignoredDemos = args.ignore.split(",")
 	wars = []
 
-	for demo in demos:
-		print("Validating demo %s" % (demo))
-		try:
-			repo = demos[demo]
-			if (isinstance(repo, tuple)):
-				checkout(demo, repo[0], repo[1])
-			else:
-				checkout(demo, repo)
-			if hasattr(args, "fwRepo") and args.fwRepo is not None:
-				updateRepositories(join(resultPath, demo), args.fwRepo)
-			if hasattr(args, "pluginRepo") and args.pluginRepo is not None:
-				updateRepositories(join(resultPath, demo), args.pluginRepo, postfix="plugin")
-			mavenValidate(demo, logFile=getLogFile(demo))
-			wars.extend(copyWarFiles(demo))
-			log_status("%s demo validation succeeded!" % (demo))
-		except Exception as e:
-			log_status("%s demo validation failed: %s" % (demo, e))
-			if demo not in ignoredDemos:
-				demosFailed = True
-		except EnvironmentError as e:
-			log_status("%s demo validation failed: %s" % (demo, e))
-			if demo not in ignoredDemos:
-				demosFailed = True
-		try:
-			removeDir(demo)
-		except:
-			pass
-		print("")
+	if not args.skipDemos:
+		for demo in demos:
+			print("Validating demo %s" % (demo))
+			try:
+				repo = demos[demo]
+				if (isinstance(repo, tuple)):
+					checkout(demo, repo[0], repo[1])
+				else:
+					checkout(demo, repo)
+				mavenValidate(demo, logFile=getLogFile(demo))
+				wars.extend(copyWarFiles(demo))
+				log_status("%s demo validation succeeded!" % (demo))
+			except Exception as e:
+				log_status("%s demo validation failed: %s" % (demo, e))
+				if demo not in ignoredDemos:
+					demosFailed = True
+			except EnvironmentError as e:
+				log_status("%s demo validation failed: %s" % (demo, e))
+				if demo not in ignoredDemos:
+					demosFailed = True
+			try:
+				removeDir(demo)
+			except:
+				pass
+			log_status("")
+
+	if not args.skipArchetypes:
+		for archetype in archetypes:
+			artifactId = "test-%s-%s" % (archetype, args.version.replace(".", "-"))
+			try:
+				log = getLogFile(archetype)
+				generateArchetype(archetype, artifactId, args.pluginRepo, log)
+				mavenValidate(artifactId, logFile=log)
+				wars.extend(copyWarFiles(artifactId, name=archetype))
+				log_status("%s validation succeeded!" % (archetype))
+			except Exception as e:
+				print("Archetype %s build failed:" % (archetype), e)
+				if archetype not in ignoredDemos:
+					demosFailed = True
+	
+			try:
+				removeDir(artifactId)
+			except:
+				pass
+			log_status("")
 
 	if args.deploy_mode:
 		for war in wars:

scripts/BuildHelpers.py

 		copiedWars.append(join(resultDir, deployName))
 	return copiedWars
 
-def readPomFile(pomFile):
-	# pom.xml namespace workaround
-	root = ElementTree.parse(pomFile).getroot()
-	nameSpace = root.tag[1:root.tag.index('}')]
-	ElementTree.register_namespace('', nameSpace)
-
-	# Read the pom.xml correctly
-	return ElementTree.parse(pomFile), nameSpace 
-
-# Recursive pom.xml update script
-def updateRepositories(path, repoUrl = None, version = None, postfix = "staging"):
-	# If versions are not supplied, parse arguments
-	if version is None:
-		version = getArgs().version
-
-	# Read pom.xml
-	pomXml = join(path, "pom.xml")
-	if isfile(pomXml):
-		# Read the pom.xml correctly
-		tree, nameSpace = readPomFile(pomXml)
-		
-		# NameSpace needed for finding the repositories node
-		repoNode = tree.getroot().find("{%s}repositories" % (nameSpace))
-	else:
-		return
-	
-	if repoNode is not None:
-		print("Add staging repositories to " + pomXml)
-		
-		# Add framework staging repository
-		addRepo(repoNode, "repository", "vaadin-%s-%s" % (version, postfix), repoUrl)
-		
-		# Find the correct pluginRepositories node
-		pluginRepo = tree.getroot().find("{%s}pluginRepositories" % (nameSpace))
-		if pluginRepo is None:
-			# Add pluginRepositories node if needed
-			pluginRepo = ElementTree.SubElement(tree.getroot(), "pluginRepositories")
-		
-		# Add plugin staging repository
-		addRepo(pluginRepo, "pluginRepository", "vaadin-%s-%s" % (version, postfix), repoUrl)
-		
-		# Overwrite the modified pom.xml
-		tree.write(pomXml, encoding='UTF-8')
-	
-	# Recursive pom.xml search.
-	for i in listdir(path):
-		file = join(path, i)
-		if isdir(file):
-			updateRepositories(join(path, i), repoUrl, version, postfix)
+# Generates and modifies a maven pom file
+def generateArchetype(archetype, artifactId, repo, logFile, group="testpkg", archetypeGroup="com.vaadin"):
+	# Generate the required command line for archetype generation
+	args = getArgs()
+	cmd = [mavenCmd, "archetype:generate"]
+	cmd.append("-DarchetypeGroupId=%s" % (archetypeGroup))
+	cmd.append("-DarchetypeArtifactId=%s" % (archetype))
+	cmd.append("-DarchetypeVersion=%s" % (args.version))
+	if repo is not None:
+		cmd.append("-DarchetypeRepository=%s" % repo)
+	cmd.append("-DgroupId=%s" % (group))
+	cmd.append("-DartifactId=%s" % (artifactId))
+	cmd.append("-Dversion=1.0-SNAPSHOT")
+	cmd.append("-DinteractiveMode=false")
+	if hasattr(args, "maven") and args.maven is not None:
+		cmd.extend(args.maven.strip('"').split(" "))
+
+	# Generate pom.xml
+	print("Generating archetype %s" % (archetype))
+	subprocess.check_call(cmd, cwd=resultPath, stdout=logFile)
 
 # Add a repository of repoType to given repoNode with id and URL
 def addRepo(repoNode, repoType, id, url):
 		return
 	rmtree(join(resultPath, subdir))
 
-def mavenInstall(pomFile, jarFile = None, mvnCmd = mavenCmd, logFile = sys.stdout):
-	cmd = [mvnCmd, "install:install-file"]
-	cmd.append("-Dfile=%s" % (jarFile if jarFile is not None else pomFile))
-	cmd.append("-DpomFile=%s" % (pomFile))
-	print("executing: %s" % (" ".join(cmd)))
-	subprocess.check_call(cmd, stdout=logFile)	
-
 def dockerWrap(imageVersion, imageName = "demo-validation"):
 	dockerFileContent = """FROM jetty:jre8-alpine
 MAINTAINER FrameworkTeam