scripts/make_release.py

# -*- coding: utf-8 -*-

#-------------------------------------------------------------------------
# drawElements Quality Program utilities
# --------------------------------------
#
# Copyright 2015 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
#-------------------------------------------------------------------------

import os
import re
import sys
import copy
import zlib
import time
import shlex
import shutil
import fnmatch
import tarfile
import argparse
import platform
import datetime
import tempfile
import posixpath
import subprocess

from build.common import *
from build.config import *
from build.build import *

def die (msg):
	print msg
	sys.exit(-1)

def removeLeadingPath (path, basePath):
	# Both inputs must be normalized already
	assert os.path.normpath(path) == path
	assert os.path.normpath(basePath) == basePath
	return path[len(basePath) + 1:]

def findFile (candidates):
	for file in candidates:
		if os.path.exists(file):
			return file
	return None

def getFileList (basePath):
	allFiles	= []
	basePath	= os.path.normpath(basePath)
	for root, dirs, files in os.walk(basePath):
		for file in files:
			relPath = removeLeadingPath(os.path.normpath(os.path.join(root, file)), basePath)
			allFiles.append(relPath)
	return allFiles

def toDatetime (dateTuple):
	Y, M, D = dateTuple
	return datetime.datetime(Y, M, D)

class PackageBuildInfo:
	def __init__ (self, releaseConfig, srcBasePath, dstBasePath, tmpBasePath):
		self.releaseConfig	= releaseConfig
		self.srcBasePath	= srcBasePath
		self.dstBasePath	= dstBasePath
		self.tmpBasePath	= tmpBasePath

	def getReleaseConfig (self):
		return self.releaseConfig

	def getReleaseVersion (self):
		return self.releaseConfig.getVersion()

	def getReleaseId (self):
		# Release id is crc32(releaseConfig + release)
		return zlib.crc32(self.releaseConfig.getName() + self.releaseConfig.getVersion()) & 0xffffffff

	def getSrcBasePath (self):
		return self.srcBasePath

	def getTmpBasePath (self):
		return self.tmpBasePath

class DstFile (object):
	def __init__ (self, dstFile):
		self.dstFile = dstFile

	def makeDir (self):
		dirName = os.path.dirname(self.dstFile)
		if not os.path.exists(dirName):
			os.makedirs(dirName)

	def make (self, packageBuildInfo):
		assert False # Should not be called

class CopyFile (DstFile):
	def __init__ (self, srcFile, dstFile):
		super(CopyFile, self).__init__(dstFile)
		self.srcFile = srcFile

	def make (self, packageBuildInfo):
		self.makeDir()
		if os.path.exists(self.dstFile):
			die("%s already exists" % self.dstFile)
		shutil.copyfile(self.srcFile, self.dstFile)

class GenReleaseInfoFileTarget (DstFile):
	def __init__ (self, dstFile):
		super(GenReleaseInfoFileTarget, self).__init__(dstFile)

	def make (self, packageBuildInfo):
		self.makeDir()

		scriptPath = os.path.normpath(os.path.join(packageBuildInfo.srcBasePath, "framework", "qphelper", "gen_release_info.py"))
		execute([
				"python",
				"-B", # no .py[co]
				scriptPath,
				"--name=%s" % packageBuildInfo.getReleaseVersion(),
				"--id=0x%08x" % packageBuildInfo.getReleaseId(),
				"--out=%s" % self.dstFile
			])

class GenCMake (DstFile):
	def __init__ (self, srcFile, dstFile, replaceVars):
		super(GenCMake, self).__init__(dstFile)
		self.srcFile		= srcFile
		self.replaceVars	= replaceVars

	def make (self, packageBuildInfo):
		self.makeDir()
		print "    GenCMake: %s" % removeLeadingPath(self.dstFile, packageBuildInfo.dstBasePath)
		src = readFile(self.srcFile)
		for var, value in self.replaceVars:
			src = re.sub('set\(%s\s+"[^"]*"' % re.escape(var),
						 'set(%s "%s"' % (var, value), src)
		writeFile(self.dstFile, src)

def createFileTargets (srcBasePath, dstBasePath, files, filters):
	usedFiles	= set() # Files that are already included by other filters
	targets		= []

	for isMatch, createFileObj in filters:
		# Build list of files that match filter
		matchingFiles = []
		for file in files:
			if not file in usedFiles and isMatch(file):
				matchingFiles.append(file)

		# Build file objects, add to used set
		for file in matchingFiles:
			usedFiles.add(file)
			targets.append(createFileObj(os.path.join(srcBasePath, file), os.path.join(dstBasePath, file)))

	return targets

# Generates multiple file targets based on filters
class FileTargetGroup:
	def __init__ (self, srcBasePath, dstBasePath, filters, srcBasePathFunc=PackageBuildInfo.getSrcBasePath):
		self.srcBasePath	= srcBasePath
		self.dstBasePath	= dstBasePath
		self.filters		= filters
		self.getSrcBasePath	= srcBasePathFunc

	def make (self, packageBuildInfo):
		fullSrcPath		= os.path.normpath(os.path.join(self.getSrcBasePath(packageBuildInfo), self.srcBasePath))
		fullDstPath		= os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, self.dstBasePath))

		allFiles		= getFileList(fullSrcPath)
		targets		 	= createFileTargets(fullSrcPath, fullDstPath, allFiles, self.filters)

		# Make all file targets
		for file in targets:
			file.make(packageBuildInfo)

# Single file target
class SingleFileTarget:
	def __init__ (self, srcFile, dstFile, makeTarget):
		self.srcFile	= srcFile
		self.dstFile	= dstFile
		self.makeTarget	= makeTarget

	def make (self, packageBuildInfo):
		fullSrcPath		= os.path.normpath(os.path.join(packageBuildInfo.srcBasePath, self.srcFile))
		fullDstPath		= os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, self.dstFile))

		target = self.makeTarget(fullSrcPath, fullDstPath)
		target.make(packageBuildInfo)

class BuildTarget:
	def __init__ (self, baseConfig, generator, targets = None):
		self.baseConfig	= baseConfig
		self.generator	= generator
		self.targets	= targets

	def make (self, packageBuildInfo):
		print "    Building %s" % self.baseConfig.getBuildDir()

		# Create config with full build dir path
		config = BuildConfig(os.path.join(packageBuildInfo.getTmpBasePath(), self.baseConfig.getBuildDir()),
							 self.baseConfig.getBuildType(),
							 self.baseConfig.getArgs(),
							 srcPath = os.path.join(packageBuildInfo.dstBasePath, "src"))

		assert not os.path.exists(config.getBuildDir())
		build(config, self.generator, self.targets)

class BuildAndroidTarget:
	def __init__ (self, dstFile):
		self.dstFile = dstFile

	def make (self, packageBuildInfo):
		print "    Building Android binary"

		buildRoot = os.path.join(packageBuildInfo.tmpBasePath, "android-build")

		assert not os.path.exists(buildRoot)
		os.makedirs(buildRoot)

		# Execute build script
		scriptPath = os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, "src", "android", "scripts", "build.py"))
		execute([
				"python",
				"-B", # no .py[co]
				scriptPath,
				"--build-root=%s" % buildRoot,
			])

		srcFile		= os.path.normpath(os.path.join(buildRoot, "package", "bin", "dEQP-debug.apk"))
		dstFile		= os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, self.dstFile))

		CopyFile(srcFile, dstFile).make(packageBuildInfo)

class FetchExternalSourcesTarget:
	def __init__ (self):
		pass

	def make (self, packageBuildInfo):
		scriptPath = os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, "src", "external", "fetch_sources.py"))
		execute([
				"python",
				"-B", # no .py[co]
				scriptPath,
			])

class RemoveSourcesTarget:
	def __init__ (self):
		pass

	def make (self, packageBuildInfo):
		shutil.rmtree(os.path.join(packageBuildInfo.dstBasePath, "src"), ignore_errors=False)

class Module:
	def __init__ (self, name, targets):
		self.name		= name
		self.targets	= targets

	def make (self, packageBuildInfo):
		for target in self.targets:
			target.make(packageBuildInfo)

class ReleaseConfig:
	def __init__ (self, name, version, modules, sources = True):
		self.name			= name
		self.version		= version
		self.modules		= modules
		self.sources		= sources

	def getName (self):
		return self.name

	def getVersion (self):
		return self.version

	def getModules (self):
		return self.modules

	def packageWithSources (self):
		return self.sources

def matchIncludeExclude (includePatterns, excludePatterns, filename):
	components = os.path.normpath(filename).split(os.sep)
	for pattern in excludePatterns:
		for component in components:
			if fnmatch.fnmatch(component, pattern):
				return False

	for pattern in includePatterns:
		for component in components:
			if fnmatch.fnmatch(component, pattern):
				return True

	return False

def copyFileFilter (includePatterns, excludePatterns=[]):
	return (lambda f: matchIncludeExclude(includePatterns, excludePatterns, f),
			lambda s, d: CopyFile(s, d))

def makeFileCopyGroup (srcDir, dstDir, includePatterns, excludePatterns=[]):
	return FileTargetGroup(srcDir, dstDir, [copyFileFilter(includePatterns, excludePatterns)])

def makeTmpFileCopyGroup (srcDir, dstDir, includePatterns, excludePatterns=[]):
	return FileTargetGroup(srcDir, dstDir, [copyFileFilter(includePatterns, excludePatterns)], PackageBuildInfo.getTmpBasePath)

def makeFileCopy (srcFile, dstFile):
	return SingleFileTarget(srcFile, dstFile, lambda s, d: CopyFile(s, d))

def getReleaseFileName (configName, releaseName):
	today = datetime.date.today()
	return "dEQP-%s-%04d-%02d-%02d-%s" % (releaseName, today.year, today.month, today.day, configName)

def getTempDir ():
	dirName = os.path.join(tempfile.gettempdir(), "dEQP-Releases")
	if not os.path.exists(dirName):
		os.makedirs(dirName)
	return dirName

def makeRelease (releaseConfig):
	releaseName			= getReleaseFileName(releaseConfig.getName(), releaseConfig.getVersion())
	tmpPath				= getTempDir()
	srcBasePath			= DEQP_DIR
	dstBasePath			= os.path.join(tmpPath, releaseName)
	tmpBasePath			= os.path.join(tmpPath, releaseName + "-tmp")
	packageBuildInfo	= PackageBuildInfo(releaseConfig, srcBasePath, dstBasePath, tmpBasePath)
	dstArchiveName		= releaseName + ".tar.bz2"

	print "Creating release %s to %s" % (releaseName, tmpPath)

	# Remove old temporary dirs
	for path in [dstBasePath, tmpBasePath]:
		if os.path.exists(path):
			shutil.rmtree(path, ignore_errors=False)

	# Make all modules
	for module in releaseConfig.getModules():
		print "  Processing module %s" % module.name
		module.make(packageBuildInfo)

	# Remove sources?
	if not releaseConfig.packageWithSources():
		shutil.rmtree(os.path.join(dstBasePath, "src"), ignore_errors=False)

	# Create archive
	print "Creating %s" % dstArchiveName
	archive	= tarfile.open(dstArchiveName, 'w:bz2')
	archive.add(dstBasePath, arcname=releaseName)
	archive.close()

	# Remove tmp dirs
	for path in [dstBasePath, tmpBasePath]:
		if os.path.exists(path):
			shutil.rmtree(path, ignore_errors=False)

	print "Done!"

# Module declarations

SRC_FILE_PATTERNS	= ["*.h", "*.hpp", "*.c", "*.cpp", "*.m", "*.mm", "*.inl", "*.java", "*.aidl", "CMakeLists.txt", "LICENSE.txt", "*.cmake"]
TARGET_PATTERNS		= ["*.cmake", "*.h", "*.lib", "*.dll", "*.so", "*.txt"]

BASE = Module("Base", [
	makeFileCopy		("LICENSE",									"src/LICENSE"),
	makeFileCopy		("CMakeLists.txt",							"src/CMakeLists.txt"),
	makeFileCopyGroup	("targets",									"src/targets",							TARGET_PATTERNS),
	makeFileCopyGroup	("execserver",								"src/execserver",						SRC_FILE_PATTERNS),
	makeFileCopyGroup	("executor",								"src/executor",							SRC_FILE_PATTERNS),
	makeFileCopy		("modules/CMakeLists.txt", 					"src/modules/CMakeLists.txt"),
	makeFileCopyGroup	("external", 								"src/external",							["CMakeLists.txt", "*.py"]),

	# Stylesheet for displaying test logs on browser
	makeFileCopyGroup	("doc/testlog-stylesheet",					"doc/testlog-stylesheet",				["*"]),

	# Non-optional parts of framework
	makeFileCopy		("framework/CMakeLists.txt", 				"src/framework/CMakeLists.txt"),
	makeFileCopyGroup	("framework/delibs",						"src/framework/delibs",					SRC_FILE_PATTERNS),
	makeFileCopyGroup	("framework/common",						"src/framework/common",					SRC_FILE_PATTERNS),
	makeFileCopyGroup	("framework/qphelper",						"src/framework/qphelper",				SRC_FILE_PATTERNS),
	makeFileCopyGroup	("framework/platform",						"src/framework/platform",				SRC_FILE_PATTERNS),
	makeFileCopyGroup	("framework/opengl",						"src/framework/opengl",					SRC_FILE_PATTERNS, ["simplereference"]),
	makeFileCopyGroup	("framework/egl",							"src/framework/egl",					SRC_FILE_PATTERNS),

	# android sources
	makeFileCopyGroup	("android/package/src",						"src/android/package/src",				SRC_FILE_PATTERNS),
	makeFileCopy		("android/package/AndroidManifest.xml",		"src/android/package/AndroidManifest.xml"),
	makeFileCopyGroup	("android/package/res",						"src/android/package/res",				["*.png", "*.xml"]),
	makeFileCopyGroup	("android/scripts",							"src/android/scripts", [
		"common.py",
		"build.py",
		"resources.py",
		"install.py",
		"launch.py",
		"debug.py"
		]),

	# Release info
	GenReleaseInfoFileTarget("src/framework/qphelper/qpReleaseInfo.inl")
])

DOCUMENTATION = Module("Documentation", [
	makeFileCopyGroup	("doc/pdf",									"doc",									["*.pdf"]),
	makeFileCopyGroup	("doc",										"doc",									["porting_layer_changes_*.txt"]),
])

GLSHARED = Module("Shared GL Tests", [
	# Optional framework components
	makeFileCopyGroup	("framework/randomshaders",					"src/framework/randomshaders",			SRC_FILE_PATTERNS),
	makeFileCopyGroup	("framework/opengl/simplereference",		"src/framework/opengl/simplereference",	SRC_FILE_PATTERNS),
	makeFileCopyGroup	("framework/referencerenderer",				"src/framework/referencerenderer",		SRC_FILE_PATTERNS),

	makeFileCopyGroup	("modules/glshared",						"src/modules/glshared",					SRC_FILE_PATTERNS),
])

GLES2 = Module("GLES2", [
	makeFileCopyGroup	("modules/gles2",							"src/modules/gles2",					SRC_FILE_PATTERNS),
	makeFileCopyGroup	("data/gles2",								"src/data/gles2", 						["*.*"]),
	makeFileCopyGroup	("doc/testspecs/GLES2",						"doc/testspecs/GLES2",					["*.txt"])
])

GLES3 = Module("GLES3", [
	makeFileCopyGroup	("modules/gles3",							"src/modules/gles3",					SRC_FILE_PATTERNS),
	makeFileCopyGroup	("data/gles3",								"src/data/gles3", 						["*.*"]),
	makeFileCopyGroup	("doc/testspecs/GLES3",						"doc/testspecs/GLES3",					["*.txt"])
])

GLES31 = Module("GLES31", [
	makeFileCopyGroup	("modules/gles31",							"src/modules/gles31",					SRC_FILE_PATTERNS),
	makeFileCopyGroup	("data/gles31",								"src/data/gles31", 						["*.*"]),
	makeFileCopyGroup	("doc/testspecs/GLES31",					"doc/testspecs/GLES31",					["*.txt"])
])

EGL = Module("EGL", [
	makeFileCopyGroup	("modules/egl",								"src/modules/egl",						SRC_FILE_PATTERNS)
])

INTERNAL = Module("Internal", [
	makeFileCopyGroup	("modules/internal",						"src/modules/internal",					SRC_FILE_PATTERNS),
	makeFileCopyGroup	("data/internal",							"src/data/internal", 					["*.*"]),
])

EXTERNAL_SRCS = Module("External sources", [
	FetchExternalSourcesTarget()
])

ANDROID_BINARIES = Module("Android Binaries", [
	BuildAndroidTarget	("bin/android/dEQP.apk"),
	makeFileCopyGroup	("targets/android",							"bin/android",							["*.bat", "*.sh"]),
])

COMMON_BUILD_ARGS	= ['-DPNG_SRC_PATH=%s' % os.path.realpath(os.path.join(DEQP_DIR, '..', 'libpng'))]
NULL_X32_CONFIG		= BuildConfig('null-x32',	'Release', ['-DDEQP_TARGET=null', '-DCMAKE_C_FLAGS=-m32', '-DCMAKE_CXX_FLAGS=-m32'] + COMMON_BUILD_ARGS)
NULL_X64_CONFIG		= BuildConfig('null-x64',	'Release', ['-DDEQP_TARGET=null', '-DCMAKE_C_FLAGS=-m64', '-DCMAKE_CXX_FLAGS=-m64'] + COMMON_BUILD_ARGS)
GLX_X32_CONFIG		= BuildConfig('glx-x32',	'Release', ['-DDEQP_TARGET=x11_glx', '-DCMAKE_C_FLAGS=-m32', '-DCMAKE_CXX_FLAGS=-m32'] + COMMON_BUILD_ARGS)
GLX_X64_CONFIG		= BuildConfig('glx-x64',	'Release', ['-DDEQP_TARGET=x11_glx', '-DCMAKE_C_FLAGS=-m64', '-DCMAKE_CXX_FLAGS=-m64'] + COMMON_BUILD_ARGS)

EXCLUDE_BUILD_FILES = ["CMakeFiles", "*.a", "*.cmake"]

LINUX_X32_COMMON_BINARIES = Module("Linux x32 Common Binaries", [
	BuildTarget			(NULL_X32_CONFIG, ANY_UNIX_GENERATOR),
	makeTmpFileCopyGroup(NULL_X32_CONFIG.getBuildDir() + "/execserver",		"bin/linux32",					["*"],	EXCLUDE_BUILD_FILES),
	makeTmpFileCopyGroup(NULL_X32_CONFIG.getBuildDir() + "/executor",		"bin/linux32",					["*"],	EXCLUDE_BUILD_FILES),
])

LINUX_X64_COMMON_BINARIES = Module("Linux x64 Common Binaries", [
	BuildTarget			(NULL_X64_CONFIG, ANY_UNIX_GENERATOR),
	makeTmpFileCopyGroup(NULL_X64_CONFIG.getBuildDir() + "/execserver",		"bin/linux64",					["*"],	EXCLUDE_BUILD_FILES),
	makeTmpFileCopyGroup(NULL_X64_CONFIG.getBuildDir() + "/executor",		"bin/linux64",					["*"],	EXCLUDE_BUILD_FILES),
])

# Special module to remove src dir, for example after binary build
REMOVE_SOURCES = Module("Remove sources from package", [
	RemoveSourcesTarget()
])

# Release configuration

ALL_MODULES		= [
	BASE,
	DOCUMENTATION,
	GLSHARED,
	GLES2,
	GLES3,
	GLES31,
	EGL,
	INTERNAL,
	EXTERNAL_SRCS,
]

ALL_BINARIES	= [
	LINUX_X64_COMMON_BINARIES,
	ANDROID_BINARIES,
]

RELEASE_CONFIGS	= {
	"src":		ALL_MODULES,
	"src-bin":	ALL_MODULES + ALL_BINARIES,
	"bin":		ALL_MODULES + ALL_BINARIES + [REMOVE_SOURCES],
}

def parseArgs ():
	parser = argparse.ArgumentParser(description = "Build release package")
	parser.add_argument("-c",
						"--config",
						dest="config",
						choices=RELEASE_CONFIGS.keys(),
						required=True,
						help="Release configuration")
	parser.add_argument("-n",
						"--name",
						dest="name",
						required=True,
						help="Package-specific name")
	parser.add_argument("-v",
						"--version",
						dest="version",
						required=True,
						help="Version code")
	return parser.parse_args()

if __name__ == "__main__":
	args	= parseArgs()
	config	= ReleaseConfig(args.name, args.version, RELEASE_CONFIGS[args.config])
	makeRelease(config)