blob: 07c6c93a66b222246d04f7b224f0c157568744cb [file] [log] [blame]
// Copyright 2014 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package bootstrap
import (
"fmt"
"go/build"
"path/filepath"
"runtime"
"strings"
"github.com/google/blueprint"
"github.com/google/blueprint/pathtools"
)
var (
pctx = blueprint.NewPackageContext("github.com/google/blueprint/bootstrap")
goTestMainCmd = pctx.StaticVariable("goTestMainCmd", filepath.Join("$ToolDir", "gotestmain"))
goTestRunnerCmd = pctx.StaticVariable("goTestRunnerCmd", filepath.Join("$ToolDir", "gotestrunner"))
pluginGenSrcCmd = pctx.StaticVariable("pluginGenSrcCmd", filepath.Join("$ToolDir", "loadplugins"))
parallelCompile = pctx.StaticVariable("parallelCompile", func() string {
// Parallel compilation is only supported on >= go1.9
for _, r := range build.Default.ReleaseTags {
if r == "go1.9" {
numCpu := runtime.NumCPU()
// This will cause us to recompile all go programs if the
// number of cpus changes. We don't get a lot of benefit from
// higher values, so cap this to make it cheaper to move trees
// between machines.
if numCpu > 8 {
numCpu = 8
}
return fmt.Sprintf("-c %d", numCpu)
}
}
return ""
}())
compile = pctx.StaticRule("compile",
blueprint.RuleParams{
Command: "GOROOT='$goRoot' $compileCmd $parallelCompile -o $out.tmp " +
"$debugFlags -p $pkgPath -complete $incFlags -pack $in && " +
"if cmp --quiet $out.tmp $out; then rm $out.tmp; else mv -f $out.tmp $out; fi",
CommandDeps: []string{"$compileCmd"},
Description: "compile $out",
Restat: true,
},
"pkgPath", "incFlags")
link = pctx.StaticRule("link",
blueprint.RuleParams{
Command: "GOROOT='$goRoot' $linkCmd -o $out.tmp $libDirFlags $in && " +
"if cmp --quiet $out.tmp $out; then rm $out.tmp; else mv -f $out.tmp $out; fi",
CommandDeps: []string{"$linkCmd"},
Description: "link $out",
Restat: true,
},
"libDirFlags")
goTestMain = pctx.StaticRule("gotestmain",
blueprint.RuleParams{
Command: "$goTestMainCmd -o $out -pkg $pkg $in",
CommandDeps: []string{"$goTestMainCmd"},
Description: "gotestmain $out",
},
"pkg")
pluginGenSrc = pctx.StaticRule("pluginGenSrc",
blueprint.RuleParams{
Command: "$pluginGenSrcCmd -o $out -p $pkg $plugins",
CommandDeps: []string{"$pluginGenSrcCmd"},
Description: "create $out",
},
"pkg", "plugins")
test = pctx.StaticRule("test",
blueprint.RuleParams{
Command: "$goTestRunnerCmd -p $pkgSrcDir -f $out -- $in -test.short",
CommandDeps: []string{"$goTestRunnerCmd"},
Description: "test $pkg",
},
"pkg", "pkgSrcDir")
cp = pctx.StaticRule("cp",
blueprint.RuleParams{
Command: "cp $in $out",
Description: "cp $out",
},
"generator")
bootstrap = pctx.StaticRule("bootstrap",
blueprint.RuleParams{
Command: "BUILDDIR=$soongOutDir $bootstrapCmd -i $in",
CommandDeps: []string{"$bootstrapCmd"},
Description: "bootstrap $in",
Generator: true,
})
touch = pctx.StaticRule("touch",
blueprint.RuleParams{
Command: "touch $out",
Description: "touch $out",
},
"depfile", "generator")
generateBuildNinja = pctx.StaticRule("build.ninja",
blueprint.RuleParams{
// TODO: it's kinda ugly that some parameters are computed from
// environment variables and some from Ninja parameters, but it's probably
// better to not to touch that while Blueprint and Soong are separate
// NOTE: The spaces at EOL are important because otherwise Ninja would
// omit all spaces between the different options.
Command: `cd "$$(dirname "$builder")" && ` +
`BUILDER="$$PWD/$$(basename "$builder")" && ` +
`cd / && ` +
`env -i "$$BUILDER" ` +
` --top "$$TOP" ` +
` --soong_out "$soongOutDir" ` +
` --out "$outDir" ` +
` $extra`,
CommandDeps: []string{"$builder"},
Description: "$builder $out",
Deps: blueprint.DepsGCC,
Depfile: "$out.d",
Restat: true,
},
"builder", "extra")
// Work around a Ninja issue. See https://github.com/martine/ninja/pull/634
phony = pctx.StaticRule("phony",
blueprint.RuleParams{
Command: "# phony $out",
Description: "phony $out",
Generator: true,
},
"depfile")
_ = pctx.VariableFunc("ToolDir", func(config interface{}) (string, error) {
return config.(BootstrapConfig).HostToolDir(), nil
})
)
type GoBinaryTool interface {
InstallPath() string
// So that other packages can't implement this interface
isGoBinary()
}
func pluginDeps(ctx blueprint.BottomUpMutatorContext) {
if pkg, ok := ctx.Module().(*goPackage); ok {
if ctx.PrimaryModule() == ctx.Module() {
for _, plugin := range pkg.properties.PluginFor {
ctx.AddReverseDependency(ctx.Module(), nil, plugin)
}
}
}
}
type goPackageProducer interface {
GoPkgRoot() string
GoPackageTarget() string
GoTestTargets() []string
}
func isGoPackageProducer(module blueprint.Module) bool {
_, ok := module.(goPackageProducer)
return ok
}
type goPluginProvider interface {
GoPkgPath() string
IsPluginFor(string) bool
}
func isGoPluginFor(name string) func(blueprint.Module) bool {
return func(module blueprint.Module) bool {
if plugin, ok := module.(goPluginProvider); ok {
return plugin.IsPluginFor(name)
}
return false
}
}
func IsBootstrapModule(module blueprint.Module) bool {
_, isPackage := module.(*goPackage)
_, isBinary := module.(*goBinary)
return isPackage || isBinary
}
func isBootstrapBinaryModule(module blueprint.Module) bool {
_, isBinary := module.(*goBinary)
return isBinary
}
// A goPackage is a module for building Go packages.
type goPackage struct {
blueprint.SimpleName
properties struct {
Deps []string
PkgPath string
Srcs []string
TestSrcs []string
PluginFor []string
Darwin struct {
Srcs []string
TestSrcs []string
}
Linux struct {
Srcs []string
TestSrcs []string
}
}
// The root dir in which the package .a file is located. The full .a file
// path will be "packageRoot/PkgPath.a"
pkgRoot string
// The path of the .a file that is to be built.
archiveFile string
// The path of the test result file.
testResultFile []string
}
var _ goPackageProducer = (*goPackage)(nil)
func newGoPackageModuleFactory() func() (blueprint.Module, []interface{}) {
return func() (blueprint.Module, []interface{}) {
module := &goPackage{}
return module, []interface{}{&module.properties, &module.SimpleName.Properties}
}
}
func (g *goPackage) DynamicDependencies(ctx blueprint.DynamicDependerModuleContext) []string {
if ctx.Module() != ctx.PrimaryModule() {
return nil
}
return g.properties.Deps
}
func (g *goPackage) GoPkgPath() string {
return g.properties.PkgPath
}
func (g *goPackage) GoPkgRoot() string {
return g.pkgRoot
}
func (g *goPackage) GoPackageTarget() string {
return g.archiveFile
}
func (g *goPackage) GoTestTargets() []string {
return g.testResultFile
}
func (g *goPackage) IsPluginFor(name string) bool {
for _, plugin := range g.properties.PluginFor {
if plugin == name {
return true
}
}
return false
}
func (g *goPackage) GenerateBuildActions(ctx blueprint.ModuleContext) {
// Allow the primary builder to create multiple variants. Any variants after the first
// will copy outputs from the first.
if ctx.Module() != ctx.PrimaryModule() {
primary := ctx.PrimaryModule().(*goPackage)
g.pkgRoot = primary.pkgRoot
g.archiveFile = primary.archiveFile
g.testResultFile = primary.testResultFile
return
}
var (
name = ctx.ModuleName()
hasPlugins = false
pluginSrc = ""
genSrcs = []string{}
)
if g.properties.PkgPath == "" {
ctx.ModuleErrorf("module %s did not specify a valid pkgPath", name)
return
}
g.pkgRoot = packageRoot(ctx)
g.archiveFile = filepath.Join(g.pkgRoot,
filepath.FromSlash(g.properties.PkgPath)+".a")
ctx.VisitDepsDepthFirstIf(isGoPluginFor(name),
func(module blueprint.Module) { hasPlugins = true })
if hasPlugins {
pluginSrc = filepath.Join(moduleGenSrcDir(ctx), "plugin.go")
genSrcs = append(genSrcs, pluginSrc)
}
if hasPlugins && !buildGoPluginLoader(ctx, g.properties.PkgPath, pluginSrc) {
return
}
var srcs, testSrcs []string
if runtime.GOOS == "darwin" {
srcs = append(g.properties.Srcs, g.properties.Darwin.Srcs...)
testSrcs = append(g.properties.TestSrcs, g.properties.Darwin.TestSrcs...)
} else if runtime.GOOS == "linux" {
srcs = append(g.properties.Srcs, g.properties.Linux.Srcs...)
testSrcs = append(g.properties.TestSrcs, g.properties.Linux.TestSrcs...)
}
if ctx.Config().(BootstrapConfig).RunGoTests() {
testArchiveFile := filepath.Join(testRoot(ctx),
filepath.FromSlash(g.properties.PkgPath)+".a")
g.testResultFile = buildGoTest(ctx, testRoot(ctx), testArchiveFile,
g.properties.PkgPath, srcs, genSrcs, testSrcs)
}
// Don't build for test-only packages
if len(srcs) == 0 && len(genSrcs) == 0 {
ctx.Build(pctx, blueprint.BuildParams{
Rule: touch,
Outputs: []string{g.archiveFile},
Optional: true,
})
return
}
buildGoPackage(ctx, g.pkgRoot, g.properties.PkgPath, g.archiveFile,
srcs, genSrcs)
}
// A goBinary is a module for building executable binaries from Go sources.
type goBinary struct {
blueprint.SimpleName
properties struct {
Deps []string
Srcs []string
TestSrcs []string
PrimaryBuilder bool
Default bool
Darwin struct {
Srcs []string
TestSrcs []string
}
Linux struct {
Srcs []string
TestSrcs []string
}
}
installPath string
}
var _ GoBinaryTool = (*goBinary)(nil)
func newGoBinaryModuleFactory() func() (blueprint.Module, []interface{}) {
return func() (blueprint.Module, []interface{}) {
module := &goBinary{}
return module, []interface{}{&module.properties, &module.SimpleName.Properties}
}
}
func (g *goBinary) DynamicDependencies(ctx blueprint.DynamicDependerModuleContext) []string {
if ctx.Module() != ctx.PrimaryModule() {
return nil
}
return g.properties.Deps
}
func (g *goBinary) isGoBinary() {}
func (g *goBinary) InstallPath() string {
return g.installPath
}
func (g *goBinary) GenerateBuildActions(ctx blueprint.ModuleContext) {
// Allow the primary builder to create multiple variants. Any variants after the first
// will copy outputs from the first.
if ctx.Module() != ctx.PrimaryModule() {
primary := ctx.PrimaryModule().(*goBinary)
g.installPath = primary.installPath
return
}
var (
name = ctx.ModuleName()
objDir = moduleObjDir(ctx)
archiveFile = filepath.Join(objDir, name+".a")
testArchiveFile = filepath.Join(testRoot(ctx), name+".a")
aoutFile = filepath.Join(objDir, "a.out")
hasPlugins = false
pluginSrc = ""
genSrcs = []string{}
)
g.installPath = filepath.Join(ctx.Config().(BootstrapConfig).HostToolDir(), name)
ctx.VisitDepsDepthFirstIf(isGoPluginFor(name),
func(module blueprint.Module) { hasPlugins = true })
if hasPlugins {
pluginSrc = filepath.Join(moduleGenSrcDir(ctx), "plugin.go")
genSrcs = append(genSrcs, pluginSrc)
}
var testDeps []string
if hasPlugins && !buildGoPluginLoader(ctx, "main", pluginSrc) {
return
}
var srcs, testSrcs []string
if runtime.GOOS == "darwin" {
srcs = append(g.properties.Srcs, g.properties.Darwin.Srcs...)
testSrcs = append(g.properties.TestSrcs, g.properties.Darwin.TestSrcs...)
} else if runtime.GOOS == "linux" {
srcs = append(g.properties.Srcs, g.properties.Linux.Srcs...)
testSrcs = append(g.properties.TestSrcs, g.properties.Linux.TestSrcs...)
}
if ctx.Config().(BootstrapConfig).RunGoTests() {
testDeps = buildGoTest(ctx, testRoot(ctx), testArchiveFile,
name, srcs, genSrcs, testSrcs)
}
buildGoPackage(ctx, objDir, "main", archiveFile, srcs, genSrcs)
var linkDeps []string
var libDirFlags []string
ctx.VisitDepsDepthFirstIf(isGoPackageProducer,
func(module blueprint.Module) {
dep := module.(goPackageProducer)
linkDeps = append(linkDeps, dep.GoPackageTarget())
libDir := dep.GoPkgRoot()
libDirFlags = append(libDirFlags, "-L "+libDir)
testDeps = append(testDeps, dep.GoTestTargets()...)
})
linkArgs := map[string]string{}
if len(libDirFlags) > 0 {
linkArgs["libDirFlags"] = strings.Join(libDirFlags, " ")
}
ctx.Build(pctx, blueprint.BuildParams{
Rule: link,
Outputs: []string{aoutFile},
Inputs: []string{archiveFile},
Implicits: linkDeps,
Args: linkArgs,
Optional: true,
})
var orderOnlyDeps, validationDeps []string
if ctx.Config().(BootstrapConfig).UseValidationsForGoTests() {
validationDeps = testDeps
} else {
orderOnlyDeps = testDeps
}
ctx.Build(pctx, blueprint.BuildParams{
Rule: cp,
Outputs: []string{g.installPath},
Inputs: []string{aoutFile},
OrderOnly: orderOnlyDeps,
Validations: validationDeps,
Optional: !g.properties.Default,
})
}
func buildGoPluginLoader(ctx blueprint.ModuleContext, pkgPath, pluginSrc string) bool {
ret := true
name := ctx.ModuleName()
var pluginPaths []string
ctx.VisitDepsDepthFirstIf(isGoPluginFor(name),
func(module blueprint.Module) {
plugin := module.(goPluginProvider)
pluginPaths = append(pluginPaths, plugin.GoPkgPath())
})
ctx.Build(pctx, blueprint.BuildParams{
Rule: pluginGenSrc,
Outputs: []string{pluginSrc},
Args: map[string]string{
"pkg": pkgPath,
"plugins": strings.Join(pluginPaths, " "),
},
Optional: true,
})
return ret
}
func buildGoPackage(ctx blueprint.ModuleContext, pkgRoot string,
pkgPath string, archiveFile string, srcs []string, genSrcs []string) {
srcDir := moduleSrcDir(ctx)
srcFiles := pathtools.PrefixPaths(srcs, srcDir)
srcFiles = append(srcFiles, genSrcs...)
var incFlags []string
var deps []string
ctx.VisitDepsDepthFirstIf(isGoPackageProducer,
func(module blueprint.Module) {
dep := module.(goPackageProducer)
incDir := dep.GoPkgRoot()
target := dep.GoPackageTarget()
incFlags = append(incFlags, "-I "+incDir)
deps = append(deps, target)
})
compileArgs := map[string]string{
"pkgPath": pkgPath,
}
if len(incFlags) > 0 {
compileArgs["incFlags"] = strings.Join(incFlags, " ")
}
ctx.Build(pctx, blueprint.BuildParams{
Rule: compile,
Outputs: []string{archiveFile},
Inputs: srcFiles,
Implicits: deps,
Args: compileArgs,
Optional: true,
})
}
func buildGoTest(ctx blueprint.ModuleContext, testRoot, testPkgArchive,
pkgPath string, srcs, genSrcs, testSrcs []string) []string {
if len(testSrcs) == 0 {
return nil
}
srcDir := moduleSrcDir(ctx)
testFiles := pathtools.PrefixPaths(testSrcs, srcDir)
mainFile := filepath.Join(testRoot, "test.go")
testArchive := filepath.Join(testRoot, "test.a")
testFile := filepath.Join(testRoot, "test")
testPassed := filepath.Join(testRoot, "test.passed")
buildGoPackage(ctx, testRoot, pkgPath, testPkgArchive,
append(srcs, testSrcs...), genSrcs)
ctx.Build(pctx, blueprint.BuildParams{
Rule: goTestMain,
Outputs: []string{mainFile},
Inputs: testFiles,
Args: map[string]string{
"pkg": pkgPath,
},
Optional: true,
})
linkDeps := []string{testPkgArchive}
libDirFlags := []string{"-L " + testRoot}
testDeps := []string{}
ctx.VisitDepsDepthFirstIf(isGoPackageProducer,
func(module blueprint.Module) {
dep := module.(goPackageProducer)
linkDeps = append(linkDeps, dep.GoPackageTarget())
libDir := dep.GoPkgRoot()
libDirFlags = append(libDirFlags, "-L "+libDir)
testDeps = append(testDeps, dep.GoTestTargets()...)
})
ctx.Build(pctx, blueprint.BuildParams{
Rule: compile,
Outputs: []string{testArchive},
Inputs: []string{mainFile},
Implicits: []string{testPkgArchive},
Args: map[string]string{
"pkgPath": "main",
"incFlags": "-I " + testRoot,
},
Optional: true,
})
ctx.Build(pctx, blueprint.BuildParams{
Rule: link,
Outputs: []string{testFile},
Inputs: []string{testArchive},
Implicits: linkDeps,
Args: map[string]string{
"libDirFlags": strings.Join(libDirFlags, " "),
},
Optional: true,
})
var orderOnlyDeps, validationDeps []string
if ctx.Config().(BootstrapConfig).UseValidationsForGoTests() {
validationDeps = testDeps
} else {
orderOnlyDeps = testDeps
}
ctx.Build(pctx, blueprint.BuildParams{
Rule: test,
Outputs: []string{testPassed},
Inputs: []string{testFile},
OrderOnly: orderOnlyDeps,
Validations: validationDeps,
Args: map[string]string{
"pkg": pkgPath,
"pkgSrcDir": filepath.Dir(testFiles[0]),
},
Optional: true,
})
return []string{testPassed}
}
type singleton struct {
}
func newSingletonFactory() func() blueprint.Singleton {
return func() blueprint.Singleton {
return &singleton{}
}
}
func (s *singleton) GenerateBuildActions(ctx blueprint.SingletonContext) {
// Find the module that's marked as the "primary builder", which means it's
// creating the binary that we'll use to generate the non-bootstrap
// build.ninja file.
var primaryBuilders []*goBinary
// blueprintTools contains blueprint go binaries that will be built in StageMain
var blueprintTools []string
// blueprintGoPackages contains all blueprint go packages that can be built in StageMain
var blueprintGoPackages []string
ctx.VisitAllModulesIf(IsBootstrapModule,
func(module blueprint.Module) {
if ctx.PrimaryModule(module) == module {
if binaryModule, ok := module.(*goBinary); ok {
blueprintTools = append(blueprintTools, binaryModule.InstallPath())
if binaryModule.properties.PrimaryBuilder {
primaryBuilders = append(primaryBuilders, binaryModule)
}
}
if packageModule, ok := module.(*goPackage); ok {
blueprintGoPackages = append(blueprintGoPackages,
packageModule.GoPackageTarget())
blueprintGoPackages = append(blueprintGoPackages,
packageModule.GoTestTargets()...)
}
}
})
var primaryBuilderCmdlinePrefix []string
var primaryBuilderName string
if len(primaryBuilders) == 0 {
ctx.Errorf("no primary builder module present")
return
} else if len(primaryBuilders) > 1 {
ctx.Errorf("multiple primary builder modules present:")
for _, primaryBuilder := range primaryBuilders {
ctx.ModuleErrorf(primaryBuilder, "<-- module %s",
ctx.ModuleName(primaryBuilder))
}
return
} else {
primaryBuilderName = ctx.ModuleName(primaryBuilders[0])
}
primaryBuilderFile := filepath.Join("$ToolDir", primaryBuilderName)
ctx.SetOutDir(pctx, "${outDir}")
for _, subninja := range ctx.Config().(BootstrapConfig).Subninjas() {
ctx.AddSubninja(subninja)
}
for _, i := range ctx.Config().(BootstrapConfig).PrimaryBuilderInvocations() {
flags := make([]string, 0)
flags = append(flags, primaryBuilderCmdlinePrefix...)
flags = append(flags, i.Args...)
// Build the main build.ninja
ctx.Build(pctx, blueprint.BuildParams{
Rule: generateBuildNinja,
Outputs: i.Outputs,
Inputs: i.Inputs,
Args: map[string]string{
"builder": primaryBuilderFile,
"extra": strings.Join(flags, " "),
},
// soong_ui explicitly requests what it wants to be build. This is
// because the same Ninja file contains instructions to run
// soong_build, run bp2build and to generate the JSON module graph.
Optional: true,
})
}
// Add a phony target for building various tools that are part of blueprint
ctx.Build(pctx, blueprint.BuildParams{
Rule: blueprint.Phony,
Outputs: []string{"blueprint_tools"},
Inputs: blueprintTools,
})
// Add a phony target for running go tests
ctx.Build(pctx, blueprint.BuildParams{
Rule: blueprint.Phony,
Outputs: []string{"blueprint_go_packages"},
Inputs: blueprintGoPackages,
Optional: true,
})
}
// packageRoot returns the module-specific package root directory path. This
// directory is where the final package .a files are output and where dependant
// modules search for this package via -I arguments.
func packageRoot(ctx blueprint.ModuleContext) string {
toolDir := ctx.Config().(BootstrapConfig).HostToolDir()
return filepath.Join(toolDir, "go", ctx.ModuleName(), "pkg")
}
// testRoot returns the module-specific package root directory path used for
// building tests. The .a files generated here will include everything from
// packageRoot, plus the test-only code.
func testRoot(ctx blueprint.ModuleContext) string {
toolDir := ctx.Config().(BootstrapConfig).HostToolDir()
return filepath.Join(toolDir, "go", ctx.ModuleName(), "test")
}
// moduleSrcDir returns the path of the directory that all source file paths are
// specified relative to.
func moduleSrcDir(ctx blueprint.ModuleContext) string {
return ctx.ModuleDir()
}
// moduleObjDir returns the module-specific object directory path.
func moduleObjDir(ctx blueprint.ModuleContext) string {
toolDir := ctx.Config().(BootstrapConfig).HostToolDir()
return filepath.Join(toolDir, "go", ctx.ModuleName(), "obj")
}
// moduleGenSrcDir returns the module-specific generated sources path.
func moduleGenSrcDir(ctx blueprint.ModuleContext) string {
toolDir := ctx.Config().(BootstrapConfig).HostToolDir()
return filepath.Join(toolDir, "go", ctx.ModuleName(), "gen")
}