ninja.go - platform/build/kati - Gitiles

 // Copyright 2015 Google Inc. All rights reserved
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
 //      http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.

 package kati

 import (
 	"bytes"
 	"fmt"
 	"os"
 	"path/filepath"
 	"regexp"
 	"runtime"
 	"strings"
 	"time"
 )

 type ninjaGenerator struct {
 	f       *os.File
 	nodes   []*DepNode
 	exports map[string]bool

 	ctx *execContext

 	ruleID  int
 	done    map[string]bool
 	gomaDir string
 }

 var ccRE = regexp.MustCompile(`^prebuilts/(gcc|clang)/.*(gcc|g\+\+|clang|clang\+\+) .* -c `)

 func newNinjaGenerator(g *DepGraph, gomaDir string) *ninjaGenerator {
 	ctx := newExecContext(g.vars, true)
 	return &ninjaGenerator{
 		nodes:   g.nodes,
 		exports: g.exports,
 		ctx:     ctx,
 		done:    make(map[string]bool),
 		gomaDir: gomaDir,
 	}
 }

 func getDepfileImpl(ss string) (string, error) {
 	tss := ss + " "
 	if !strings.Contains(tss, " -MD ") && !strings.Contains(tss, " -MMD ") {
 		return "", nil
 	}

 	mfIndex := strings.Index(ss, " -MF ")
 	if mfIndex >= 0 {
 		mf := trimLeftSpace(ss[mfIndex+4:])
 		if strings.Index(mf, " -MF ") >= 0 {
 			return "", fmt.Errorf("Multiple output file candidates in %s", ss)
 		}
 		mfEndIndex := strings.IndexAny(mf, " \t\n")
 		if mfEndIndex >= 0 {
 			mf = mf[:mfEndIndex]
 		}

 		return mf, nil
 	}

 	outIndex := strings.Index(ss, " -o ")
 	if outIndex < 0 {
 		return "", fmt.Errorf("Cannot find the depfile in %s", ss)
 	}
 	out := trimLeftSpace(ss[outIndex+4:])
 	if strings.Index(out, " -o ") >= 0 {
 		return "", fmt.Errorf("Multiple output file candidates in %s", ss)
 	}
 	outEndIndex := strings.IndexAny(out, " \t\n")
 	if outEndIndex >= 0 {
 		out = out[:outEndIndex]
 	}
 	return stripExt(out) + ".d", nil
 }

 func getDepfile(ss string) (string, error) {
 	// A hack for Android - llvm-rs-cc seems not to emit a dep file.
 	if strings.Contains(ss, "bin/llvm-rs-cc ") {
 		return "", nil
 	}

 	r, err := getDepfileImpl(ss)
 	if r == "" || err != nil {
 		return r, err
 	}

 	// A hack for Makefiles generated by automake.
 	mvCmd := "(mv -f " + r + " "
 	if i := strings.LastIndex(ss, mvCmd); i >= 0 {
 		rest := ss[i+len(mvCmd):]
 		ei := strings.IndexByte(rest, ')')
 		if ei < 0 {
 			return "", fmt.Errorf("unbalanced parenthes? %s", ss)
 		}
 		return rest[:ei], nil
 	}

 	// A hack for Android to get .P files instead of .d.
 	p := stripExt(r) + ".P"
 	if strings.Contains(ss, p) {
 		return p, nil
 	}

 	// A hack for Android. For .s files, GCC does not use
 	// C preprocessor, so it ignores -MF flag.
 	as := "/" + stripExt(filepath.Base(r)) + ".s"
 	if strings.Contains(ss, as) {
 		return "", nil
 	}

 	return r, nil
 }

 func stripShellComment(s string) string {
 	if strings.IndexByte(s, '#') < 0 {
 		// Fast path.
 		return s
 	}
 	var escape bool
 	var quote rune
 	for i, c := range s {
 		if quote > 0 {
 			if quote == c && (quote == '\'' || !escape) {
 				quote = 0
 			}
 		} else if !escape {
 			if c == '#' {
 				return s[:i]
 			} else if c == '\'' || c == '"' || c == '`' {
 				quote = c
 			}
 		}
 		if escape {
 			escape = false
 		} else if c == '\\' {
 			escape = true
 		} else {
 			escape = false
 		}
 	}
 	return s
 }

 func (n *ninjaGenerator) genShellScript(runners []runner) (string, bool) {
 	useGomacc := false
 	var buf bytes.Buffer
 	for i, r := range runners {
 		if i > 0 {
 			if runners[i-1].ignoreError {
 				buf.WriteString(" ; ")
 			} else {
 				buf.WriteString(" && ")
 			}
 		}
 		cmd := stripShellComment(r.cmd)
 		cmd = trimLeftSpace(cmd)
 		cmd = strings.Replace(cmd, "\\\n", " ", -1)
 		cmd = strings.TrimRight(cmd, " \t\n;")
 		cmd = strings.Replace(cmd, "$", "$$", -1)
 		cmd = strings.Replace(cmd, "\t", " ", -1)
 		if cmd == "" {
 			cmd = "true"
 		}
 		if n.gomaDir != "" && ccRE.MatchString(cmd) {
 			cmd = fmt.Sprintf("%s/gomacc %s", n.gomaDir, cmd)
 			useGomacc = true
 		}

 		needsSubShell := i > 0 || len(runners) > 1
 		if cmd[0] == '(' {
 			needsSubShell = false
 		}

 		if needsSubShell {
 			buf.WriteByte('(')
 		}
 		buf.WriteString(cmd)
 		if i == len(runners)-1 && r.ignoreError {
 			buf.WriteString(" ; true")
 		}
 		if needsSubShell {
 			buf.WriteByte(')')
 		}
 	}
 	return buf.String(), n.gomaDir != "" && !useGomacc
 }

 func (n *ninjaGenerator) genRuleName() string {
 	ruleName := fmt.Sprintf("rule%d", n.ruleID)
 	n.ruleID++
 	return ruleName
 }

 func (n *ninjaGenerator) emitBuild(output, rule, dep string) {
 	fmt.Fprintf(n.f, "build %s: %s%s\n", output, rule, dep)
 }

 func getDepString(node *DepNode) string {
 	var deps []string
 	for _, d := range node.Deps {
 		deps = append(deps, d.Output)
 	}
 	var orderOnlys []string
 	for _, d := range node.OrderOnlys {
 		orderOnlys = append(orderOnlys, d.Output)
 	}
 	dep := ""
 	if len(deps) > 0 {
 		dep += fmt.Sprintf(" %s", strings.Join(deps, " "))
 	}
 	if len(orderOnlys) > 0 {
 		dep += fmt.Sprintf(" || %s", strings.Join(orderOnlys, " "))
 	}
 	return dep
 }

 func (n *ninjaGenerator) emitNode(node *DepNode) error {
 	if n.done[node.Output] {
 		return nil
 	}
 	n.done[node.Output] = true

 	if len(node.Cmds) == 0 && len(node.Deps) == 0 && len(node.OrderOnlys) == 0 && !node.IsPhony {
 		return nil
 	}

 	runners, _, err := createRunners(n.ctx, node)
 	if err != nil {
 		return err
 	}
 	ruleName := "phony"
 	useLocalPool := false
 	if len(runners) > 0 {
 		ruleName = n.genRuleName()
 		fmt.Fprintf(n.f, "\n# rule for %s\n", node.Output)
 		fmt.Fprintf(n.f, "rule %s\n", ruleName)
 		fmt.Fprintf(n.f, " description = build $out\n")

 		ss, ulp := n.genShellScript(runners)
 		if ulp {
 			useLocalPool = true
 		}
 		depfile, err := getDepfile(ss)
 		if err != nil {
 			return err
 		}
 		if depfile != "" {
 			fmt.Fprintf(n.f, " depfile = %s\n", depfile)
 		}
 		// It seems Linux is OK with ~130kB.
 		// TODO: Find this number automatically.
 		ArgLenLimit := 100 * 1000
 		if len(ss) > ArgLenLimit {
 			fmt.Fprintf(n.f, " rspfile = $out.rsp\n")
 			fmt.Fprintf(n.f, " rspfile_content = %s\n", ss)
 			ss = "sh $out.rsp"
 		}
 		fmt.Fprintf(n.f, " command = %s\n", ss)

 	}
 	n.emitBuild(node.Output, ruleName, getDepString(node))
 	if useLocalPool {
 		fmt.Fprintf(n.f, " pool = local_pool\n")
 	}
 	fmt.Fprintf(n.f, "\n")

 	for _, d := range node.Deps {
 		err := n.emitNode(d)
 		if err != nil {
 			return err
 		}
 	}
 	for _, d := range node.OrderOnlys {
 		err := n.emitNode(d)
 		if err != nil {
 			return err
 		}
 	}
 	return nil
 }

 func (n *ninjaGenerator) generateShell() (err error) {
 	f, err := os.Create("ninja.sh")
 	if err != nil {
 		return err
 	}
 	defer func() {
 		cerr := f.Close()
 		if err == nil {
 			err = cerr
 		}
 	}()

 	fmt.Fprintf(f, "#!%s\n", n.ctx.shell)
 	for name, export := range n.exports {
 		if export {
 			v, err := n.ctx.ev.EvaluateVar(name)
 			if err != nil {
 				return err
 			}
 			fmt.Fprintf(f, "export %s=%s\n", name, v)
 		} else {
 			fmt.Fprintf(f, "unset %s\n", name)
 		}
 	}
 	if n.gomaDir == "" {
 		fmt.Fprintln(f, `exec ninja "$@"`)
 	} else {
 		fmt.Fprintln(f, `exec ninja -j300 "$@"`)
 	}

 	return f.Chmod(0755)
 }

 func (n *ninjaGenerator) generateNinja() (err error) {
 	f, err := os.Create("build.ninja")
 	if err != nil {
 		return err
 	}
 	defer func() {
 		cerr := f.Close()
 		if err == nil {
 			err = cerr
 		}
 	}()

 	n.f = f
 	fmt.Fprintf(n.f, "# Generated by kati\n")
 	fmt.Fprintf(n.f, "\n")

 	if n.gomaDir != "" {
 		fmt.Fprintf(n.f, "pool local_pool\n")
 		fmt.Fprintf(n.f, " depth = %d\n", runtime.NumCPU())
 	}

 	for _, node := range n.nodes {
 		err := n.emitNode(node)
 		if err != nil {
 			return err
 		}
 	}
 	return nil
 }

 // GenerateNinja generates build.ninja from DepGraph.
 func GenerateNinja(g *DepGraph, gomaDir string) error {
 	startTime := time.Now()
 	n := newNinjaGenerator(g, gomaDir)
 	err := n.generateShell()
 	if err != nil {
 		return err
 	}
 	err = n.generateNinja()
 	if err != nil {
 		return err
 	}
 	logStats("generate ninja time: %q", time.Since(startTime))
 	return nil
 }
	// Copyright 2015 Google Inc. All rights reserved
	//
	// Licensed under the Apache License, Version 2.0 (the "License");
	// you may not use this file except in compliance with the License.
	// You may obtain a copy of the License at
	//
	// http://www.apache.org/licenses/LICENSE-2.0
	//
	// Unless required by applicable law or agreed to in writing, software
	// distributed under the License is distributed on an "AS IS" BASIS,
	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	// See the License for the specific language governing permissions and
	// limitations under the License.

	package kati

	import (
	"bytes"
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"
	"time"
	)

	type ninjaGenerator struct {
	f *os.File
	nodes []*DepNode
	exports map[string]bool

	ctx *execContext

	ruleID int
	done map[string]bool
	gomaDir string
	}

	var ccRE = regexp.MustCompile(`^prebuilts/(gcc\|clang)/.(gcc\|g\+\+\|clang\|clang\+\+) . -c `)

	func newNinjaGenerator(g DepGraph, gomaDir string) ninjaGenerator {
	ctx := newExecContext(g.vars, true)
	return &ninjaGenerator{
	nodes: g.nodes,
	exports: g.exports,
	ctx: ctx,
	done: make(map[string]bool),
	gomaDir: gomaDir,
	}
	}

	func getDepfileImpl(ss string) (string, error) {
	tss := ss + " "
	if !strings.Contains(tss, " -MD ") && !strings.Contains(tss, " -MMD ") {
	return "", nil
	}

	mfIndex := strings.Index(ss, " -MF ")
	if mfIndex >= 0 {
	mf := trimLeftSpace(ss[mfIndex+4:])
	if strings.Index(mf, " -MF ") >= 0 {
	return "", fmt.Errorf("Multiple output file candidates in %s", ss)
	}
	mfEndIndex := strings.IndexAny(mf, " \t\n")
	if mfEndIndex >= 0 {
	mf = mf[:mfEndIndex]
	}

	return mf, nil
	}

	outIndex := strings.Index(ss, " -o ")
	if outIndex < 0 {
	return "", fmt.Errorf("Cannot find the depfile in %s", ss)
	}
	out := trimLeftSpace(ss[outIndex+4:])
	if strings.Index(out, " -o ") >= 0 {
	return "", fmt.Errorf("Multiple output file candidates in %s", ss)
	}
	outEndIndex := strings.IndexAny(out, " \t\n")
	if outEndIndex >= 0 {
	out = out[:outEndIndex]
	}
	return stripExt(out) + ".d", nil
	}

	func getDepfile(ss string) (string, error) {
	// A hack for Android - llvm-rs-cc seems not to emit a dep file.
	if strings.Contains(ss, "bin/llvm-rs-cc ") {
	return "", nil
	}

	r, err := getDepfileImpl(ss)
	if r == "" \|\| err != nil {
	return r, err
	}

	// A hack for Makefiles generated by automake.
	mvCmd := "(mv -f " + r + " "
	if i := strings.LastIndex(ss, mvCmd); i >= 0 {
	rest := ss[i+len(mvCmd):]
	ei := strings.IndexByte(rest, ')')
	if ei < 0 {
	return "", fmt.Errorf("unbalanced parenthes? %s", ss)
	}
	return rest[:ei], nil
	}

	// A hack for Android to get .P files instead of .d.
	p := stripExt(r) + ".P"
	if strings.Contains(ss, p) {
	return p, nil
	}

	// A hack for Android. For .s files, GCC does not use
	// C preprocessor, so it ignores -MF flag.
	as := "/" + stripExt(filepath.Base(r)) + ".s"
	if strings.Contains(ss, as) {
	return "", nil
	}

	return r, nil
	}

	func stripShellComment(s string) string {
	if strings.IndexByte(s, '#') < 0 {
	// Fast path.
	return s
	}
	var escape bool
	var quote rune
	for i, c := range s {
	if quote > 0 {
	if quote == c && (quote == '\'' \|\| !escape) {
	quote = 0
	}
	} else if !escape {
	if c == '#' {
	return s[:i]
	} else if c == '\'' \|\| c == '"' \|\| c == '`' {
	quote = c
	}
	}
	if escape {
	escape = false
	} else if c == '\\' {
	escape = true
	} else {
	escape = false
	}
	}
	return s
	}

	func (n *ninjaGenerator) genShellScript(runners []runner) (string, bool) {
	useGomacc := false
	var buf bytes.Buffer
	for i, r := range runners {
	if i > 0 {
	if runners[i-1].ignoreError {
	buf.WriteString(" ; ")
	} else {
	buf.WriteString(" && ")
	}
	}
	cmd := stripShellComment(r.cmd)
	cmd = trimLeftSpace(cmd)
	cmd = strings.Replace(cmd, "\\\n", " ", -1)
	cmd = strings.TrimRight(cmd, " \t\n;")
	cmd = strings.Replace(cmd, "$", "$$", -1)
	cmd = strings.Replace(cmd, "\t", " ", -1)
	if cmd == "" {
	cmd = "true"
	}
	if n.gomaDir != "" && ccRE.MatchString(cmd) {
	cmd = fmt.Sprintf("%s/gomacc %s", n.gomaDir, cmd)
	useGomacc = true
	}

	needsSubShell := i > 0 \|\| len(runners) > 1
	if cmd[0] == '(' {
	needsSubShell = false
	}

	if needsSubShell {
	buf.WriteByte('(')
	}
	buf.WriteString(cmd)
	if i == len(runners)-1 && r.ignoreError {
	buf.WriteString(" ; true")
	}
	if needsSubShell {
	buf.WriteByte(')')
	}
	}
	return buf.String(), n.gomaDir != "" && !useGomacc
	}

	func (n *ninjaGenerator) genRuleName() string {
	ruleName := fmt.Sprintf("rule%d", n.ruleID)
	n.ruleID++
	return ruleName
	}

	func (n *ninjaGenerator) emitBuild(output, rule, dep string) {
	fmt.Fprintf(n.f, "build %s: %s%s\n", output, rule, dep)
	}

	func getDepString(node *DepNode) string {
	var deps []string
	for _, d := range node.Deps {
	deps = append(deps, d.Output)
	}
	var orderOnlys []string
	for _, d := range node.OrderOnlys {
	orderOnlys = append(orderOnlys, d.Output)
	}
	dep := ""
	if len(deps) > 0 {
	dep += fmt.Sprintf(" %s", strings.Join(deps, " "))
	}
	if len(orderOnlys) > 0 {
	dep += fmt.Sprintf(" \|\| %s", strings.Join(orderOnlys, " "))
	}
	return dep
	}

	func (n ninjaGenerator) emitNode(node DepNode) error {
	if n.done[node.Output] {
	return nil
	}
	n.done[node.Output] = true

	if len(node.Cmds) == 0 && len(node.Deps) == 0 && len(node.OrderOnlys) == 0 && !node.IsPhony {
	return nil
	}

	runners, _, err := createRunners(n.ctx, node)
	if err != nil {
	return err
	}
	ruleName := "phony"
	useLocalPool := false
	if len(runners) > 0 {
	ruleName = n.genRuleName()
	fmt.Fprintf(n.f, "\n# rule for %s\n", node.Output)
	fmt.Fprintf(n.f, "rule %s\n", ruleName)
	fmt.Fprintf(n.f, " description = build $out\n")

	ss, ulp := n.genShellScript(runners)
	if ulp {
	useLocalPool = true
	}
	depfile, err := getDepfile(ss)
	if err != nil {
	return err
	}
	if depfile != "" {
	fmt.Fprintf(n.f, " depfile = %s\n", depfile)
	}
	// It seems Linux is OK with ~130kB.
	// TODO: Find this number automatically.
	ArgLenLimit := 100 * 1000
	if len(ss) > ArgLenLimit {
	fmt.Fprintf(n.f, " rspfile = $out.rsp\n")
	fmt.Fprintf(n.f, " rspfile_content = %s\n", ss)
	ss = "sh $out.rsp"
	}
	fmt.Fprintf(n.f, " command = %s\n", ss)

	}
	n.emitBuild(node.Output, ruleName, getDepString(node))
	if useLocalPool {
	fmt.Fprintf(n.f, " pool = local_pool\n")
	}
	fmt.Fprintf(n.f, "\n")

	for _, d := range node.Deps {
	err := n.emitNode(d)
	if err != nil {
	return err
	}
	}
	for _, d := range node.OrderOnlys {
	err := n.emitNode(d)
	if err != nil {
	return err
	}
	}
	return nil
	}

	func (n *ninjaGenerator) generateShell() (err error) {
	f, err := os.Create("ninja.sh")
	if err != nil {
	return err
	}
	defer func() {
	cerr := f.Close()
	if err == nil {
	err = cerr
	}
	}()

	fmt.Fprintf(f, "#!%s\n", n.ctx.shell)
	for name, export := range n.exports {
	if export {
	v, err := n.ctx.ev.EvaluateVar(name)
	if err != nil {
	return err
	}
	fmt.Fprintf(f, "export %s=%s\n", name, v)
	} else {
	fmt.Fprintf(f, "unset %s\n", name)
	}
	}
	if n.gomaDir == "" {
	fmt.Fprintln(f, `exec ninja "$@"`)
	} else {
	fmt.Fprintln(f, `exec ninja -j300 "$@"`)
	}

	return f.Chmod(0755)
	}

	func (n *ninjaGenerator) generateNinja() (err error) {
	f, err := os.Create("build.ninja")
	if err != nil {
	return err
	}
	defer func() {
	cerr := f.Close()
	if err == nil {
	err = cerr
	}
	}()

	n.f = f
	fmt.Fprintf(n.f, "# Generated by kati\n")
	fmt.Fprintf(n.f, "\n")

	if n.gomaDir != "" {
	fmt.Fprintf(n.f, "pool local_pool\n")
	fmt.Fprintf(n.f, " depth = %d\n", runtime.NumCPU())
	}

	for _, node := range n.nodes {
	err := n.emitNode(node)
	if err != nil {
	return err
	}
	}
	return nil
	}

	// GenerateNinja generates build.ninja from DepGraph.
	func GenerateNinja(g *DepGraph, gomaDir string) error {
	startTime := time.Now()
	n := newNinjaGenerator(g, gomaDir)
	err := n.generateShell()
	if err != nil {
	return err
	}
	err = n.generateNinja()
	if err != nil {
	return err
	}
	logStats("generate ninja time: %q", time.Since(startTime))
	return nil
	}