bootstrap/bpglob/bpglob.go

// Copyright 2015 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// bpglob is the command line tool that checks if the list of files matching a glob has
// changed, and only updates the output file list if it has changed.  It is used to optimize
// out build.ninja regenerations when non-matching files are added.  See
// github.com/google/blueprint/bootstrap/glob.go for a longer description.
package main

import (
	"bytes"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"strconv"
	"time"

	"github.com/google/blueprint/deptools"
	"github.com/google/blueprint/pathtools"
)

var (
	// flagSet is a flag.FlagSet with flag.ContinueOnError so that we can handle the versionMismatchError
	// error from versionArg.
	flagSet = flag.NewFlagSet("bpglob", flag.ContinueOnError)

	out = flagSet.String("o", "", "file to write list of files that match glob")

	versionMatch versionArg
	globs        []globArg
)

func init() {
	flagSet.Var(&versionMatch, "v", "version number the command line was generated for")
	flagSet.Var((*patternsArgs)(&globs), "p", "pattern to include in results")
	flagSet.Var((*excludeArgs)(&globs), "e", "pattern to exclude from results from the most recent pattern")
}

// bpglob is executed through the rules in build-globs.ninja to determine whether soong_build
// needs to rerun.  That means when the arguments accepted by bpglob change it will be called
// with the old arguments, then soong_build will rerun and update build-globs.ninja with the new
// arguments.
//
// To avoid having to maintain backwards compatibility with old arguments across the transition,
// a version argument is used to detect the transition in order to stop parsing arguments, touch the
// output file and exit immediately.  Aborting parsing arguments is necessary to handle parsing
// errors that would be fatal, for example the removal of a flag.  The version number in
// pathtools.BPGlobArgumentVersion should be manually incremented when the bpglob argument format
// changes.
//
// If the version argument is not passed then a version mismatch is assumed.

// versionArg checks the argument against pathtools.BPGlobArgumentVersion, returning a
// versionMismatchError error if it does not match.
type versionArg bool

var versionMismatchError = errors.New("version mismatch")

func (v *versionArg) String() string { return "" }

func (v *versionArg) Set(s string) error {
	vers, err := strconv.Atoi(s)
	if err != nil {
		return fmt.Errorf("error parsing version argument: %w", err)
	}

	// Force the -o argument to come before the -v argument so that the output file can be
	// updated on error.
	if *out == "" {
		return fmt.Errorf("-o argument must be passed before -v")
	}

	if vers != pathtools.BPGlobArgumentVersion {
		return versionMismatchError
	}

	*v = true

	return nil
}

// A glob arg holds a single -p argument with zero or more following -e arguments.
type globArg struct {
	pattern  string
	excludes []string
}

// patternsArgs implements flag.Value to handle -p arguments by adding a new globArg to the list.
type patternsArgs []globArg

func (p *patternsArgs) String() string { return `""` }

func (p *patternsArgs) Set(s string) error {
	globs = append(globs, globArg{
		pattern: s,
	})
	return nil
}

// excludeArgs implements flag.Value to handle -e arguments by adding to the last globArg in the
// list.
type excludeArgs []globArg

func (e *excludeArgs) String() string { return `""` }

func (e *excludeArgs) Set(s string) error {
	if len(*e) == 0 {
		return fmt.Errorf("-p argument is required before the first -e argument")
	}

	glob := &(*e)[len(*e)-1]
	glob.excludes = append(glob.excludes, s)
	return nil
}

func usage() {
	fmt.Fprintln(os.Stderr, "usage: bpglob -o out -v version -p glob [-e excludes ...] [-p glob ...]")
	flagSet.PrintDefaults()
	os.Exit(2)
}

func main() {
	// Save the command line flag error output to a buffer, the flag package unconditionally
	// writes an error message to the output on error, and we want to hide the error for the
	// version mismatch case.
	flagErrorBuffer := &bytes.Buffer{}
	flagSet.SetOutput(flagErrorBuffer)

	err := flagSet.Parse(os.Args[1:])

	if !versionMatch {
		// A version mismatch error occurs when the arguments written into build-globs.ninja
		// don't match the format expected by the bpglob binary.  This happens during the
		// first incremental build after bpglob is changed.  Handle this case by aborting
		// argument parsing and updating the output file with something that will always cause
		// the primary builder to rerun.
		// This can happen when there is no -v argument or if the -v argument doesn't match
		// pathtools.BPGlobArgumentVersion.
		writeErrorOutput(*out, versionMismatchError)
		os.Exit(0)
	}

	if err != nil {
		os.Stderr.Write(flagErrorBuffer.Bytes())
		fmt.Fprintln(os.Stderr, "error:", err.Error())
		usage()
	}

	if *out == "" {
		fmt.Fprintln(os.Stderr, "error: -o is required")
		usage()
	}

	if flagSet.NArg() > 0 {
		usage()
	}

	err = globsWithDepFile(*out, *out+".d", globs)
	if err != nil {
		// Globs here were already run in the primary builder without error.  The only errors here should be if the glob
		// pattern was made invalid by a change in the pathtools glob implementation, in which case the primary builder
		// needs to be rerun anyways.  Update the output file with something that will always cause the primary builder
		// to rerun.
		writeErrorOutput(*out, err)
	}
}

// writeErrorOutput writes an error to the output file with a timestamp to ensure that it is
// considered dirty by ninja.
func writeErrorOutput(path string, globErr error) {
	s := fmt.Sprintf("%s: error: %s\n", time.Now().Format(time.StampNano), globErr.Error())
	err := ioutil.WriteFile(path, []byte(s), 0666)
	if err != nil {
		fmt.Fprintf(os.Stderr, "error: %s\n", err.Error())
		os.Exit(1)
	}
}

// globsWithDepFile finds all files and directories that match glob.  Directories
// will have a trailing '/'.  It compares the list of matches against the
// contents of fileListFile, and rewrites fileListFile if it has changed.  It
// also writes all of the directories it traversed as dependencies on fileListFile
// to depFile.
//
// The format of glob is either path/*.ext for a single directory glob, or
// path/**/*.ext for a recursive glob.
func globsWithDepFile(fileListFile, depFile string, globs []globArg) error {
	var results pathtools.MultipleGlobResults
	for _, glob := range globs {
		result, err := pathtools.Glob(glob.pattern, glob.excludes, pathtools.FollowSymlinks)
		if err != nil {
			return err
		}
		results = append(results, result)
	}

	// Only write the output file if it has changed.
	err := pathtools.WriteFileIfChanged(fileListFile, results.FileList(), 0666)
	if err != nil {
		return fmt.Errorf("failed to write file list to %q: %w", fileListFile, err)
	}

	// The depfile can be written unconditionally as its timestamp doesn't affect ninja's restat
	// feature.
	err = deptools.WriteDepFile(depFile, fileListFile, results.Deps())
	if err != nil {
		return fmt.Errorf("failed to write dep file to %q: %w", depFile, err)
	}

	return nil
}