regen.cc

// Copyright 2016 Google Inc. All rights reserved
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build ignore

#include "regen.h"

#include <sys/stat.h>

#include <algorithm>
#include <memory>
#include <mutex>
#include <vector>

#include "affinity.h"
#include "fileutil.h"
#include "find.h"
#include "func.h"
#include "io.h"
#include "log.h"
#include "ninja.h"
#include "stats.h"
#include "strutil.h"
#include "thread_pool.h"

namespace {

#define RETURN_TRUE              \
  do {                           \
    if (g_flags.dump_kati_stamp) \
      needs_regen_ = true;       \
    else                         \
      return true;               \
  } while (0)

bool ShouldIgnoreDirty(StringPiece s) {
  Pattern pat(g_flags.ignore_dirty_pattern);
  Pattern nopat(g_flags.no_ignore_dirty_pattern);
  return pat.Match(s) && !nopat.Match(s);
}

class StampChecker {
  struct GlobResult {
    string pat;
    vector<string> result;
  };

  struct ShellResult {
    CommandOp op;
    string shell;
    string shellflag;
    string cmd;
    string result;
    vector<string> missing_dirs;
    vector<string> files;
    vector<string> read_dirs;
  };

 public:
  StampChecker() : needs_regen_(false) {}

  ~StampChecker() {
    for (GlobResult* gr : globs_) {
      delete gr;
    }
    for (ShellResult* sr : commands_) {
      delete sr;
    }
  }

  bool NeedsRegen(double start_time, const string& orig_args) {
    if (IsMissingOutputs())
      RETURN_TRUE;

    if (CheckStep1(orig_args))
      RETURN_TRUE;

    if (CheckStep2())
      RETURN_TRUE;

    if (!needs_regen_) {
      FILE* fp = fopen(GetNinjaStampFilename().c_str(), "rb+");
      if (!fp)
        return true;
      ScopedFile sfp(fp);
      if (fseek(fp, 0, SEEK_SET) < 0)
        PERROR("fseek");
      size_t r = fwrite(&start_time, sizeof(start_time), 1, fp);
      CHECK(r == 1);
    }
    return needs_regen_;
  }

 private:
  bool IsMissingOutputs() {
    if (!Exists(GetNinjaFilename())) {
      fprintf(stderr, "%s is missing, regenerating...\n",
              GetNinjaFilename().c_str());
      return true;
    }
    if (!Exists(GetNinjaShellScriptFilename())) {
      fprintf(stderr, "%s is missing, regenerating...\n",
              GetNinjaShellScriptFilename().c_str());
      return true;
    }
    return false;
  }

  bool CheckStep1(const string& orig_args) {
#define LOAD_INT(fp)                                               \
  ({                                                               \
    int v = LoadInt(fp);                                           \
    if (v < 0) {                                                   \
      fprintf(stderr, "incomplete kati_stamp, regenerating...\n"); \
      RETURN_TRUE;                                                 \
    }                                                              \
    v;                                                             \
  })

#define LOAD_STRING(fp, s)                                         \
  ({                                                               \
    if (!LoadString(fp, s)) {                                      \
      fprintf(stderr, "incomplete kati_stamp, regenerating...\n"); \
      RETURN_TRUE;                                                 \
    }                                                              \
  })

    const string& stamp_filename = GetNinjaStampFilename();
    FILE* fp = fopen(stamp_filename.c_str(), "rb");
    if (!fp) {
      if (g_flags.regen_debug)
        printf("%s: %s\n", stamp_filename.c_str(), strerror(errno));
      return true;
    }
    ScopedFile sfp(fp);

    double gen_time;
    size_t r = fread(&gen_time, sizeof(gen_time), 1, fp);
    gen_time_ = gen_time;
    if (r != 1) {
      fprintf(stderr, "incomplete kati_stamp, regenerating...\n");
      RETURN_TRUE;
    }
    if (g_flags.regen_debug)
      printf("Generated time: %f\n", gen_time);

    string s, s2;
    int num_files = LOAD_INT(fp);
    for (int i = 0; i < num_files; i++) {
      LOAD_STRING(fp, &s);
      double ts = GetTimestamp(s);
      if (gen_time < ts) {
        if (g_flags.regen_ignoring_kati_binary) {
          string kati_binary;
          GetExecutablePath(&kati_binary);
          if (s == kati_binary) {
            fprintf(stderr, "%s was modified, ignored.\n", s.c_str());
            continue;
          }
        }
        if (ShouldIgnoreDirty(s)) {
          if (g_flags.regen_debug)
            printf("file %s: ignored (%f)\n", s.c_str(), ts);
          continue;
        }
        if (g_flags.dump_kati_stamp)
          printf("file %s: dirty (%f)\n", s.c_str(), ts);
        else
          fprintf(stderr, "%s was modified, regenerating...\n", s.c_str());
        RETURN_TRUE;
      } else if (g_flags.dump_kati_stamp) {
        printf("file %s: clean (%f)\n", s.c_str(), ts);
      }
    }

    int num_undefineds = LOAD_INT(fp);
    for (int i = 0; i < num_undefineds; i++) {
      LOAD_STRING(fp, &s);
      if (getenv(s.c_str())) {
        if (g_flags.dump_kati_stamp) {
          printf("env %s: dirty (unset => %s)\n", s.c_str(), getenv(s.c_str()));
        } else {
          fprintf(stderr, "Environment variable %s was set, regenerating...\n",
                  s.c_str());
        }
        RETURN_TRUE;
      } else if (g_flags.dump_kati_stamp) {
        printf("env %s: clean (unset)\n", s.c_str());
      }
    }

    int num_envs = LOAD_INT(fp);
    for (int i = 0; i < num_envs; i++) {
      LOAD_STRING(fp, &s);
      StringPiece val(getenv(s.c_str()));
      LOAD_STRING(fp, &s2);
      if (val != s2) {
        if (g_flags.dump_kati_stamp) {
          printf("env %s: dirty (%s => %.*s)\n", s.c_str(), s2.c_str(),
                 SPF(val));
        } else {
          fprintf(stderr,
                  "Environment variable %s was modified (%s => %.*s), "
                  "regenerating...\n",
                  s.c_str(), s2.c_str(), SPF(val));
        }
        RETURN_TRUE;
      } else if (g_flags.dump_kati_stamp) {
        printf("env %s: clean (%.*s)\n", s.c_str(), SPF(val));
      }
    }

    int num_globs = LOAD_INT(fp);
    string pat;
    for (int i = 0; i < num_globs; i++) {
      GlobResult* gr = new GlobResult;
      globs_.push_back(gr);

      LOAD_STRING(fp, &gr->pat);
      int num_files = LOAD_INT(fp);
      gr->result.resize(num_files);
      for (int j = 0; j < num_files; j++) {
        LOAD_STRING(fp, &gr->result[j]);
      }
    }

    int num_crs = LOAD_INT(fp);
    for (int i = 0; i < num_crs; i++) {
      ShellResult* sr = new ShellResult;
      commands_.push_back(sr);
      sr->op = static_cast<CommandOp>(LOAD_INT(fp));
      LOAD_STRING(fp, &sr->shell);
      LOAD_STRING(fp, &sr->shellflag);
      LOAD_STRING(fp, &sr->cmd);
      LOAD_STRING(fp, &sr->result);

      if (sr->op == CommandOp::FIND) {
        int num_missing_dirs = LOAD_INT(fp);
        for (int j = 0; j < num_missing_dirs; j++) {
          LOAD_STRING(fp, &s);
          sr->missing_dirs.push_back(s);
        }
        int num_files = LOAD_INT(fp);
        for (int j = 0; j < num_files; j++) {
          LOAD_STRING(fp, &s);
          sr->files.push_back(s);
        }
        int num_read_dirs = LOAD_INT(fp);
        for (int j = 0; j < num_read_dirs; j++) {
          LOAD_STRING(fp, &s);
          sr->read_dirs.push_back(s);
        }
      }
    }

    LoadString(fp, &s);
    if (orig_args != s) {
      fprintf(stderr, "arguments changed, regenerating...\n");
      RETURN_TRUE;
    }

    return needs_regen_;
  }

  bool CheckGlobResult(const GlobResult* gr, string* err) {
    COLLECT_STATS("glob time (regen)");
    vector<string>* files;
    Glob(gr->pat.c_str(), &files);
    bool needs_regen = files->size() != gr->result.size();
    for (size_t i = 0; i < gr->result.size(); i++) {
      if (!needs_regen) {
        if ((*files)[i] != gr->result[i]) {
          needs_regen = true;
          break;
        }
      }
    }
    if (needs_regen) {
      if (ShouldIgnoreDirty(gr->pat)) {
        if (g_flags.dump_kati_stamp) {
          printf("wildcard %s: ignored\n", gr->pat.c_str());
        }
        return false;
      }
      if (g_flags.dump_kati_stamp) {
        printf("wildcard %s: dirty\n", gr->pat.c_str());
      } else {
        *err = StringPrintf("wildcard(%s) was changed, regenerating...\n",
                            gr->pat.c_str());
      }
    } else if (g_flags.dump_kati_stamp) {
      printf("wildcard %s: clean\n", gr->pat.c_str());
    }
    return needs_regen;
  }

  bool ShouldRunCommand(const ShellResult* sr) {
    if (sr->op != CommandOp::FIND)
      return true;

    COLLECT_STATS("stat time (regen)");
    for (const string& dir : sr->missing_dirs) {
      if (Exists(dir))
        return true;
    }
    for (const string& file : sr->files) {
      if (!Exists(file))
        return true;
    }
    for (const string& dir : sr->read_dirs) {
      // We assume we rarely do a significant change for the top
      // directory which affects the results of find command.
      if (dir == "" || dir == "." || ShouldIgnoreDirty(dir))
        continue;

      struct stat st;
      if (lstat(dir.c_str(), &st) != 0) {
        return true;
      }
      double ts = GetTimestampFromStat(st);
      if (gen_time_ < ts) {
        return true;
      }
      if (S_ISLNK(st.st_mode)) {
        ts = GetTimestamp(dir);
        if (ts < 0 || gen_time_ < ts)
          return true;
      }
    }
    return false;
  }

  bool CheckShellResult(const ShellResult* sr, string* err) {
    if (sr->op == CommandOp::READ_MISSING) {
      if (Exists(sr->cmd)) {
        if (g_flags.dump_kati_stamp)
          printf("file %s: dirty\n", sr->cmd.c_str());
        else
          *err = StringPrintf("$(file <%s) was changed, regenerating...\n",
                              sr->cmd.c_str());
        return true;
      }
      if (g_flags.dump_kati_stamp)
        printf("file %s: clean\n", sr->cmd.c_str());
      return false;
    }

    if (sr->op == CommandOp::READ) {
      double ts = GetTimestamp(sr->cmd);
      if (gen_time_ < ts) {
        if (g_flags.dump_kati_stamp)
          printf("file %s: dirty\n", sr->cmd.c_str());
        else
          *err = StringPrintf("$(file <%s) was changed, regenerating...\n",
                              sr->cmd.c_str());
        return true;
      }
      if (g_flags.dump_kati_stamp)
        printf("file %s: clean\n", sr->cmd.c_str());
      return false;
    }

    if (sr->op == CommandOp::WRITE || sr->op == CommandOp::APPEND) {
      FILE* f =
          fopen(sr->cmd.c_str(), (sr->op == CommandOp::WRITE) ? "wb" : "ab");
      if (f == NULL) {
        PERROR("fopen");
      }

      if (fwrite(&sr->result[0], sr->result.size(), 1, f) != 1) {
        PERROR("fwrite");
      }

      if (fclose(f) != 0) {
        PERROR("fclose");
      }

      if (g_flags.dump_kati_stamp)
        printf("file %s: clean (write)\n", sr->cmd.c_str());
      return false;
    }

    if (!ShouldRunCommand(sr)) {
      if (g_flags.regen_debug)
        printf("shell %s: clean (no rerun)\n", sr->cmd.c_str());
      return false;
    }

    FindCommand fc;
    if (fc.Parse(sr->cmd) && !fc.chdir.empty() && ShouldIgnoreDirty(fc.chdir)) {
      if (g_flags.dump_kati_stamp)
        printf("shell %s: ignored\n", sr->cmd.c_str());
      return false;
    }

    COLLECT_STATS_WITH_SLOW_REPORT("shell time (regen)", sr->cmd.c_str());
    string result;
    RunCommand(sr->shell, sr->shellflag, sr->cmd, RedirectStderr::DEV_NULL,
               &result);
    FormatForCommandSubstitution(&result);
    if (sr->result != result) {
      if (g_flags.dump_kati_stamp) {
        printf("shell %s: dirty\n", sr->cmd.c_str());
      } else {
        *err = StringPrintf("$(shell %s) was changed, regenerating...\n",
                            sr->cmd.c_str());
        //*err += StringPrintf("%s => %s\n", expected.c_str(), result.c_str());
      }
      return true;
    } else if (g_flags.regen_debug) {
      printf("shell %s: clean (rerun)\n", sr->cmd.c_str());
    }
    return false;
  }

  bool CheckStep2() {
    unique_ptr<ThreadPool> tp(NewThreadPool(g_flags.num_jobs));

    tp->Submit([this]() {
      string err;
      // TODO: Make glob cache thread safe and create a task for each glob.
      SetAffinityForSingleThread();
      for (GlobResult* gr : globs_) {
        if (CheckGlobResult(gr, &err)) {
          unique_lock<mutex> lock(mu_);
          if (!needs_regen_) {
            needs_regen_ = true;
            msg_ = err;
          }
          break;
        }
      }
    });

    tp->Submit([this]() {
      SetAffinityForSingleThread();
      for (ShellResult* sr : commands_) {
        string err;
        if (CheckShellResult(sr, &err)) {
          unique_lock<mutex> lock(mu_);
          if (!needs_regen_) {
            needs_regen_ = true;
            msg_ = err;
          }
        }
      }
    });

    tp->Wait();
    if (needs_regen_) {
      fprintf(stderr, "%s", msg_.c_str());
    }
    return needs_regen_;
  }

 private:
  double gen_time_;
  vector<GlobResult*> globs_;
  vector<ShellResult*> commands_;
  mutex mu_;
  bool needs_regen_;
  string msg_;
};

}  // namespace

bool NeedsRegen(double start_time, const string& orig_args) {
  return StampChecker().NeedsRegen(start_time, orig_args);
}