Shinichiro Hamaji | b69bf8a | 2015-06-10 14:52:06 +0900 | [diff] [blame] | 1 | // Copyright 2015 Google Inc. All rights reserved |
| 2 | // |
| 3 | // Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | // you may not use this file except in compliance with the License. |
| 5 | // You may obtain a copy of the License at |
| 6 | // |
| 7 | // http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | // |
| 9 | // Unless required by applicable law or agreed to in writing, software |
| 10 | // distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | // See the License for the specific language governing permissions and |
| 13 | // limitations under the License. |
| 14 | |
Fumitoshi Ukai | 744bb2b | 2015-06-25 00:10:52 +0900 | [diff] [blame] | 15 | package kati |
Fumitoshi Ukai | 119dc91 | 2015-03-30 16:52:41 +0900 | [diff] [blame] | 16 | |
| 17 | import ( |
| 18 | "fmt" |
Fumitoshi Ukai | ba408a2 | 2015-07-09 12:41:32 +0900 | [diff] [blame] | 19 | "os" |
Fumitoshi Ukai | 49599e5 | 2015-06-26 10:10:24 +0900 | [diff] [blame] | 20 | "time" |
Fumitoshi Ukai | 6450d0f | 2015-07-10 16:34:06 +0900 | [diff] [blame] | 21 | |
| 22 | "github.com/golang/glog" |
Fumitoshi Ukai | 119dc91 | 2015-03-30 16:52:41 +0900 | [diff] [blame] | 23 | ) |
| 24 | |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 25 | // Executor manages execution of makefile rules. |
Fumitoshi Ukai | 119dc91 | 2015-03-30 16:52:41 +0900 | [diff] [blame] | 26 | type Executor struct { |
Fumitoshi Ukai | adc1444 | 2015-06-25 16:10:30 +0900 | [diff] [blame] | 27 | rules map[string]*rule |
| 28 | implicitRules []*rule |
| 29 | suffixRules map[string][]*rule |
| 30 | firstRule *rule |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 31 | // target -> Job, nil means the target is currently being processed. |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 32 | done map[string]*job |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 33 | |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 34 | wm *workerManager |
Shinichiro Hamaji | 110af33 | 2015-04-13 19:04:50 +0900 | [diff] [blame] | 35 | |
Fumitoshi Ukai | a70b4ea | 2015-06-30 15:31:49 +0900 | [diff] [blame] | 36 | ctx *execContext |
Shinichiro Hamaji | b0f6548 | 2015-04-16 18:28:05 +0900 | [diff] [blame] | 37 | |
Shinichiro Hamaji | 17a8a6e | 2015-04-20 21:44:42 +0900 | [diff] [blame] | 38 | trace []string |
| 39 | buildCnt int |
| 40 | alreadyDoneCnt int |
| 41 | noRuleCnt int |
| 42 | upToDateCnt int |
| 43 | runCommandCnt int |
Shinichiro Hamaji | 110af33 | 2015-04-13 19:04:50 +0900 | [diff] [blame] | 44 | } |
| 45 | |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 46 | func (ex *Executor) makeJobs(n *DepNode, neededBy *job) error { |
Fumitoshi Ukai | 09fcd52 | 2015-07-15 14:31:50 +0900 | [diff] [blame] | 47 | output, _ := ex.ctx.vpaths.exists(n.Output) |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 48 | if neededBy != nil { |
Fumitoshi Ukai | 6450d0f | 2015-07-10 16:34:06 +0900 | [diff] [blame] | 49 | glog.V(1).Infof("MakeJob: %s for %s", output, neededBy.n.Output) |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 50 | } |
Fumitoshi Ukai | dd058ff | 2015-07-03 14:29:41 +0900 | [diff] [blame] | 51 | n.Output = output |
Shinichiro Hamaji | 294a58b | 2015-04-14 15:31:52 +0900 | [diff] [blame] | 52 | ex.buildCnt++ |
| 53 | if ex.buildCnt%100 == 0 { |
| 54 | ex.reportStats() |
| 55 | } |
| 56 | |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 57 | j, present := ex.done[output] |
Fumitoshi Ukai | 119dc91 | 2015-03-30 16:52:41 +0900 | [diff] [blame] | 58 | |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 59 | if present { |
| 60 | if j == nil { |
| 61 | if !n.IsPhony { |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 62 | fmt.Printf("Circular %s <- %s dependency dropped.\n", neededBy.n.Output, n.Output) |
| 63 | } |
| 64 | if neededBy != nil { |
| 65 | neededBy.numDeps-- |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 66 | } |
Shinichiro Hamaji | eb4da7f | 2015-04-11 20:28:54 +0900 | [diff] [blame] | 67 | } else { |
Fumitoshi Ukai | 6450d0f | 2015-07-10 16:34:06 +0900 | [diff] [blame] | 68 | glog.Infof("%s already done: %d", j.n.Output, j.outputTs) |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 69 | if neededBy != nil { |
Shinichiro Hamaji | 55c50bd | 2015-04-27 21:05:47 +0900 | [diff] [blame] | 70 | ex.wm.ReportNewDep(j, neededBy) |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 71 | } |
Shinichiro Hamaji | eb4da7f | 2015-04-11 20:28:54 +0900 | [diff] [blame] | 72 | } |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 73 | return nil |
Fumitoshi Ukai | 119dc91 | 2015-03-30 16:52:41 +0900 | [diff] [blame] | 74 | } |
Shinichiro Hamaji | 4e9ab1a | 2015-04-12 01:31:58 +0900 | [diff] [blame] | 75 | |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 76 | j = &job{ |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 77 | n: n, |
| 78 | ex: ex, |
Fumitoshi Ukai | c916ea2 | 2015-06-30 09:52:13 +0900 | [diff] [blame] | 79 | numDeps: len(n.Deps) + len(n.OrderOnlys), |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 80 | depsTs: int64(-1), |
| 81 | } |
| 82 | if neededBy != nil { |
| 83 | j.parents = append(j.parents, neededBy) |
| 84 | } |
| 85 | |
| 86 | ex.done[output] = nil |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 87 | // We iterate n.Deps twice. In the first run, we may modify |
| 88 | // numDeps. There will be a race if we do so after the first |
| 89 | // ex.makeJobs(d, j). |
| 90 | var deps []*DepNode |
Shinichiro Hamaji | 17a8a6e | 2015-04-20 21:44:42 +0900 | [diff] [blame] | 91 | for _, d := range n.Deps { |
Fumitoshi Ukai | c916ea2 | 2015-06-30 09:52:13 +0900 | [diff] [blame] | 92 | deps = append(deps, d) |
| 93 | } |
| 94 | for _, d := range n.OrderOnlys { |
Fumitoshi Ukai | 09fcd52 | 2015-07-15 14:31:50 +0900 | [diff] [blame] | 95 | if _, ok := ex.ctx.vpaths.exists(d.Output); ok { |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 96 | j.numDeps-- |
Shinichiro Hamaji | 5c53b57 | 2015-04-02 05:36:42 +0900 | [diff] [blame] | 97 | continue |
| 98 | } |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 99 | deps = append(deps, d) |
| 100 | } |
Fumitoshi Ukai | 6450d0f | 2015-07-10 16:34:06 +0900 | [diff] [blame] | 101 | glog.V(1).Infof("new: %s (%d)", j.n.Output, j.numDeps) |
Shinichiro Hamaji | 294a58b | 2015-04-14 15:31:52 +0900 | [diff] [blame] | 102 | |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 103 | for _, d := range deps { |
Shinichiro Hamaji | 17a8a6e | 2015-04-20 21:44:42 +0900 | [diff] [blame] | 104 | ex.trace = append(ex.trace, d.Output) |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 105 | err := ex.makeJobs(d, j) |
Shinichiro Hamaji | 294a58b | 2015-04-14 15:31:52 +0900 | [diff] [blame] | 106 | ex.trace = ex.trace[0 : len(ex.trace)-1] |
Shinichiro Hamaji | 5c53b57 | 2015-04-02 05:36:42 +0900 | [diff] [blame] | 107 | if err != nil { |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 108 | return err |
Shinichiro Hamaji | 5c53b57 | 2015-04-02 05:36:42 +0900 | [diff] [blame] | 109 | } |
| 110 | } |
| 111 | |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 112 | ex.done[output] = j |
Fumitoshi Ukai | 531c5d2 | 2015-06-27 00:58:56 +0900 | [diff] [blame] | 113 | return ex.wm.PostJob(j) |
Fumitoshi Ukai | 119dc91 | 2015-03-30 16:52:41 +0900 | [diff] [blame] | 114 | } |
| 115 | |
Shinichiro Hamaji | 294a58b | 2015-04-14 15:31:52 +0900 | [diff] [blame] | 116 | func (ex *Executor) reportStats() { |
Fumitoshi Ukai | 6450d0f | 2015-07-10 16:34:06 +0900 | [diff] [blame] | 117 | if !PeriodicStatsFlag { |
Shinichiro Hamaji | 294a58b | 2015-04-14 15:31:52 +0900 | [diff] [blame] | 118 | return |
| 119 | } |
| 120 | |
Fumitoshi Ukai | 49599e5 | 2015-06-26 10:10:24 +0900 | [diff] [blame] | 121 | logStats("build=%d alreadyDone=%d noRule=%d, upToDate=%d runCommand=%d", |
Shinichiro Hamaji | 294a58b | 2015-04-14 15:31:52 +0900 | [diff] [blame] | 122 | ex.buildCnt, ex.alreadyDoneCnt, ex.noRuleCnt, ex.upToDateCnt, ex.runCommandCnt) |
Shinichiro Hamaji | 294a58b | 2015-04-14 15:31:52 +0900 | [diff] [blame] | 123 | if len(ex.trace) > 1 { |
Fumitoshi Ukai | 49599e5 | 2015-06-26 10:10:24 +0900 | [diff] [blame] | 124 | logStats("trace=%q", ex.trace) |
Shinichiro Hamaji | 294a58b | 2015-04-14 15:31:52 +0900 | [diff] [blame] | 125 | } |
| 126 | } |
| 127 | |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 128 | // ExecutorOpt is an option for Executor. |
Fumitoshi Ukai | 744bb2b | 2015-06-25 00:10:52 +0900 | [diff] [blame] | 129 | type ExecutorOpt struct { |
Shinichiro Hamaji | 18287f0 | 2015-07-06 14:48:48 +0900 | [diff] [blame] | 130 | NumJobs int |
Fumitoshi Ukai | 744bb2b | 2015-06-25 00:10:52 +0900 | [diff] [blame] | 131 | } |
| 132 | |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 133 | // NewExecutor creates new Executor. |
Fumitoshi Ukai | ba408a2 | 2015-07-09 12:41:32 +0900 | [diff] [blame] | 134 | func NewExecutor(opt *ExecutorOpt) (*Executor, error) { |
Fumitoshi Ukai | 744bb2b | 2015-06-25 00:10:52 +0900 | [diff] [blame] | 135 | if opt == nil { |
| 136 | opt = &ExecutorOpt{NumJobs: 1} |
| 137 | } |
| 138 | if opt.NumJobs < 1 { |
| 139 | opt.NumJobs = 1 |
| 140 | } |
Shinichiro Hamaji | 18287f0 | 2015-07-06 14:48:48 +0900 | [diff] [blame] | 141 | wm, err := newWorkerManager(opt.NumJobs) |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 142 | if err != nil { |
| 143 | return nil, err |
| 144 | } |
Shinichiro Hamaji | 17a8a6e | 2015-04-20 21:44:42 +0900 | [diff] [blame] | 145 | ex := &Executor{ |
Fumitoshi Ukai | adc1444 | 2015-06-25 16:10:30 +0900 | [diff] [blame] | 146 | rules: make(map[string]*rule), |
| 147 | suffixRules: make(map[string][]*rule), |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 148 | done: make(map[string]*job), |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 149 | wm: wm, |
Shinichiro Hamaji | 17a8a6e | 2015-04-20 21:44:42 +0900 | [diff] [blame] | 150 | } |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 151 | return ex, nil |
Shinichiro Hamaji | 8843a05 | 2015-04-13 16:46:56 +0900 | [diff] [blame] | 152 | } |
Shinichiro Hamaji | 9b7d462 | 2015-04-02 01:37:04 +0900 | [diff] [blame] | 153 | |
Fumitoshi Ukai | e457469 | 2015-07-16 16:55:51 +0900 | [diff] [blame] | 154 | // Exec executes to build targets, or first target in DepGraph. |
| 155 | func (ex *Executor) Exec(g *DepGraph, targets []string) error { |
Fumitoshi Ukai | 09fcd52 | 2015-07-15 14:31:50 +0900 | [diff] [blame] | 156 | ex.ctx = newExecContext(g.vars, g.vpaths, false) |
Fumitoshi Ukai | ba408a2 | 2015-07-09 12:41:32 +0900 | [diff] [blame] | 157 | |
| 158 | // TODO: Handle target specific variables. |
| 159 | for name, export := range g.exports { |
| 160 | if export { |
| 161 | v, err := ex.ctx.ev.EvaluateVar(name) |
| 162 | if err != nil { |
| 163 | return err |
| 164 | } |
| 165 | os.Setenv(name, v) |
| 166 | } else { |
| 167 | os.Unsetenv(name) |
| 168 | } |
| 169 | } |
| 170 | |
Fumitoshi Ukai | 49599e5 | 2015-06-26 10:10:24 +0900 | [diff] [blame] | 171 | startTime := time.Now() |
Fumitoshi Ukai | e457469 | 2015-07-16 16:55:51 +0900 | [diff] [blame] | 172 | var nodes []*DepNode |
| 173 | if len(targets) == 0 { |
| 174 | if len(g.nodes) > 0 { |
| 175 | nodes = append(nodes, g.nodes[0]) |
| 176 | } |
| 177 | } else { |
| 178 | m := make(map[string]*DepNode) |
| 179 | for _, n := range g.nodes { |
| 180 | m[n.Output] = n |
| 181 | } |
| 182 | for _, t := range targets { |
| 183 | n := m[t] |
| 184 | if n != nil { |
| 185 | nodes = append(nodes, n) |
| 186 | } |
| 187 | } |
| 188 | } |
| 189 | for _, root := range nodes { |
Fumitoshi Ukai | 531c5d2 | 2015-06-27 00:58:56 +0900 | [diff] [blame] | 190 | err := ex.makeJobs(root, nil) |
| 191 | if err != nil { |
| 192 | break |
| 193 | } |
Shinichiro Hamaji | b13f3d5 | 2015-03-30 19:29:44 +0900 | [diff] [blame] | 194 | } |
Fumitoshi Ukai | a43b96f | 2015-07-17 19:54:21 +0900 | [diff] [blame] | 195 | n, err := ex.wm.Wait() |
Fumitoshi Ukai | 49599e5 | 2015-06-26 10:10:24 +0900 | [diff] [blame] | 196 | logStats("exec time: %q", time.Since(startTime)) |
Fumitoshi Ukai | a43b96f | 2015-07-17 19:54:21 +0900 | [diff] [blame] | 197 | if n == 0 { |
| 198 | for _, root := range nodes { |
| 199 | fmt.Printf("kati: Nothing to be done for `%s'.\n", root.Output) |
| 200 | } |
| 201 | } |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 202 | return err |
Fumitoshi Ukai | 119dc91 | 2015-03-30 16:52:41 +0900 | [diff] [blame] | 203 | } |