Shinichiro Hamaji | b69bf8a | 2015-06-10 14:52:06 +0900 | [diff] [blame] | 1 | // Copyright 2015 Google Inc. All rights reserved |
| 2 | // |
| 3 | // Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | // you may not use this file except in compliance with the License. |
| 5 | // You may obtain a copy of the License at |
| 6 | // |
| 7 | // http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | // |
| 9 | // Unless required by applicable law or agreed to in writing, software |
| 10 | // distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | // See the License for the specific language governing permissions and |
| 13 | // limitations under the License. |
| 14 | |
Fumitoshi Ukai | 744bb2b | 2015-06-25 00:10:52 +0900 | [diff] [blame] | 15 | package kati |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 16 | |
| 17 | import ( |
Shinichiro Hamaji | dbc6c13 | 2015-04-28 18:26:36 +0900 | [diff] [blame] | 18 | "container/heap" |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 19 | "fmt" |
Shinichiro Hamaji | cedc5c8 | 2015-05-13 17:03:20 +0900 | [diff] [blame] | 20 | "os" |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 21 | "os/exec" |
| 22 | "strings" |
| 23 | "syscall" |
| 24 | "time" |
| 25 | ) |
| 26 | |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 27 | type job struct { |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 28 | n *DepNode |
| 29 | ex *Executor |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 30 | parents []*job |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 31 | outputTs int64 |
| 32 | numDeps int |
| 33 | depsTs int64 |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 34 | id int |
Shinichiro Hamaji | a680842 | 2015-05-13 18:00:50 +0900 | [diff] [blame] | 35 | |
| 36 | runners []runner |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 37 | } |
| 38 | |
| 39 | type runner struct { |
| 40 | output string |
| 41 | cmd string |
| 42 | echo bool |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 43 | ignoreError bool |
| 44 | shell string |
| 45 | } |
| 46 | |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 47 | type jobResult struct { |
| 48 | j *job |
| 49 | w *worker |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 50 | } |
| 51 | |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 52 | type newDep struct { |
| 53 | j *job |
| 54 | neededBy *job |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 55 | } |
| 56 | |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 57 | type worker struct { |
| 58 | wm *workerManager |
| 59 | jobChan chan *job |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 60 | waitChan chan bool |
| 61 | doneChan chan bool |
| 62 | } |
| 63 | |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 64 | type jobQueue []*job |
Shinichiro Hamaji | dbc6c13 | 2015-04-28 18:26:36 +0900 | [diff] [blame] | 65 | |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 66 | func (jq jobQueue) Len() int { return len(jq) } |
| 67 | func (jq jobQueue) Swap(i, j int) { jq[i], jq[j] = jq[j], jq[i] } |
Shinichiro Hamaji | dbc6c13 | 2015-04-28 18:26:36 +0900 | [diff] [blame] | 68 | |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 69 | func (jq jobQueue) Less(i, j int) bool { |
Shinichiro Hamaji | dbc6c13 | 2015-04-28 18:26:36 +0900 | [diff] [blame] | 70 | // First come, first serve, for GNU make compatibility. |
| 71 | return jq[i].id < jq[j].id |
| 72 | } |
| 73 | |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 74 | func (jq *jobQueue) Push(x interface{}) { |
| 75 | item := x.(*job) |
Shinichiro Hamaji | dbc6c13 | 2015-04-28 18:26:36 +0900 | [diff] [blame] | 76 | *jq = append(*jq, item) |
| 77 | } |
| 78 | |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 79 | func (jq *jobQueue) Pop() interface{} { |
Shinichiro Hamaji | dbc6c13 | 2015-04-28 18:26:36 +0900 | [diff] [blame] | 80 | old := *jq |
| 81 | n := len(old) |
| 82 | item := old[n-1] |
| 83 | *jq = old[0 : n-1] |
| 84 | return item |
| 85 | } |
| 86 | |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 87 | func newWorker(wm *workerManager) *worker { |
| 88 | w := &worker{ |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 89 | wm: wm, |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 90 | jobChan: make(chan *job), |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 91 | waitChan: make(chan bool), |
| 92 | doneChan: make(chan bool), |
| 93 | } |
| 94 | return w |
| 95 | } |
| 96 | |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 97 | func (w *worker) Run() { |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 98 | done := false |
| 99 | for !done { |
| 100 | select { |
| 101 | case j := <-w.jobChan: |
| 102 | j.build() |
| 103 | w.wm.ReportResult(w, j) |
| 104 | case done = <-w.waitChan: |
| 105 | } |
| 106 | } |
| 107 | w.doneChan <- true |
| 108 | } |
| 109 | |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 110 | func (w *worker) PostJob(j *job) { |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 111 | w.jobChan <- j |
| 112 | } |
| 113 | |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 114 | func (w *worker) Wait() { |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 115 | w.waitChan <- true |
| 116 | <-w.doneChan |
| 117 | } |
| 118 | |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 119 | func evalCmd(ev *Evaluator, r runner, s string) []runner { |
| 120 | r = newRunner(r, s) |
| 121 | if strings.IndexByte(r.cmd, '$') < 0 { |
| 122 | // fast path |
| 123 | return []runner{r} |
| 124 | } |
| 125 | // TODO(ukai): parse once more earlier? |
Fumitoshi Ukai | 7c9aa9f | 2015-06-12 23:51:38 +0900 | [diff] [blame] | 126 | expr, _, err := parseExpr([]byte(r.cmd), nil, false) |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 127 | if err != nil { |
| 128 | panic(fmt.Errorf("parse cmd %q: %v", r.cmd, err)) |
| 129 | } |
Fumitoshi Ukai | b06cd9d | 2015-05-07 12:56:12 +0900 | [diff] [blame] | 130 | buf := newBuf() |
| 131 | expr.Eval(buf, ev) |
| 132 | cmds := buf.String() |
| 133 | freeBuf(buf) |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 134 | var runners []runner |
| 135 | for _, cmd := range strings.Split(cmds, "\n") { |
Shinichiro Hamaji | 212abfb | 2015-04-29 03:02:59 +0900 | [diff] [blame] | 136 | if len(runners) > 0 && strings.HasSuffix(runners[len(runners)-1].cmd, "\\") { |
| 137 | runners[len(runners)-1].cmd += "\n" |
| 138 | runners[len(runners)-1].cmd += cmd |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 139 | } else { |
| 140 | runners = append(runners, newRunner(r, cmd)) |
| 141 | } |
| 142 | } |
| 143 | return runners |
| 144 | } |
| 145 | |
| 146 | func newRunner(r runner, s string) runner { |
| 147 | for { |
| 148 | s = trimLeftSpace(s) |
| 149 | if s == "" { |
| 150 | return runner{} |
| 151 | } |
| 152 | switch s[0] { |
| 153 | case '@': |
Fumitoshi Ukai | 744bb2b | 2015-06-25 00:10:52 +0900 | [diff] [blame] | 154 | if !DryRunFlag { |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 155 | r.echo = false |
| 156 | } |
| 157 | s = s[1:] |
| 158 | continue |
| 159 | case '-': |
| 160 | r.ignoreError = true |
| 161 | s = s[1:] |
| 162 | continue |
| 163 | } |
| 164 | break |
| 165 | } |
| 166 | r.cmd = s |
| 167 | return r |
| 168 | } |
| 169 | |
| 170 | func (r runner) run(output string) error { |
Fumitoshi Ukai | 744bb2b | 2015-06-25 00:10:52 +0900 | [diff] [blame] | 171 | if r.echo || DryRunFlag { |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 172 | fmt.Printf("%s\n", r.cmd) |
| 173 | } |
Fumitoshi Ukai | 744bb2b | 2015-06-25 00:10:52 +0900 | [diff] [blame] | 174 | if DryRunFlag { |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 175 | return nil |
| 176 | } |
| 177 | args := []string{r.shell, "-c", r.cmd} |
| 178 | cmd := exec.Cmd{ |
| 179 | Path: args[0], |
| 180 | Args: args, |
| 181 | } |
| 182 | out, err := cmd.CombinedOutput() |
| 183 | fmt.Printf("%s", out) |
| 184 | exit := exitStatus(err) |
| 185 | if r.ignoreError && exit != 0 { |
| 186 | fmt.Printf("[%s] Error %d (ignored)\n", output, exit) |
| 187 | err = nil |
| 188 | } |
| 189 | return err |
| 190 | } |
| 191 | |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 192 | func (j *job) createRunners() []runner { |
Shinichiro Hamaji | b41fd50 | 2015-04-29 03:34:07 +0900 | [diff] [blame] | 193 | runners, _ := j.ex.createRunners(j.n, false) |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 194 | return runners |
| 195 | } |
| 196 | |
Shinichiro Hamaji | 71fae4c | 2015-05-25 17:48:34 +0900 | [diff] [blame] | 197 | // TODO(ukai): use time.Time? |
| 198 | func getTimestamp(filename string) int64 { |
| 199 | st, err := os.Stat(filename) |
| 200 | if err != nil { |
| 201 | return -2 |
| 202 | } |
| 203 | return st.ModTime().Unix() |
| 204 | } |
| 205 | |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 206 | func (j *job) build() { |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 207 | if j.n.IsPhony { |
| 208 | j.outputTs = -2 // trigger cmd even if all inputs don't exist. |
| 209 | } else { |
| 210 | j.outputTs = getTimestamp(j.n.Output) |
| 211 | } |
| 212 | |
| 213 | if !j.n.HasRule { |
| 214 | if j.outputTs >= 0 || j.n.IsPhony { |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 215 | return |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 216 | } |
| 217 | if len(j.parents) == 0 { |
Fumitoshi Ukai | 07cf121 | 2015-06-25 17:16:25 +0900 | [diff] [blame^] | 218 | errorNoLocationExit("*** No rule to make target %q.", j.n.Output) |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 219 | } else { |
Fumitoshi Ukai | 07cf121 | 2015-06-25 17:16:25 +0900 | [diff] [blame^] | 220 | errorNoLocationExit("*** No rule to make target %q, needed by %q.", j.n.Output, j.parents[0].n.Output) |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 221 | } |
Fumitoshi Ukai | 07cf121 | 2015-06-25 17:16:25 +0900 | [diff] [blame^] | 222 | errorNoLocationExit("no rule to make target %q", j.n.Output) |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 223 | } |
| 224 | |
| 225 | if j.outputTs >= j.depsTs { |
| 226 | // TODO: stats. |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 227 | return |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 228 | } |
| 229 | |
| 230 | for _, r := range j.createRunners() { |
| 231 | err := r.run(j.n.Output) |
| 232 | if err != nil { |
| 233 | exit := exitStatus(err) |
Fumitoshi Ukai | 07cf121 | 2015-06-25 17:16:25 +0900 | [diff] [blame^] | 234 | errorNoLocationExit("[%s] Error %d: %v", j.n.Output, exit, err) |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 235 | } |
| 236 | } |
| 237 | |
| 238 | if j.n.IsPhony { |
| 239 | j.outputTs = time.Now().Unix() |
| 240 | } else { |
| 241 | j.outputTs = getTimestamp(j.n.Output) |
| 242 | if j.outputTs < 0 { |
| 243 | j.outputTs = time.Now().Unix() |
| 244 | } |
| 245 | } |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 246 | } |
| 247 | |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 248 | func (wm *workerManager) handleJobs() { |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 249 | for { |
Fumitoshi Ukai | 744bb2b | 2015-06-25 00:10:52 +0900 | [diff] [blame] | 250 | if wm.para == nil && len(wm.freeWorkers) == 0 { |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 251 | return |
| 252 | } |
Shinichiro Hamaji | dbc6c13 | 2015-04-28 18:26:36 +0900 | [diff] [blame] | 253 | if wm.readyQueue.Len() == 0 { |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 254 | return |
| 255 | } |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 256 | j := heap.Pop(&wm.readyQueue).(*job) |
Fumitoshi Ukai | 07cf121 | 2015-06-25 17:16:25 +0900 | [diff] [blame^] | 257 | logf("run: %s", j.n.Output) |
Shinichiro Hamaji | cedc5c8 | 2015-05-13 17:03:20 +0900 | [diff] [blame] | 258 | |
Fumitoshi Ukai | 744bb2b | 2015-06-25 00:10:52 +0900 | [diff] [blame] | 259 | if wm.para != nil { |
Shinichiro Hamaji | a680842 | 2015-05-13 18:00:50 +0900 | [diff] [blame] | 260 | j.runners = j.createRunners() |
| 261 | if len(j.runners) == 0 { |
| 262 | wm.updateParents(j) |
| 263 | wm.finishCnt++ |
| 264 | } else { |
| 265 | wm.runnings[j.n.Output] = j |
| 266 | wm.para.RunCommand(j.runners) |
| 267 | } |
Shinichiro Hamaji | cedc5c8 | 2015-05-13 17:03:20 +0900 | [diff] [blame] | 268 | } else { |
| 269 | j.numDeps = -1 // Do not let other workers pick this. |
| 270 | w := wm.freeWorkers[0] |
| 271 | wm.freeWorkers = wm.freeWorkers[1:] |
| 272 | wm.busyWorkers[w] = true |
| 273 | w.jobChan <- j |
| 274 | } |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 275 | } |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 276 | } |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 277 | |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 278 | func (wm *workerManager) updateParents(j *job) { |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 279 | for _, p := range j.parents { |
| 280 | p.numDeps-- |
Fumitoshi Ukai | 07cf121 | 2015-06-25 17:16:25 +0900 | [diff] [blame^] | 281 | logf("child: %s (%d)", p.n.Output, p.numDeps) |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 282 | if p.depsTs < j.outputTs { |
| 283 | p.depsTs = j.outputTs |
| 284 | } |
Shinichiro Hamaji | dbc6c13 | 2015-04-28 18:26:36 +0900 | [diff] [blame] | 285 | wm.maybePushToReadyQueue(p) |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 286 | } |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 287 | } |
| 288 | |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 289 | type workerManager struct { |
Fumitoshi Ukai | 744bb2b | 2015-06-25 00:10:52 +0900 | [diff] [blame] | 290 | maxJobs int |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 291 | jobs []*job |
| 292 | readyQueue jobQueue |
| 293 | jobChan chan *job |
| 294 | resultChan chan jobResult |
| 295 | newDepChan chan newDep |
Shinichiro Hamaji | 55c50bd | 2015-04-27 21:05:47 +0900 | [diff] [blame] | 296 | waitChan chan bool |
| 297 | doneChan chan bool |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 298 | freeWorkers []*worker |
| 299 | busyWorkers map[*worker]bool |
Shinichiro Hamaji | cedc5c8 | 2015-05-13 17:03:20 +0900 | [diff] [blame] | 300 | ex *Executor |
Fumitoshi Ukai | ff4e580 | 2015-06-25 13:12:26 +0900 | [diff] [blame] | 301 | para *paraWorker |
| 302 | paraChan chan *paraResult |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 303 | runnings map[string]*job |
Shinichiro Hamaji | dbc6c13 | 2015-04-28 18:26:36 +0900 | [diff] [blame] | 304 | |
| 305 | finishCnt int |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 306 | } |
| 307 | |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 308 | func newWorkerManager(numJobs int, paraPath string) *workerManager { |
| 309 | wm := &workerManager{ |
Fumitoshi Ukai | 744bb2b | 2015-06-25 00:10:52 +0900 | [diff] [blame] | 310 | maxJobs: numJobs, |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 311 | jobChan: make(chan *job), |
| 312 | resultChan: make(chan jobResult), |
| 313 | newDepChan: make(chan newDep), |
Shinichiro Hamaji | 55c50bd | 2015-04-27 21:05:47 +0900 | [diff] [blame] | 314 | waitChan: make(chan bool), |
| 315 | doneChan: make(chan bool), |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 316 | busyWorkers: make(map[*worker]bool), |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 317 | } |
Shinichiro Hamaji | cedc5c8 | 2015-05-13 17:03:20 +0900 | [diff] [blame] | 318 | |
Fumitoshi Ukai | 744bb2b | 2015-06-25 00:10:52 +0900 | [diff] [blame] | 319 | if paraPath != "" { |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 320 | wm.runnings = make(map[string]*job) |
Fumitoshi Ukai | ff4e580 | 2015-06-25 13:12:26 +0900 | [diff] [blame] | 321 | wm.paraChan = make(chan *paraResult) |
Fumitoshi Ukai | 744bb2b | 2015-06-25 00:10:52 +0900 | [diff] [blame] | 322 | wm.para = newParaWorker(wm.paraChan, numJobs, paraPath) |
Shinichiro Hamaji | cedc5c8 | 2015-05-13 17:03:20 +0900 | [diff] [blame] | 323 | go wm.para.Run() |
| 324 | } else { |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 325 | wm.busyWorkers = make(map[*worker]bool) |
Fumitoshi Ukai | 744bb2b | 2015-06-25 00:10:52 +0900 | [diff] [blame] | 326 | for i := 0; i < numJobs; i++ { |
| 327 | w := newWorker(wm) |
Shinichiro Hamaji | cedc5c8 | 2015-05-13 17:03:20 +0900 | [diff] [blame] | 328 | wm.freeWorkers = append(wm.freeWorkers, w) |
| 329 | go w.Run() |
| 330 | } |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 331 | } |
Shinichiro Hamaji | cedc5c8 | 2015-05-13 17:03:20 +0900 | [diff] [blame] | 332 | heap.Init(&wm.readyQueue) |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 333 | go wm.Run() |
| 334 | return wm |
| 335 | } |
| 336 | |
| 337 | func exitStatus(err error) int { |
| 338 | if err == nil { |
| 339 | return 0 |
| 340 | } |
| 341 | exit := 1 |
| 342 | if err, ok := err.(*exec.ExitError); ok { |
| 343 | if w, ok := err.ProcessState.Sys().(syscall.WaitStatus); ok { |
| 344 | return w.ExitStatus() |
| 345 | } |
| 346 | } |
| 347 | return exit |
| 348 | } |
| 349 | |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 350 | func (wm *workerManager) hasTodo() bool { |
Shinichiro Hamaji | dbc6c13 | 2015-04-28 18:26:36 +0900 | [diff] [blame] | 351 | return wm.finishCnt != len(wm.jobs) |
| 352 | } |
| 353 | |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 354 | func (wm *workerManager) maybePushToReadyQueue(j *job) { |
Shinichiro Hamaji | dbc6c13 | 2015-04-28 18:26:36 +0900 | [diff] [blame] | 355 | if j.numDeps != 0 { |
| 356 | return |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 357 | } |
Shinichiro Hamaji | dbc6c13 | 2015-04-28 18:26:36 +0900 | [diff] [blame] | 358 | heap.Push(&wm.readyQueue, j) |
Fumitoshi Ukai | 07cf121 | 2015-06-25 17:16:25 +0900 | [diff] [blame^] | 359 | logf("ready: %s", j.n.Output) |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 360 | } |
| 361 | |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 362 | func (wm *workerManager) handleNewDep(j *job, neededBy *job) { |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 363 | if j.numDeps < 0 { |
| 364 | neededBy.numDeps-- |
Shinichiro Hamaji | 5180c97 | 2015-04-28 20:14:38 +0900 | [diff] [blame] | 365 | if neededBy.id > 0 { |
| 366 | panic("already in WM... can this happen?") |
Shinichiro Hamaji | 5180c97 | 2015-04-28 20:14:38 +0900 | [diff] [blame] | 367 | } |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 368 | } else { |
| 369 | j.parents = append(j.parents, neededBy) |
| 370 | } |
| 371 | } |
| 372 | |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 373 | func (wm *workerManager) Run() { |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 374 | done := false |
Shinichiro Hamaji | cedc5c8 | 2015-05-13 17:03:20 +0900 | [diff] [blame] | 375 | for wm.hasTodo() || len(wm.busyWorkers) > 0 || len(wm.runnings) > 0 || !done { |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 376 | select { |
| 377 | case j := <-wm.jobChan: |
Fumitoshi Ukai | 07cf121 | 2015-06-25 17:16:25 +0900 | [diff] [blame^] | 378 | logf("wait: %s (%d)", j.n.Output, j.numDeps) |
Shinichiro Hamaji | 5180c97 | 2015-04-28 20:14:38 +0900 | [diff] [blame] | 379 | j.id = len(wm.jobs) + 1 |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 380 | wm.jobs = append(wm.jobs, j) |
Shinichiro Hamaji | dbc6c13 | 2015-04-28 18:26:36 +0900 | [diff] [blame] | 381 | wm.maybePushToReadyQueue(j) |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 382 | case jr := <-wm.resultChan: |
Fumitoshi Ukai | 07cf121 | 2015-06-25 17:16:25 +0900 | [diff] [blame^] | 383 | logf("done: %s", jr.j.n.Output) |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 384 | delete(wm.busyWorkers, jr.w) |
| 385 | wm.freeWorkers = append(wm.freeWorkers, jr.w) |
| 386 | wm.updateParents(jr.j) |
Shinichiro Hamaji | dbc6c13 | 2015-04-28 18:26:36 +0900 | [diff] [blame] | 387 | wm.finishCnt++ |
Shinichiro Hamaji | 55c50bd | 2015-04-27 21:05:47 +0900 | [diff] [blame] | 388 | case af := <-wm.newDepChan: |
| 389 | wm.handleNewDep(af.j, af.neededBy) |
Fumitoshi Ukai | 07cf121 | 2015-06-25 17:16:25 +0900 | [diff] [blame^] | 390 | logf("dep: %s (%d) %s", af.neededBy.n.Output, af.neededBy.numDeps, af.j.n.Output) |
Shinichiro Hamaji | cedc5c8 | 2015-05-13 17:03:20 +0900 | [diff] [blame] | 391 | case pr := <-wm.paraChan: |
Shinichiro Hamaji | a680842 | 2015-05-13 18:00:50 +0900 | [diff] [blame] | 392 | if pr.status < 0 && pr.signal < 0 { |
| 393 | j := wm.runnings[pr.output] |
| 394 | for _, r := range j.runners { |
Fumitoshi Ukai | 744bb2b | 2015-06-25 00:10:52 +0900 | [diff] [blame] | 395 | if r.echo || DryRunFlag { |
Shinichiro Hamaji | a680842 | 2015-05-13 18:00:50 +0900 | [diff] [blame] | 396 | fmt.Printf("%s\n", r.cmd) |
| 397 | } |
| 398 | } |
| 399 | } else { |
Fumitoshi Ukai | 145598a | 2015-06-19 10:08:17 +0900 | [diff] [blame] | 400 | fmt.Fprint(os.Stdout, pr.stdout) |
| 401 | fmt.Fprint(os.Stderr, pr.stderr) |
Shinichiro Hamaji | a680842 | 2015-05-13 18:00:50 +0900 | [diff] [blame] | 402 | j := wm.runnings[pr.output] |
| 403 | wm.updateParents(j) |
| 404 | delete(wm.runnings, pr.output) |
| 405 | wm.finishCnt++ |
| 406 | } |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 407 | case done = <-wm.waitChan: |
| 408 | } |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 409 | wm.handleJobs() |
Shinichiro Hamaji | cedc5c8 | 2015-05-13 17:03:20 +0900 | [diff] [blame] | 410 | |
Fumitoshi Ukai | 744bb2b | 2015-06-25 00:10:52 +0900 | [diff] [blame] | 411 | if wm.para != nil { |
Shinichiro Hamaji | cedc5c8 | 2015-05-13 17:03:20 +0900 | [diff] [blame] | 412 | numBusy := len(wm.runnings) |
Fumitoshi Ukai | 744bb2b | 2015-06-25 00:10:52 +0900 | [diff] [blame] | 413 | if numBusy > wm.maxJobs { |
| 414 | numBusy = wm.maxJobs |
Shinichiro Hamaji | cedc5c8 | 2015-05-13 17:03:20 +0900 | [diff] [blame] | 415 | } |
Fumitoshi Ukai | 07cf121 | 2015-06-25 17:16:25 +0900 | [diff] [blame^] | 416 | logf("job=%d ready=%d free=%d busy=%d", len(wm.jobs)-wm.finishCnt, wm.readyQueue.Len(), wm.maxJobs-numBusy, numBusy) |
Shinichiro Hamaji | cedc5c8 | 2015-05-13 17:03:20 +0900 | [diff] [blame] | 417 | } else { |
Fumitoshi Ukai | 07cf121 | 2015-06-25 17:16:25 +0900 | [diff] [blame^] | 418 | logf("job=%d ready=%d free=%d busy=%d", len(wm.jobs)-wm.finishCnt, wm.readyQueue.Len(), len(wm.freeWorkers), len(wm.busyWorkers)) |
Shinichiro Hamaji | cedc5c8 | 2015-05-13 17:03:20 +0900 | [diff] [blame] | 419 | } |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 420 | } |
| 421 | |
Fumitoshi Ukai | 744bb2b | 2015-06-25 00:10:52 +0900 | [diff] [blame] | 422 | if wm.para != nil { |
Fumitoshi Ukai | 07cf121 | 2015-06-25 17:16:25 +0900 | [diff] [blame^] | 423 | logf("Wait for para to finish") |
Shinichiro Hamaji | cedc5c8 | 2015-05-13 17:03:20 +0900 | [diff] [blame] | 424 | wm.para.Wait() |
| 425 | } else { |
| 426 | for _, w := range wm.freeWorkers { |
| 427 | w.Wait() |
| 428 | } |
| 429 | for w := range wm.busyWorkers { |
| 430 | w.Wait() |
| 431 | } |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 432 | } |
| 433 | wm.doneChan <- true |
| 434 | } |
| 435 | |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 436 | func (wm *workerManager) PostJob(j *job) { |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 437 | wm.jobChan <- j |
| 438 | } |
| 439 | |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 440 | func (wm *workerManager) ReportResult(w *worker, j *job) { |
| 441 | wm.resultChan <- jobResult{w: w, j: j} |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 442 | } |
| 443 | |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 444 | func (wm *workerManager) ReportNewDep(j *job, neededBy *job) { |
| 445 | wm.newDepChan <- newDep{j: j, neededBy: neededBy} |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 446 | } |
| 447 | |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 448 | func (wm *workerManager) Wait() { |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 449 | wm.waitChan <- true |
| 450 | <-wm.doneChan |
| 451 | } |