Shinichiro Hamaji | b69bf8a | 2015-06-10 14:52:06 +0900 | [diff] [blame] | 1 | // Copyright 2015 Google Inc. All rights reserved |
| 2 | // |
| 3 | // Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | // you may not use this file except in compliance with the License. |
| 5 | // You may obtain a copy of the License at |
| 6 | // |
| 7 | // http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | // |
| 9 | // Unless required by applicable law or agreed to in writing, software |
| 10 | // distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | // See the License for the specific language governing permissions and |
| 13 | // limitations under the License. |
| 14 | |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 15 | package main |
| 16 | |
| 17 | import ( |
Shinichiro Hamaji | dbc6c13 | 2015-04-28 18:26:36 +0900 | [diff] [blame] | 18 | "container/heap" |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 19 | "fmt" |
Shinichiro Hamaji | cedc5c8 | 2015-05-13 17:03:20 +0900 | [diff] [blame] | 20 | "os" |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 21 | "os/exec" |
| 22 | "strings" |
| 23 | "syscall" |
| 24 | "time" |
| 25 | ) |
| 26 | |
| 27 | type Job struct { |
| 28 | n *DepNode |
| 29 | ex *Executor |
| 30 | parents []*Job |
| 31 | outputTs int64 |
| 32 | numDeps int |
| 33 | depsTs int64 |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 34 | id int |
Shinichiro Hamaji | a680842 | 2015-05-13 18:00:50 +0900 | [diff] [blame] | 35 | |
| 36 | runners []runner |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 37 | } |
| 38 | |
| 39 | type runner struct { |
| 40 | output string |
| 41 | cmd string |
| 42 | echo bool |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 43 | ignoreError bool |
| 44 | shell string |
| 45 | } |
| 46 | |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 47 | type JobResult struct { |
| 48 | j *Job |
| 49 | w *Worker |
| 50 | } |
| 51 | |
Shinichiro Hamaji | 55c50bd | 2015-04-27 21:05:47 +0900 | [diff] [blame] | 52 | type NewDep struct { |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 53 | j *Job |
| 54 | neededBy *Job |
| 55 | } |
| 56 | |
| 57 | type Worker struct { |
| 58 | wm *WorkerManager |
| 59 | jobChan chan *Job |
| 60 | waitChan chan bool |
| 61 | doneChan chan bool |
| 62 | } |
| 63 | |
Shinichiro Hamaji | dbc6c13 | 2015-04-28 18:26:36 +0900 | [diff] [blame] | 64 | type JobQueue []*Job |
| 65 | |
| 66 | func (jq JobQueue) Len() int { return len(jq) } |
| 67 | |
| 68 | func (jq JobQueue) Less(i, j int) bool { |
| 69 | // First come, first serve, for GNU make compatibility. |
| 70 | return jq[i].id < jq[j].id |
| 71 | } |
| 72 | |
| 73 | func (jq JobQueue) Swap(i, j int) { |
| 74 | jq[i], jq[j] = jq[j], jq[i] |
| 75 | } |
| 76 | |
| 77 | func (jq *JobQueue) Push(x interface{}) { |
| 78 | item := x.(*Job) |
| 79 | *jq = append(*jq, item) |
| 80 | } |
| 81 | |
| 82 | func (jq *JobQueue) Pop() interface{} { |
| 83 | old := *jq |
| 84 | n := len(old) |
| 85 | item := old[n-1] |
| 86 | *jq = old[0 : n-1] |
| 87 | return item |
| 88 | } |
| 89 | |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 90 | func NewWorker(wm *WorkerManager) *Worker { |
| 91 | w := &Worker{ |
| 92 | wm: wm, |
| 93 | jobChan: make(chan *Job), |
| 94 | waitChan: make(chan bool), |
| 95 | doneChan: make(chan bool), |
| 96 | } |
| 97 | return w |
| 98 | } |
| 99 | |
| 100 | func (w *Worker) Run() { |
| 101 | done := false |
| 102 | for !done { |
| 103 | select { |
| 104 | case j := <-w.jobChan: |
| 105 | j.build() |
| 106 | w.wm.ReportResult(w, j) |
| 107 | case done = <-w.waitChan: |
| 108 | } |
| 109 | } |
| 110 | w.doneChan <- true |
| 111 | } |
| 112 | |
| 113 | func (w *Worker) PostJob(j *Job) { |
| 114 | w.jobChan <- j |
| 115 | } |
| 116 | |
| 117 | func (w *Worker) Wait() { |
| 118 | w.waitChan <- true |
| 119 | <-w.doneChan |
| 120 | } |
| 121 | |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 122 | func evalCmd(ev *Evaluator, r runner, s string) []runner { |
| 123 | r = newRunner(r, s) |
| 124 | if strings.IndexByte(r.cmd, '$') < 0 { |
| 125 | // fast path |
| 126 | return []runner{r} |
| 127 | } |
| 128 | // TODO(ukai): parse once more earlier? |
Fumitoshi Ukai | 7c9aa9f | 2015-06-12 23:51:38 +0900 | [diff] [blame] | 129 | expr, _, err := parseExpr([]byte(r.cmd), nil, false) |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 130 | if err != nil { |
| 131 | panic(fmt.Errorf("parse cmd %q: %v", r.cmd, err)) |
| 132 | } |
Fumitoshi Ukai | b06cd9d | 2015-05-07 12:56:12 +0900 | [diff] [blame] | 133 | buf := newBuf() |
| 134 | expr.Eval(buf, ev) |
| 135 | cmds := buf.String() |
| 136 | freeBuf(buf) |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 137 | var runners []runner |
| 138 | for _, cmd := range strings.Split(cmds, "\n") { |
Shinichiro Hamaji | 212abfb | 2015-04-29 03:02:59 +0900 | [diff] [blame] | 139 | if len(runners) > 0 && strings.HasSuffix(runners[len(runners)-1].cmd, "\\") { |
| 140 | runners[len(runners)-1].cmd += "\n" |
| 141 | runners[len(runners)-1].cmd += cmd |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 142 | } else { |
| 143 | runners = append(runners, newRunner(r, cmd)) |
| 144 | } |
| 145 | } |
| 146 | return runners |
| 147 | } |
| 148 | |
| 149 | func newRunner(r runner, s string) runner { |
| 150 | for { |
| 151 | s = trimLeftSpace(s) |
| 152 | if s == "" { |
| 153 | return runner{} |
| 154 | } |
| 155 | switch s[0] { |
| 156 | case '@': |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 157 | if !dryRunFlag { |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 158 | r.echo = false |
| 159 | } |
| 160 | s = s[1:] |
| 161 | continue |
| 162 | case '-': |
| 163 | r.ignoreError = true |
| 164 | s = s[1:] |
| 165 | continue |
| 166 | } |
| 167 | break |
| 168 | } |
| 169 | r.cmd = s |
| 170 | return r |
| 171 | } |
| 172 | |
| 173 | func (r runner) run(output string) error { |
| 174 | if r.echo || dryRunFlag { |
| 175 | fmt.Printf("%s\n", r.cmd) |
| 176 | } |
| 177 | if dryRunFlag { |
| 178 | return nil |
| 179 | } |
| 180 | args := []string{r.shell, "-c", r.cmd} |
| 181 | cmd := exec.Cmd{ |
| 182 | Path: args[0], |
| 183 | Args: args, |
| 184 | } |
| 185 | out, err := cmd.CombinedOutput() |
| 186 | fmt.Printf("%s", out) |
| 187 | exit := exitStatus(err) |
| 188 | if r.ignoreError && exit != 0 { |
| 189 | fmt.Printf("[%s] Error %d (ignored)\n", output, exit) |
| 190 | err = nil |
| 191 | } |
| 192 | return err |
| 193 | } |
| 194 | |
Fumitoshi Ukai | 868fb1c | 2015-06-05 13:25:53 +0900 | [diff] [blame] | 195 | func (j *Job) createRunners() []runner { |
Shinichiro Hamaji | b41fd50 | 2015-04-29 03:34:07 +0900 | [diff] [blame] | 196 | runners, _ := j.ex.createRunners(j.n, false) |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 197 | return runners |
| 198 | } |
| 199 | |
Shinichiro Hamaji | 71fae4c | 2015-05-25 17:48:34 +0900 | [diff] [blame] | 200 | // TODO(ukai): use time.Time? |
| 201 | func getTimestamp(filename string) int64 { |
| 202 | st, err := os.Stat(filename) |
| 203 | if err != nil { |
| 204 | return -2 |
| 205 | } |
| 206 | return st.ModTime().Unix() |
| 207 | } |
| 208 | |
Fumitoshi Ukai | 868fb1c | 2015-06-05 13:25:53 +0900 | [diff] [blame] | 209 | func (j *Job) build() { |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 210 | if j.n.IsPhony { |
| 211 | j.outputTs = -2 // trigger cmd even if all inputs don't exist. |
| 212 | } else { |
| 213 | j.outputTs = getTimestamp(j.n.Output) |
| 214 | } |
| 215 | |
| 216 | if !j.n.HasRule { |
| 217 | if j.outputTs >= 0 || j.n.IsPhony { |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 218 | return |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 219 | } |
| 220 | if len(j.parents) == 0 { |
| 221 | ErrorNoLocation("*** No rule to make target %q.", j.n.Output) |
| 222 | } else { |
| 223 | ErrorNoLocation("*** No rule to make target %q, needed by %q.", j.n.Output, j.parents[0].n.Output) |
| 224 | } |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 225 | ErrorNoLocation("no rule to make target %q", j.n.Output) |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 226 | } |
| 227 | |
| 228 | if j.outputTs >= j.depsTs { |
| 229 | // TODO: stats. |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 230 | return |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 231 | } |
| 232 | |
| 233 | for _, r := range j.createRunners() { |
| 234 | err := r.run(j.n.Output) |
| 235 | if err != nil { |
| 236 | exit := exitStatus(err) |
| 237 | ErrorNoLocation("[%s] Error %d: %v", j.n.Output, exit, err) |
| 238 | } |
| 239 | } |
| 240 | |
| 241 | if j.n.IsPhony { |
| 242 | j.outputTs = time.Now().Unix() |
| 243 | } else { |
| 244 | j.outputTs = getTimestamp(j.n.Output) |
| 245 | if j.outputTs < 0 { |
| 246 | j.outputTs = time.Now().Unix() |
| 247 | } |
| 248 | } |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 249 | } |
| 250 | |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 251 | func (wm *WorkerManager) handleJobs() { |
| 252 | for { |
Shinichiro Hamaji | cedc5c8 | 2015-05-13 17:03:20 +0900 | [diff] [blame] | 253 | if !useParaFlag && len(wm.freeWorkers) == 0 { |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 254 | return |
| 255 | } |
Shinichiro Hamaji | dbc6c13 | 2015-04-28 18:26:36 +0900 | [diff] [blame] | 256 | if wm.readyQueue.Len() == 0 { |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 257 | return |
| 258 | } |
Shinichiro Hamaji | dbc6c13 | 2015-04-28 18:26:36 +0900 | [diff] [blame] | 259 | j := heap.Pop(&wm.readyQueue).(*Job) |
Fumitoshi Ukai | 8fabdd0 | 2015-06-08 10:27:47 +0900 | [diff] [blame] | 260 | Logf("run: %s", j.n.Output) |
Shinichiro Hamaji | cedc5c8 | 2015-05-13 17:03:20 +0900 | [diff] [blame] | 261 | |
| 262 | if useParaFlag { |
Shinichiro Hamaji | a680842 | 2015-05-13 18:00:50 +0900 | [diff] [blame] | 263 | j.runners = j.createRunners() |
| 264 | if len(j.runners) == 0 { |
| 265 | wm.updateParents(j) |
| 266 | wm.finishCnt++ |
| 267 | } else { |
| 268 | wm.runnings[j.n.Output] = j |
| 269 | wm.para.RunCommand(j.runners) |
| 270 | } |
Shinichiro Hamaji | cedc5c8 | 2015-05-13 17:03:20 +0900 | [diff] [blame] | 271 | } else { |
| 272 | j.numDeps = -1 // Do not let other workers pick this. |
| 273 | w := wm.freeWorkers[0] |
| 274 | wm.freeWorkers = wm.freeWorkers[1:] |
| 275 | wm.busyWorkers[w] = true |
| 276 | w.jobChan <- j |
| 277 | } |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 278 | } |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 279 | } |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 280 | |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 281 | func (wm *WorkerManager) updateParents(j *Job) { |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 282 | for _, p := range j.parents { |
| 283 | p.numDeps-- |
Fumitoshi Ukai | 8fabdd0 | 2015-06-08 10:27:47 +0900 | [diff] [blame] | 284 | Logf("child: %s (%d)", p.n.Output, p.numDeps) |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 285 | if p.depsTs < j.outputTs { |
| 286 | p.depsTs = j.outputTs |
| 287 | } |
Shinichiro Hamaji | dbc6c13 | 2015-04-28 18:26:36 +0900 | [diff] [blame] | 288 | wm.maybePushToReadyQueue(p) |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 289 | } |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 290 | } |
| 291 | |
| 292 | type WorkerManager struct { |
Shinichiro Hamaji | 55c50bd | 2015-04-27 21:05:47 +0900 | [diff] [blame] | 293 | jobs []*Job |
Shinichiro Hamaji | dbc6c13 | 2015-04-28 18:26:36 +0900 | [diff] [blame] | 294 | readyQueue JobQueue |
Shinichiro Hamaji | 55c50bd | 2015-04-27 21:05:47 +0900 | [diff] [blame] | 295 | jobChan chan *Job |
| 296 | resultChan chan JobResult |
| 297 | newDepChan chan NewDep |
| 298 | waitChan chan bool |
| 299 | doneChan chan bool |
| 300 | freeWorkers []*Worker |
| 301 | busyWorkers map[*Worker]bool |
Shinichiro Hamaji | cedc5c8 | 2015-05-13 17:03:20 +0900 | [diff] [blame] | 302 | ex *Executor |
| 303 | para *ParaWorker |
| 304 | paraChan chan *ParaResult |
| 305 | runnings map[string]*Job |
Shinichiro Hamaji | dbc6c13 | 2015-04-28 18:26:36 +0900 | [diff] [blame] | 306 | |
| 307 | finishCnt int |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 308 | } |
| 309 | |
| 310 | func NewWorkerManager() *WorkerManager { |
| 311 | wm := &WorkerManager{ |
Shinichiro Hamaji | 55c50bd | 2015-04-27 21:05:47 +0900 | [diff] [blame] | 312 | jobChan: make(chan *Job), |
| 313 | resultChan: make(chan JobResult), |
| 314 | newDepChan: make(chan NewDep), |
| 315 | waitChan: make(chan bool), |
| 316 | doneChan: make(chan bool), |
| 317 | busyWorkers: make(map[*Worker]bool), |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 318 | } |
Shinichiro Hamaji | cedc5c8 | 2015-05-13 17:03:20 +0900 | [diff] [blame] | 319 | |
| 320 | if useParaFlag { |
| 321 | wm.runnings = make(map[string]*Job) |
| 322 | wm.paraChan = make(chan *ParaResult) |
| 323 | wm.para = NewParaWorker(wm.paraChan) |
| 324 | go wm.para.Run() |
| 325 | } else { |
| 326 | wm.busyWorkers = make(map[*Worker]bool) |
| 327 | for i := 0; i < jobsFlag; i++ { |
| 328 | w := NewWorker(wm) |
| 329 | wm.freeWorkers = append(wm.freeWorkers, w) |
| 330 | go w.Run() |
| 331 | } |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 332 | } |
Shinichiro Hamaji | cedc5c8 | 2015-05-13 17:03:20 +0900 | [diff] [blame] | 333 | heap.Init(&wm.readyQueue) |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 334 | go wm.Run() |
| 335 | return wm |
| 336 | } |
| 337 | |
| 338 | func exitStatus(err error) int { |
| 339 | if err == nil { |
| 340 | return 0 |
| 341 | } |
| 342 | exit := 1 |
| 343 | if err, ok := err.(*exec.ExitError); ok { |
| 344 | if w, ok := err.ProcessState.Sys().(syscall.WaitStatus); ok { |
| 345 | return w.ExitStatus() |
| 346 | } |
| 347 | } |
| 348 | return exit |
| 349 | } |
| 350 | |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 351 | func (wm *WorkerManager) hasTodo() bool { |
Shinichiro Hamaji | dbc6c13 | 2015-04-28 18:26:36 +0900 | [diff] [blame] | 352 | return wm.finishCnt != len(wm.jobs) |
| 353 | } |
| 354 | |
| 355 | func (wm *WorkerManager) maybePushToReadyQueue(j *Job) { |
| 356 | if j.numDeps != 0 { |
| 357 | return |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 358 | } |
Shinichiro Hamaji | dbc6c13 | 2015-04-28 18:26:36 +0900 | [diff] [blame] | 359 | heap.Push(&wm.readyQueue, j) |
Fumitoshi Ukai | 8fabdd0 | 2015-06-08 10:27:47 +0900 | [diff] [blame] | 360 | Logf("ready: %s", j.n.Output) |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 361 | } |
| 362 | |
Shinichiro Hamaji | 55c50bd | 2015-04-27 21:05:47 +0900 | [diff] [blame] | 363 | func (wm *WorkerManager) handleNewDep(j *Job, neededBy *Job) { |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 364 | if j.numDeps < 0 { |
| 365 | neededBy.numDeps-- |
Shinichiro Hamaji | 5180c97 | 2015-04-28 20:14:38 +0900 | [diff] [blame] | 366 | if neededBy.id > 0 { |
| 367 | panic("already in WM... can this happen?") |
Shinichiro Hamaji | 5180c97 | 2015-04-28 20:14:38 +0900 | [diff] [blame] | 368 | } |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 369 | } else { |
| 370 | j.parents = append(j.parents, neededBy) |
| 371 | } |
| 372 | } |
| 373 | |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 374 | func (wm *WorkerManager) Run() { |
| 375 | done := false |
Shinichiro Hamaji | cedc5c8 | 2015-05-13 17:03:20 +0900 | [diff] [blame] | 376 | for wm.hasTodo() || len(wm.busyWorkers) > 0 || len(wm.runnings) > 0 || !done { |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 377 | select { |
| 378 | case j := <-wm.jobChan: |
Fumitoshi Ukai | 8fabdd0 | 2015-06-08 10:27:47 +0900 | [diff] [blame] | 379 | Logf("wait: %s (%d)", j.n.Output, j.numDeps) |
Shinichiro Hamaji | 5180c97 | 2015-04-28 20:14:38 +0900 | [diff] [blame] | 380 | j.id = len(wm.jobs) + 1 |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 381 | wm.jobs = append(wm.jobs, j) |
Shinichiro Hamaji | dbc6c13 | 2015-04-28 18:26:36 +0900 | [diff] [blame] | 382 | wm.maybePushToReadyQueue(j) |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 383 | case jr := <-wm.resultChan: |
Fumitoshi Ukai | 8fabdd0 | 2015-06-08 10:27:47 +0900 | [diff] [blame] | 384 | Logf("done: %s", jr.j.n.Output) |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 385 | delete(wm.busyWorkers, jr.w) |
| 386 | wm.freeWorkers = append(wm.freeWorkers, jr.w) |
| 387 | wm.updateParents(jr.j) |
Shinichiro Hamaji | dbc6c13 | 2015-04-28 18:26:36 +0900 | [diff] [blame] | 388 | wm.finishCnt++ |
Shinichiro Hamaji | 55c50bd | 2015-04-27 21:05:47 +0900 | [diff] [blame] | 389 | case af := <-wm.newDepChan: |
| 390 | wm.handleNewDep(af.j, af.neededBy) |
Fumitoshi Ukai | 8fabdd0 | 2015-06-08 10:27:47 +0900 | [diff] [blame] | 391 | Logf("dep: %s (%d) %s", af.neededBy.n.Output, af.neededBy.numDeps, af.j.n.Output) |
Shinichiro Hamaji | cedc5c8 | 2015-05-13 17:03:20 +0900 | [diff] [blame] | 392 | case pr := <-wm.paraChan: |
Shinichiro Hamaji | a680842 | 2015-05-13 18:00:50 +0900 | [diff] [blame] | 393 | if pr.status < 0 && pr.signal < 0 { |
| 394 | j := wm.runnings[pr.output] |
| 395 | for _, r := range j.runners { |
| 396 | if r.echo || dryRunFlag { |
| 397 | fmt.Printf("%s\n", r.cmd) |
| 398 | } |
| 399 | } |
| 400 | } else { |
| 401 | os.Stdout.Write([]byte(pr.stdout)) |
| 402 | os.Stderr.Write([]byte(pr.stderr)) |
| 403 | j := wm.runnings[pr.output] |
| 404 | wm.updateParents(j) |
| 405 | delete(wm.runnings, pr.output) |
| 406 | wm.finishCnt++ |
| 407 | } |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 408 | case done = <-wm.waitChan: |
| 409 | } |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 410 | wm.handleJobs() |
Shinichiro Hamaji | cedc5c8 | 2015-05-13 17:03:20 +0900 | [diff] [blame] | 411 | |
| 412 | if useParaFlag { |
| 413 | numBusy := len(wm.runnings) |
| 414 | if numBusy > jobsFlag { |
| 415 | numBusy = jobsFlag |
| 416 | } |
Fumitoshi Ukai | 8fabdd0 | 2015-06-08 10:27:47 +0900 | [diff] [blame] | 417 | Logf("job=%d ready=%d free=%d busy=%d", len(wm.jobs)-wm.finishCnt, wm.readyQueue.Len(), jobsFlag-numBusy, numBusy) |
Shinichiro Hamaji | cedc5c8 | 2015-05-13 17:03:20 +0900 | [diff] [blame] | 418 | } else { |
Fumitoshi Ukai | 8fabdd0 | 2015-06-08 10:27:47 +0900 | [diff] [blame] | 419 | Logf("job=%d ready=%d free=%d busy=%d", len(wm.jobs)-wm.finishCnt, wm.readyQueue.Len(), len(wm.freeWorkers), len(wm.busyWorkers)) |
Shinichiro Hamaji | cedc5c8 | 2015-05-13 17:03:20 +0900 | [diff] [blame] | 420 | } |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 421 | } |
| 422 | |
Shinichiro Hamaji | cedc5c8 | 2015-05-13 17:03:20 +0900 | [diff] [blame] | 423 | if useParaFlag { |
Fumitoshi Ukai | 8fabdd0 | 2015-06-08 10:27:47 +0900 | [diff] [blame] | 424 | Logf("Wait for para to finish") |
Shinichiro Hamaji | cedc5c8 | 2015-05-13 17:03:20 +0900 | [diff] [blame] | 425 | wm.para.Wait() |
| 426 | } else { |
| 427 | for _, w := range wm.freeWorkers { |
| 428 | w.Wait() |
| 429 | } |
| 430 | for w := range wm.busyWorkers { |
| 431 | w.Wait() |
| 432 | } |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 433 | } |
| 434 | wm.doneChan <- true |
| 435 | } |
| 436 | |
| 437 | func (wm *WorkerManager) PostJob(j *Job) { |
| 438 | wm.jobChan <- j |
| 439 | } |
| 440 | |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 441 | func (wm *WorkerManager) ReportResult(w *Worker, j *Job) { |
| 442 | wm.resultChan <- JobResult{w: w, j: j} |
| 443 | } |
| 444 | |
Shinichiro Hamaji | 55c50bd | 2015-04-27 21:05:47 +0900 | [diff] [blame] | 445 | func (wm *WorkerManager) ReportNewDep(j *Job, neededBy *Job) { |
| 446 | wm.newDepChan <- NewDep{j: j, neededBy: neededBy} |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 447 | } |
| 448 | |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 449 | func (wm *WorkerManager) Wait() { |
| 450 | wm.waitChan <- true |
| 451 | <-wm.doneChan |
| 452 | } |