Shinichiro Hamaji | b69bf8a | 2015-06-10 14:52:06 +0900 | [diff] [blame] | 1 | // Copyright 2015 Google Inc. All rights reserved |
| 2 | // |
| 3 | // Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | // you may not use this file except in compliance with the License. |
| 5 | // You may obtain a copy of the License at |
| 6 | // |
| 7 | // http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | // |
| 9 | // Unless required by applicable law or agreed to in writing, software |
| 10 | // distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | // See the License for the specific language governing permissions and |
| 13 | // limitations under the License. |
| 14 | |
Fumitoshi Ukai | 744bb2b | 2015-06-25 00:10:52 +0900 | [diff] [blame] | 15 | package kati |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 16 | |
| 17 | import ( |
Shinichiro Hamaji | dbc6c13 | 2015-04-28 18:26:36 +0900 | [diff] [blame] | 18 | "container/heap" |
Fumitoshi Ukai | 531c5d2 | 2015-06-27 00:58:56 +0900 | [diff] [blame^] | 19 | "errors" |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 20 | "fmt" |
Shinichiro Hamaji | cedc5c8 | 2015-05-13 17:03:20 +0900 | [diff] [blame] | 21 | "os" |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 22 | "os/exec" |
| 23 | "strings" |
| 24 | "syscall" |
| 25 | "time" |
| 26 | ) |
| 27 | |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 28 | type job struct { |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 29 | n *DepNode |
| 30 | ex *Executor |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 31 | parents []*job |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 32 | outputTs int64 |
| 33 | numDeps int |
| 34 | depsTs int64 |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 35 | id int |
Shinichiro Hamaji | a680842 | 2015-05-13 18:00:50 +0900 | [diff] [blame] | 36 | |
| 37 | runners []runner |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 38 | } |
| 39 | |
| 40 | type runner struct { |
| 41 | output string |
| 42 | cmd string |
| 43 | echo bool |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 44 | ignoreError bool |
| 45 | shell string |
| 46 | } |
| 47 | |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 48 | type jobResult struct { |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 49 | j *job |
| 50 | w *worker |
| 51 | err error |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 52 | } |
| 53 | |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 54 | type newDep struct { |
| 55 | j *job |
| 56 | neededBy *job |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 57 | } |
| 58 | |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 59 | type worker struct { |
| 60 | wm *workerManager |
| 61 | jobChan chan *job |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 62 | waitChan chan bool |
| 63 | doneChan chan bool |
| 64 | } |
| 65 | |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 66 | type jobQueue []*job |
Shinichiro Hamaji | dbc6c13 | 2015-04-28 18:26:36 +0900 | [diff] [blame] | 67 | |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 68 | func (jq jobQueue) Len() int { return len(jq) } |
| 69 | func (jq jobQueue) Swap(i, j int) { jq[i], jq[j] = jq[j], jq[i] } |
Shinichiro Hamaji | dbc6c13 | 2015-04-28 18:26:36 +0900 | [diff] [blame] | 70 | |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 71 | func (jq jobQueue) Less(i, j int) bool { |
Shinichiro Hamaji | dbc6c13 | 2015-04-28 18:26:36 +0900 | [diff] [blame] | 72 | // First come, first serve, for GNU make compatibility. |
| 73 | return jq[i].id < jq[j].id |
| 74 | } |
| 75 | |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 76 | func (jq *jobQueue) Push(x interface{}) { |
| 77 | item := x.(*job) |
Shinichiro Hamaji | dbc6c13 | 2015-04-28 18:26:36 +0900 | [diff] [blame] | 78 | *jq = append(*jq, item) |
| 79 | } |
| 80 | |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 81 | func (jq *jobQueue) Pop() interface{} { |
Shinichiro Hamaji | dbc6c13 | 2015-04-28 18:26:36 +0900 | [diff] [blame] | 82 | old := *jq |
| 83 | n := len(old) |
| 84 | item := old[n-1] |
| 85 | *jq = old[0 : n-1] |
| 86 | return item |
| 87 | } |
| 88 | |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 89 | func newWorker(wm *workerManager) *worker { |
| 90 | w := &worker{ |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 91 | wm: wm, |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 92 | jobChan: make(chan *job), |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 93 | waitChan: make(chan bool), |
| 94 | doneChan: make(chan bool), |
| 95 | } |
| 96 | return w |
| 97 | } |
| 98 | |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 99 | func (w *worker) Run() { |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 100 | done := false |
| 101 | for !done { |
| 102 | select { |
| 103 | case j := <-w.jobChan: |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 104 | err := j.build() |
| 105 | w.wm.ReportResult(w, j, err) |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 106 | case done = <-w.waitChan: |
| 107 | } |
| 108 | } |
| 109 | w.doneChan <- true |
| 110 | } |
| 111 | |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 112 | func (w *worker) PostJob(j *job) { |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 113 | w.jobChan <- j |
| 114 | } |
| 115 | |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 116 | func (w *worker) Wait() { |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 117 | w.waitChan <- true |
| 118 | <-w.doneChan |
| 119 | } |
| 120 | |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 121 | func evalCmd(ev *Evaluator, r runner, s string) ([]runner, error) { |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 122 | r = newRunner(r, s) |
| 123 | if strings.IndexByte(r.cmd, '$') < 0 { |
| 124 | // fast path |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 125 | return []runner{r}, nil |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 126 | } |
| 127 | // TODO(ukai): parse once more earlier? |
Fumitoshi Ukai | 7c9aa9f | 2015-06-12 23:51:38 +0900 | [diff] [blame] | 128 | expr, _, err := parseExpr([]byte(r.cmd), nil, false) |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 129 | if err != nil { |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 130 | return nil, ev.errorf("parse cmd %q: %v", r.cmd, err) |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 131 | } |
Fumitoshi Ukai | b06cd9d | 2015-05-07 12:56:12 +0900 | [diff] [blame] | 132 | buf := newBuf() |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 133 | err = expr.Eval(buf, ev) |
| 134 | if err != nil { |
| 135 | return nil, err |
| 136 | } |
Fumitoshi Ukai | b06cd9d | 2015-05-07 12:56:12 +0900 | [diff] [blame] | 137 | cmds := buf.String() |
| 138 | freeBuf(buf) |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 139 | var runners []runner |
| 140 | for _, cmd := range strings.Split(cmds, "\n") { |
Shinichiro Hamaji | 212abfb | 2015-04-29 03:02:59 +0900 | [diff] [blame] | 141 | if len(runners) > 0 && strings.HasSuffix(runners[len(runners)-1].cmd, "\\") { |
| 142 | runners[len(runners)-1].cmd += "\n" |
| 143 | runners[len(runners)-1].cmd += cmd |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 144 | } else { |
| 145 | runners = append(runners, newRunner(r, cmd)) |
| 146 | } |
| 147 | } |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 148 | return runners, nil |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 149 | } |
| 150 | |
| 151 | func newRunner(r runner, s string) runner { |
| 152 | for { |
| 153 | s = trimLeftSpace(s) |
| 154 | if s == "" { |
| 155 | return runner{} |
| 156 | } |
| 157 | switch s[0] { |
| 158 | case '@': |
Fumitoshi Ukai | 744bb2b | 2015-06-25 00:10:52 +0900 | [diff] [blame] | 159 | if !DryRunFlag { |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 160 | r.echo = false |
| 161 | } |
| 162 | s = s[1:] |
| 163 | continue |
| 164 | case '-': |
| 165 | r.ignoreError = true |
| 166 | s = s[1:] |
| 167 | continue |
| 168 | } |
| 169 | break |
| 170 | } |
| 171 | r.cmd = s |
| 172 | return r |
| 173 | } |
| 174 | |
| 175 | func (r runner) run(output string) error { |
Fumitoshi Ukai | 744bb2b | 2015-06-25 00:10:52 +0900 | [diff] [blame] | 176 | if r.echo || DryRunFlag { |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 177 | fmt.Printf("%s\n", r.cmd) |
| 178 | } |
Fumitoshi Ukai | 744bb2b | 2015-06-25 00:10:52 +0900 | [diff] [blame] | 179 | if DryRunFlag { |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 180 | return nil |
| 181 | } |
| 182 | args := []string{r.shell, "-c", r.cmd} |
| 183 | cmd := exec.Cmd{ |
| 184 | Path: args[0], |
| 185 | Args: args, |
| 186 | } |
| 187 | out, err := cmd.CombinedOutput() |
| 188 | fmt.Printf("%s", out) |
| 189 | exit := exitStatus(err) |
| 190 | if r.ignoreError && exit != 0 { |
| 191 | fmt.Printf("[%s] Error %d (ignored)\n", output, exit) |
| 192 | err = nil |
| 193 | } |
| 194 | return err |
| 195 | } |
| 196 | |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 197 | func (j *job) createRunners() ([]runner, error) { |
| 198 | runners, _, err := j.ex.createRunners(j.n, false) |
| 199 | return runners, err |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 200 | } |
| 201 | |
Shinichiro Hamaji | 71fae4c | 2015-05-25 17:48:34 +0900 | [diff] [blame] | 202 | // TODO(ukai): use time.Time? |
| 203 | func getTimestamp(filename string) int64 { |
| 204 | st, err := os.Stat(filename) |
| 205 | if err != nil { |
| 206 | return -2 |
| 207 | } |
| 208 | return st.ModTime().Unix() |
| 209 | } |
| 210 | |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 211 | func (j *job) build() error { |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 212 | if j.n.IsPhony { |
| 213 | j.outputTs = -2 // trigger cmd even if all inputs don't exist. |
| 214 | } else { |
| 215 | j.outputTs = getTimestamp(j.n.Output) |
| 216 | } |
| 217 | |
| 218 | if !j.n.HasRule { |
| 219 | if j.outputTs >= 0 || j.n.IsPhony { |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 220 | return nil |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 221 | } |
| 222 | if len(j.parents) == 0 { |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 223 | return fmt.Errorf("*** No rule to make target %q.", j.n.Output) |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 224 | } |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 225 | return fmt.Errorf("*** No rule to make target %q, needed by %q.", j.n.Output, j.parents[0].n.Output) |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 226 | } |
| 227 | |
| 228 | if j.outputTs >= j.depsTs { |
| 229 | // TODO: stats. |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 230 | return nil |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 231 | } |
| 232 | |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 233 | rr, err := j.createRunners() |
| 234 | if err != nil { |
| 235 | return err |
| 236 | } |
| 237 | for _, r := range rr { |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 238 | err := r.run(j.n.Output) |
| 239 | if err != nil { |
| 240 | exit := exitStatus(err) |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 241 | return fmt.Errorf("[%s] Error %d: %v", j.n.Output, exit, err) |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 242 | } |
| 243 | } |
| 244 | |
| 245 | if j.n.IsPhony { |
| 246 | j.outputTs = time.Now().Unix() |
| 247 | } else { |
| 248 | j.outputTs = getTimestamp(j.n.Output) |
| 249 | if j.outputTs < 0 { |
| 250 | j.outputTs = time.Now().Unix() |
| 251 | } |
| 252 | } |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 253 | return nil |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 254 | } |
| 255 | |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 256 | func (wm *workerManager) handleJobs() error { |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 257 | for { |
Fumitoshi Ukai | 744bb2b | 2015-06-25 00:10:52 +0900 | [diff] [blame] | 258 | if wm.para == nil && len(wm.freeWorkers) == 0 { |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 259 | return nil |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 260 | } |
Shinichiro Hamaji | dbc6c13 | 2015-04-28 18:26:36 +0900 | [diff] [blame] | 261 | if wm.readyQueue.Len() == 0 { |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 262 | return nil |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 263 | } |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 264 | j := heap.Pop(&wm.readyQueue).(*job) |
Fumitoshi Ukai | 07cf121 | 2015-06-25 17:16:25 +0900 | [diff] [blame] | 265 | logf("run: %s", j.n.Output) |
Shinichiro Hamaji | cedc5c8 | 2015-05-13 17:03:20 +0900 | [diff] [blame] | 266 | |
Fumitoshi Ukai | 744bb2b | 2015-06-25 00:10:52 +0900 | [diff] [blame] | 267 | if wm.para != nil { |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 268 | var err error |
| 269 | j.runners, err = j.createRunners() |
| 270 | if err != nil { |
| 271 | return err |
| 272 | } |
Shinichiro Hamaji | a680842 | 2015-05-13 18:00:50 +0900 | [diff] [blame] | 273 | if len(j.runners) == 0 { |
| 274 | wm.updateParents(j) |
| 275 | wm.finishCnt++ |
| 276 | } else { |
| 277 | wm.runnings[j.n.Output] = j |
| 278 | wm.para.RunCommand(j.runners) |
| 279 | } |
Shinichiro Hamaji | cedc5c8 | 2015-05-13 17:03:20 +0900 | [diff] [blame] | 280 | } else { |
| 281 | j.numDeps = -1 // Do not let other workers pick this. |
| 282 | w := wm.freeWorkers[0] |
| 283 | wm.freeWorkers = wm.freeWorkers[1:] |
| 284 | wm.busyWorkers[w] = true |
| 285 | w.jobChan <- j |
| 286 | } |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 287 | } |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 288 | } |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 289 | |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 290 | func (wm *workerManager) updateParents(j *job) { |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 291 | for _, p := range j.parents { |
| 292 | p.numDeps-- |
Fumitoshi Ukai | 07cf121 | 2015-06-25 17:16:25 +0900 | [diff] [blame] | 293 | logf("child: %s (%d)", p.n.Output, p.numDeps) |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 294 | if p.depsTs < j.outputTs { |
| 295 | p.depsTs = j.outputTs |
| 296 | } |
Shinichiro Hamaji | dbc6c13 | 2015-04-28 18:26:36 +0900 | [diff] [blame] | 297 | wm.maybePushToReadyQueue(p) |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 298 | } |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 299 | } |
| 300 | |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 301 | type workerManager struct { |
Fumitoshi Ukai | 744bb2b | 2015-06-25 00:10:52 +0900 | [diff] [blame] | 302 | maxJobs int |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 303 | jobs []*job |
| 304 | readyQueue jobQueue |
| 305 | jobChan chan *job |
| 306 | resultChan chan jobResult |
| 307 | newDepChan chan newDep |
Fumitoshi Ukai | 531c5d2 | 2015-06-27 00:58:56 +0900 | [diff] [blame^] | 308 | stopChan chan bool |
Shinichiro Hamaji | 55c50bd | 2015-04-27 21:05:47 +0900 | [diff] [blame] | 309 | waitChan chan bool |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 310 | doneChan chan error |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 311 | freeWorkers []*worker |
| 312 | busyWorkers map[*worker]bool |
Shinichiro Hamaji | cedc5c8 | 2015-05-13 17:03:20 +0900 | [diff] [blame] | 313 | ex *Executor |
Fumitoshi Ukai | ff4e580 | 2015-06-25 13:12:26 +0900 | [diff] [blame] | 314 | para *paraWorker |
| 315 | paraChan chan *paraResult |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 316 | runnings map[string]*job |
Shinichiro Hamaji | dbc6c13 | 2015-04-28 18:26:36 +0900 | [diff] [blame] | 317 | |
| 318 | finishCnt int |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 319 | } |
| 320 | |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 321 | func newWorkerManager(numJobs int, paraPath string) (*workerManager, error) { |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 322 | wm := &workerManager{ |
Fumitoshi Ukai | 744bb2b | 2015-06-25 00:10:52 +0900 | [diff] [blame] | 323 | maxJobs: numJobs, |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 324 | jobChan: make(chan *job), |
| 325 | resultChan: make(chan jobResult), |
| 326 | newDepChan: make(chan newDep), |
Fumitoshi Ukai | 531c5d2 | 2015-06-27 00:58:56 +0900 | [diff] [blame^] | 327 | stopChan: make(chan bool), |
Shinichiro Hamaji | 55c50bd | 2015-04-27 21:05:47 +0900 | [diff] [blame] | 328 | waitChan: make(chan bool), |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 329 | doneChan: make(chan error), |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 330 | busyWorkers: make(map[*worker]bool), |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 331 | } |
Shinichiro Hamaji | cedc5c8 | 2015-05-13 17:03:20 +0900 | [diff] [blame] | 332 | |
Fumitoshi Ukai | 744bb2b | 2015-06-25 00:10:52 +0900 | [diff] [blame] | 333 | if paraPath != "" { |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 334 | wm.runnings = make(map[string]*job) |
Fumitoshi Ukai | ff4e580 | 2015-06-25 13:12:26 +0900 | [diff] [blame] | 335 | wm.paraChan = make(chan *paraResult) |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 336 | var err error |
| 337 | wm.para, err = newParaWorker(wm.paraChan, numJobs, paraPath) |
| 338 | if err != nil { |
| 339 | return nil, err |
| 340 | } |
Shinichiro Hamaji | cedc5c8 | 2015-05-13 17:03:20 +0900 | [diff] [blame] | 341 | go wm.para.Run() |
| 342 | } else { |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 343 | wm.busyWorkers = make(map[*worker]bool) |
Fumitoshi Ukai | 744bb2b | 2015-06-25 00:10:52 +0900 | [diff] [blame] | 344 | for i := 0; i < numJobs; i++ { |
| 345 | w := newWorker(wm) |
Shinichiro Hamaji | cedc5c8 | 2015-05-13 17:03:20 +0900 | [diff] [blame] | 346 | wm.freeWorkers = append(wm.freeWorkers, w) |
| 347 | go w.Run() |
| 348 | } |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 349 | } |
Shinichiro Hamaji | cedc5c8 | 2015-05-13 17:03:20 +0900 | [diff] [blame] | 350 | heap.Init(&wm.readyQueue) |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 351 | go wm.Run() |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 352 | return wm, nil |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 353 | } |
| 354 | |
| 355 | func exitStatus(err error) int { |
| 356 | if err == nil { |
| 357 | return 0 |
| 358 | } |
| 359 | exit := 1 |
| 360 | if err, ok := err.(*exec.ExitError); ok { |
| 361 | if w, ok := err.ProcessState.Sys().(syscall.WaitStatus); ok { |
| 362 | return w.ExitStatus() |
| 363 | } |
| 364 | } |
| 365 | return exit |
| 366 | } |
| 367 | |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 368 | func (wm *workerManager) hasTodo() bool { |
Shinichiro Hamaji | dbc6c13 | 2015-04-28 18:26:36 +0900 | [diff] [blame] | 369 | return wm.finishCnt != len(wm.jobs) |
| 370 | } |
| 371 | |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 372 | func (wm *workerManager) maybePushToReadyQueue(j *job) { |
Shinichiro Hamaji | dbc6c13 | 2015-04-28 18:26:36 +0900 | [diff] [blame] | 373 | if j.numDeps != 0 { |
| 374 | return |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 375 | } |
Shinichiro Hamaji | dbc6c13 | 2015-04-28 18:26:36 +0900 | [diff] [blame] | 376 | heap.Push(&wm.readyQueue, j) |
Fumitoshi Ukai | 07cf121 | 2015-06-25 17:16:25 +0900 | [diff] [blame] | 377 | logf("ready: %s", j.n.Output) |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 378 | } |
| 379 | |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 380 | func (wm *workerManager) handleNewDep(j *job, neededBy *job) { |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 381 | if j.numDeps < 0 { |
| 382 | neededBy.numDeps-- |
Shinichiro Hamaji | 5180c97 | 2015-04-28 20:14:38 +0900 | [diff] [blame] | 383 | if neededBy.id > 0 { |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 384 | panic("FIXME: already in WM... can this happen?") |
Shinichiro Hamaji | 5180c97 | 2015-04-28 20:14:38 +0900 | [diff] [blame] | 385 | } |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 386 | } else { |
| 387 | j.parents = append(j.parents, neededBy) |
| 388 | } |
| 389 | } |
| 390 | |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 391 | func (wm *workerManager) Run() { |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 392 | done := false |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 393 | var err error |
| 394 | Loop: |
Shinichiro Hamaji | cedc5c8 | 2015-05-13 17:03:20 +0900 | [diff] [blame] | 395 | for wm.hasTodo() || len(wm.busyWorkers) > 0 || len(wm.runnings) > 0 || !done { |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 396 | select { |
| 397 | case j := <-wm.jobChan: |
Fumitoshi Ukai | 07cf121 | 2015-06-25 17:16:25 +0900 | [diff] [blame] | 398 | logf("wait: %s (%d)", j.n.Output, j.numDeps) |
Shinichiro Hamaji | 5180c97 | 2015-04-28 20:14:38 +0900 | [diff] [blame] | 399 | j.id = len(wm.jobs) + 1 |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 400 | wm.jobs = append(wm.jobs, j) |
Shinichiro Hamaji | dbc6c13 | 2015-04-28 18:26:36 +0900 | [diff] [blame] | 401 | wm.maybePushToReadyQueue(j) |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 402 | case jr := <-wm.resultChan: |
Fumitoshi Ukai | 07cf121 | 2015-06-25 17:16:25 +0900 | [diff] [blame] | 403 | logf("done: %s", jr.j.n.Output) |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 404 | delete(wm.busyWorkers, jr.w) |
| 405 | wm.freeWorkers = append(wm.freeWorkers, jr.w) |
| 406 | wm.updateParents(jr.j) |
Shinichiro Hamaji | dbc6c13 | 2015-04-28 18:26:36 +0900 | [diff] [blame] | 407 | wm.finishCnt++ |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 408 | if jr.err != nil { |
| 409 | err = jr.err |
Fumitoshi Ukai | 531c5d2 | 2015-06-27 00:58:56 +0900 | [diff] [blame^] | 410 | close(wm.stopChan) |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 411 | break Loop |
| 412 | } |
Shinichiro Hamaji | 55c50bd | 2015-04-27 21:05:47 +0900 | [diff] [blame] | 413 | case af := <-wm.newDepChan: |
| 414 | wm.handleNewDep(af.j, af.neededBy) |
Fumitoshi Ukai | 07cf121 | 2015-06-25 17:16:25 +0900 | [diff] [blame] | 415 | logf("dep: %s (%d) %s", af.neededBy.n.Output, af.neededBy.numDeps, af.j.n.Output) |
Shinichiro Hamaji | cedc5c8 | 2015-05-13 17:03:20 +0900 | [diff] [blame] | 416 | case pr := <-wm.paraChan: |
Shinichiro Hamaji | a680842 | 2015-05-13 18:00:50 +0900 | [diff] [blame] | 417 | if pr.status < 0 && pr.signal < 0 { |
| 418 | j := wm.runnings[pr.output] |
| 419 | for _, r := range j.runners { |
Fumitoshi Ukai | 744bb2b | 2015-06-25 00:10:52 +0900 | [diff] [blame] | 420 | if r.echo || DryRunFlag { |
Shinichiro Hamaji | a680842 | 2015-05-13 18:00:50 +0900 | [diff] [blame] | 421 | fmt.Printf("%s\n", r.cmd) |
| 422 | } |
| 423 | } |
| 424 | } else { |
Fumitoshi Ukai | 145598a | 2015-06-19 10:08:17 +0900 | [diff] [blame] | 425 | fmt.Fprint(os.Stdout, pr.stdout) |
| 426 | fmt.Fprint(os.Stderr, pr.stderr) |
Shinichiro Hamaji | a680842 | 2015-05-13 18:00:50 +0900 | [diff] [blame] | 427 | j := wm.runnings[pr.output] |
| 428 | wm.updateParents(j) |
| 429 | delete(wm.runnings, pr.output) |
| 430 | wm.finishCnt++ |
| 431 | } |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 432 | case done = <-wm.waitChan: |
| 433 | } |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 434 | err = wm.handleJobs() |
| 435 | if err != nil { |
| 436 | break Loop |
| 437 | } |
Shinichiro Hamaji | cedc5c8 | 2015-05-13 17:03:20 +0900 | [diff] [blame] | 438 | |
Fumitoshi Ukai | 744bb2b | 2015-06-25 00:10:52 +0900 | [diff] [blame] | 439 | if wm.para != nil { |
Shinichiro Hamaji | cedc5c8 | 2015-05-13 17:03:20 +0900 | [diff] [blame] | 440 | numBusy := len(wm.runnings) |
Fumitoshi Ukai | 744bb2b | 2015-06-25 00:10:52 +0900 | [diff] [blame] | 441 | if numBusy > wm.maxJobs { |
| 442 | numBusy = wm.maxJobs |
Shinichiro Hamaji | cedc5c8 | 2015-05-13 17:03:20 +0900 | [diff] [blame] | 443 | } |
Fumitoshi Ukai | 07cf121 | 2015-06-25 17:16:25 +0900 | [diff] [blame] | 444 | logf("job=%d ready=%d free=%d busy=%d", len(wm.jobs)-wm.finishCnt, wm.readyQueue.Len(), wm.maxJobs-numBusy, numBusy) |
Shinichiro Hamaji | cedc5c8 | 2015-05-13 17:03:20 +0900 | [diff] [blame] | 445 | } else { |
Fumitoshi Ukai | 07cf121 | 2015-06-25 17:16:25 +0900 | [diff] [blame] | 446 | logf("job=%d ready=%d free=%d busy=%d", len(wm.jobs)-wm.finishCnt, wm.readyQueue.Len(), len(wm.freeWorkers), len(wm.busyWorkers)) |
Shinichiro Hamaji | cedc5c8 | 2015-05-13 17:03:20 +0900 | [diff] [blame] | 447 | } |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 448 | } |
Fumitoshi Ukai | 531c5d2 | 2015-06-27 00:58:56 +0900 | [diff] [blame^] | 449 | if !done { |
| 450 | <-wm.waitChan |
| 451 | } |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 452 | |
Fumitoshi Ukai | 744bb2b | 2015-06-25 00:10:52 +0900 | [diff] [blame] | 453 | if wm.para != nil { |
Fumitoshi Ukai | 07cf121 | 2015-06-25 17:16:25 +0900 | [diff] [blame] | 454 | logf("Wait for para to finish") |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 455 | err := wm.para.Wait() |
| 456 | if err != nil { |
| 457 | logf("para failed: %v", err) |
| 458 | } |
Shinichiro Hamaji | cedc5c8 | 2015-05-13 17:03:20 +0900 | [diff] [blame] | 459 | } else { |
| 460 | for _, w := range wm.freeWorkers { |
| 461 | w.Wait() |
| 462 | } |
| 463 | for w := range wm.busyWorkers { |
| 464 | w.Wait() |
| 465 | } |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 466 | } |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 467 | wm.doneChan <- err |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 468 | } |
| 469 | |
Fumitoshi Ukai | 531c5d2 | 2015-06-27 00:58:56 +0900 | [diff] [blame^] | 470 | func (wm *workerManager) PostJob(j *job) error { |
| 471 | select { |
| 472 | case wm.jobChan <- j: |
| 473 | return nil |
| 474 | case <-wm.stopChan: |
| 475 | return errors.New("worker manager stopped") |
| 476 | } |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 477 | } |
| 478 | |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 479 | func (wm *workerManager) ReportResult(w *worker, j *job, err error) { |
Fumitoshi Ukai | 531c5d2 | 2015-06-27 00:58:56 +0900 | [diff] [blame^] | 480 | select { |
| 481 | case wm.resultChan <- jobResult{w: w, j: j, err: err}: |
| 482 | case <-wm.stopChan: |
| 483 | } |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 484 | } |
| 485 | |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 486 | func (wm *workerManager) ReportNewDep(j *job, neededBy *job) { |
Fumitoshi Ukai | 531c5d2 | 2015-06-27 00:58:56 +0900 | [diff] [blame^] | 487 | select { |
| 488 | case wm.newDepChan <- newDep{j: j, neededBy: neededBy}: |
| 489 | case <-wm.stopChan: |
| 490 | } |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 491 | } |
| 492 | |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 493 | func (wm *workerManager) Wait() error { |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 494 | wm.waitChan <- true |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 495 | return <-wm.doneChan |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 496 | } |