Shinichiro Hamaji | b69bf8a | 2015-06-10 14:52:06 +0900 | [diff] [blame] | 1 | // Copyright 2015 Google Inc. All rights reserved |
| 2 | // |
| 3 | // Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | // you may not use this file except in compliance with the License. |
| 5 | // You may obtain a copy of the License at |
| 6 | // |
| 7 | // http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | // |
| 9 | // Unless required by applicable law or agreed to in writing, software |
| 10 | // distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | // See the License for the specific language governing permissions and |
| 13 | // limitations under the License. |
| 14 | |
Fumitoshi Ukai | 744bb2b | 2015-06-25 00:10:52 +0900 | [diff] [blame] | 15 | package kati |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 16 | |
| 17 | import ( |
Shinichiro Hamaji | dbc6c13 | 2015-04-28 18:26:36 +0900 | [diff] [blame] | 18 | "container/heap" |
Fumitoshi Ukai | 531c5d2 | 2015-06-27 00:58:56 +0900 | [diff] [blame] | 19 | "errors" |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 20 | "fmt" |
Shinichiro Hamaji | cedc5c8 | 2015-05-13 17:03:20 +0900 | [diff] [blame] | 21 | "os" |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 22 | "os/exec" |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 23 | "syscall" |
| 24 | "time" |
Fumitoshi Ukai | 6450d0f | 2015-07-10 16:34:06 +0900 | [diff] [blame] | 25 | |
| 26 | "github.com/golang/glog" |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 27 | ) |
| 28 | |
Fumitoshi Ukai | a43b96f | 2015-07-17 19:54:21 +0900 | [diff] [blame] | 29 | var ( |
| 30 | errNothingDone = errors.New("nothing done") |
| 31 | ) |
| 32 | |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 33 | type job struct { |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 34 | n *DepNode |
| 35 | ex *Executor |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 36 | parents []*job |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 37 | outputTs int64 |
| 38 | numDeps int |
| 39 | depsTs int64 |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 40 | id int |
Shinichiro Hamaji | a680842 | 2015-05-13 18:00:50 +0900 | [diff] [blame] | 41 | |
| 42 | runners []runner |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 43 | } |
| 44 | |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 45 | type jobResult struct { |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 46 | j *job |
| 47 | w *worker |
| 48 | err error |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 49 | } |
| 50 | |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 51 | type newDep struct { |
| 52 | j *job |
| 53 | neededBy *job |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 54 | } |
| 55 | |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 56 | type worker struct { |
| 57 | wm *workerManager |
| 58 | jobChan chan *job |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 59 | waitChan chan bool |
| 60 | doneChan chan bool |
| 61 | } |
| 62 | |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 63 | type jobQueue []*job |
Shinichiro Hamaji | dbc6c13 | 2015-04-28 18:26:36 +0900 | [diff] [blame] | 64 | |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 65 | func (jq jobQueue) Len() int { return len(jq) } |
| 66 | func (jq jobQueue) Swap(i, j int) { jq[i], jq[j] = jq[j], jq[i] } |
Shinichiro Hamaji | dbc6c13 | 2015-04-28 18:26:36 +0900 | [diff] [blame] | 67 | |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 68 | func (jq jobQueue) Less(i, j int) bool { |
Shinichiro Hamaji | dbc6c13 | 2015-04-28 18:26:36 +0900 | [diff] [blame] | 69 | // First come, first serve, for GNU make compatibility. |
| 70 | return jq[i].id < jq[j].id |
| 71 | } |
| 72 | |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 73 | func (jq *jobQueue) Push(x interface{}) { |
| 74 | item := x.(*job) |
Shinichiro Hamaji | dbc6c13 | 2015-04-28 18:26:36 +0900 | [diff] [blame] | 75 | *jq = append(*jq, item) |
| 76 | } |
| 77 | |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 78 | func (jq *jobQueue) Pop() interface{} { |
Shinichiro Hamaji | dbc6c13 | 2015-04-28 18:26:36 +0900 | [diff] [blame] | 79 | old := *jq |
| 80 | n := len(old) |
| 81 | item := old[n-1] |
| 82 | *jq = old[0 : n-1] |
| 83 | return item |
| 84 | } |
| 85 | |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 86 | func newWorker(wm *workerManager) *worker { |
| 87 | w := &worker{ |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 88 | wm: wm, |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 89 | jobChan: make(chan *job), |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 90 | waitChan: make(chan bool), |
| 91 | doneChan: make(chan bool), |
| 92 | } |
| 93 | return w |
| 94 | } |
| 95 | |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 96 | func (w *worker) Run() { |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 97 | done := false |
| 98 | for !done { |
| 99 | select { |
| 100 | case j := <-w.jobChan: |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 101 | err := j.build() |
| 102 | w.wm.ReportResult(w, j, err) |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 103 | case done = <-w.waitChan: |
| 104 | } |
| 105 | } |
| 106 | w.doneChan <- true |
| 107 | } |
| 108 | |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 109 | func (w *worker) PostJob(j *job) { |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 110 | w.jobChan <- j |
| 111 | } |
| 112 | |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 113 | func (w *worker) Wait() { |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 114 | w.waitChan <- true |
| 115 | <-w.doneChan |
| 116 | } |
| 117 | |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 118 | func (j *job) createRunners() ([]runner, error) { |
Fumitoshi Ukai | a70b4ea | 2015-06-30 15:31:49 +0900 | [diff] [blame] | 119 | runners, _, err := createRunners(j.ex.ctx, j.n) |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 120 | return runners, err |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 121 | } |
| 122 | |
Shinichiro Hamaji | 71fae4c | 2015-05-25 17:48:34 +0900 | [diff] [blame] | 123 | // TODO(ukai): use time.Time? |
| 124 | func getTimestamp(filename string) int64 { |
| 125 | st, err := os.Stat(filename) |
| 126 | if err != nil { |
| 127 | return -2 |
| 128 | } |
| 129 | return st.ModTime().Unix() |
| 130 | } |
| 131 | |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 132 | func (j *job) build() error { |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 133 | if j.n.IsPhony { |
| 134 | j.outputTs = -2 // trigger cmd even if all inputs don't exist. |
| 135 | } else { |
| 136 | j.outputTs = getTimestamp(j.n.Output) |
| 137 | } |
| 138 | |
| 139 | if !j.n.HasRule { |
| 140 | if j.outputTs >= 0 || j.n.IsPhony { |
Fumitoshi Ukai | a43b96f | 2015-07-17 19:54:21 +0900 | [diff] [blame] | 141 | return errNothingDone |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 142 | } |
| 143 | if len(j.parents) == 0 { |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 144 | return fmt.Errorf("*** No rule to make target %q.", j.n.Output) |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 145 | } |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 146 | return fmt.Errorf("*** No rule to make target %q, needed by %q.", j.n.Output, j.parents[0].n.Output) |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 147 | } |
| 148 | |
| 149 | if j.outputTs >= j.depsTs { |
| 150 | // TODO: stats. |
Fumitoshi Ukai | a43b96f | 2015-07-17 19:54:21 +0900 | [diff] [blame] | 151 | return errNothingDone |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 152 | } |
| 153 | |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 154 | rr, err := j.createRunners() |
| 155 | if err != nil { |
| 156 | return err |
| 157 | } |
| 158 | for _, r := range rr { |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 159 | err := r.run(j.n.Output) |
Fumitoshi Ukai | 60f92c0 | 2015-07-29 10:09:36 +0900 | [diff] [blame] | 160 | glog.Warningf("cmd result for %q: %v", j.n.Output, err) |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 161 | if err != nil { |
| 162 | exit := exitStatus(err) |
Fumitoshi Ukai | 08c1e94 | 2015-07-15 16:38:07 +0900 | [diff] [blame] | 163 | return fmt.Errorf("*** [%s] Error %d", j.n.Output, exit) |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 164 | } |
| 165 | } |
| 166 | |
| 167 | if j.n.IsPhony { |
| 168 | j.outputTs = time.Now().Unix() |
| 169 | } else { |
| 170 | j.outputTs = getTimestamp(j.n.Output) |
| 171 | if j.outputTs < 0 { |
| 172 | j.outputTs = time.Now().Unix() |
| 173 | } |
| 174 | } |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 175 | return nil |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 176 | } |
| 177 | |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 178 | func (wm *workerManager) handleJobs() error { |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 179 | for { |
Shinichiro Hamaji | 18287f0 | 2015-07-06 14:48:48 +0900 | [diff] [blame] | 180 | if len(wm.freeWorkers) == 0 { |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 181 | return nil |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 182 | } |
Shinichiro Hamaji | dbc6c13 | 2015-04-28 18:26:36 +0900 | [diff] [blame] | 183 | if wm.readyQueue.Len() == 0 { |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 184 | return nil |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 185 | } |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 186 | j := heap.Pop(&wm.readyQueue).(*job) |
Fumitoshi Ukai | 6450d0f | 2015-07-10 16:34:06 +0900 | [diff] [blame] | 187 | glog.V(1).Infof("run: %s", j.n.Output) |
Shinichiro Hamaji | cedc5c8 | 2015-05-13 17:03:20 +0900 | [diff] [blame] | 188 | |
Shinichiro Hamaji | 18287f0 | 2015-07-06 14:48:48 +0900 | [diff] [blame] | 189 | j.numDeps = -1 // Do not let other workers pick this. |
| 190 | w := wm.freeWorkers[0] |
| 191 | wm.freeWorkers = wm.freeWorkers[1:] |
| 192 | wm.busyWorkers[w] = true |
| 193 | w.jobChan <- j |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 194 | } |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 195 | } |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 196 | |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 197 | func (wm *workerManager) updateParents(j *job) { |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 198 | for _, p := range j.parents { |
| 199 | p.numDeps-- |
Fumitoshi Ukai | 6450d0f | 2015-07-10 16:34:06 +0900 | [diff] [blame] | 200 | glog.V(1).Infof("child: %s (%d)", p.n.Output, p.numDeps) |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 201 | if p.depsTs < j.outputTs { |
| 202 | p.depsTs = j.outputTs |
| 203 | } |
Shinichiro Hamaji | dbc6c13 | 2015-04-28 18:26:36 +0900 | [diff] [blame] | 204 | wm.maybePushToReadyQueue(p) |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 205 | } |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 206 | } |
| 207 | |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 208 | type workerManager struct { |
Fumitoshi Ukai | 744bb2b | 2015-06-25 00:10:52 +0900 | [diff] [blame] | 209 | maxJobs int |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 210 | jobs []*job |
| 211 | readyQueue jobQueue |
| 212 | jobChan chan *job |
| 213 | resultChan chan jobResult |
| 214 | newDepChan chan newDep |
Fumitoshi Ukai | 531c5d2 | 2015-06-27 00:58:56 +0900 | [diff] [blame] | 215 | stopChan chan bool |
Shinichiro Hamaji | 55c50bd | 2015-04-27 21:05:47 +0900 | [diff] [blame] | 216 | waitChan chan bool |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 217 | doneChan chan error |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 218 | freeWorkers []*worker |
| 219 | busyWorkers map[*worker]bool |
Shinichiro Hamaji | cedc5c8 | 2015-05-13 17:03:20 +0900 | [diff] [blame] | 220 | ex *Executor |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 221 | runnings map[string]*job |
Shinichiro Hamaji | dbc6c13 | 2015-04-28 18:26:36 +0900 | [diff] [blame] | 222 | |
| 223 | finishCnt int |
Fumitoshi Ukai | a43b96f | 2015-07-17 19:54:21 +0900 | [diff] [blame] | 224 | skipCnt int |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 225 | } |
| 226 | |
Shinichiro Hamaji | 18287f0 | 2015-07-06 14:48:48 +0900 | [diff] [blame] | 227 | func newWorkerManager(numJobs int) (*workerManager, error) { |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 228 | wm := &workerManager{ |
Fumitoshi Ukai | 744bb2b | 2015-06-25 00:10:52 +0900 | [diff] [blame] | 229 | maxJobs: numJobs, |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 230 | jobChan: make(chan *job), |
| 231 | resultChan: make(chan jobResult), |
| 232 | newDepChan: make(chan newDep), |
Fumitoshi Ukai | 531c5d2 | 2015-06-27 00:58:56 +0900 | [diff] [blame] | 233 | stopChan: make(chan bool), |
Shinichiro Hamaji | 55c50bd | 2015-04-27 21:05:47 +0900 | [diff] [blame] | 234 | waitChan: make(chan bool), |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 235 | doneChan: make(chan error), |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 236 | busyWorkers: make(map[*worker]bool), |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 237 | } |
Shinichiro Hamaji | cedc5c8 | 2015-05-13 17:03:20 +0900 | [diff] [blame] | 238 | |
Shinichiro Hamaji | 18287f0 | 2015-07-06 14:48:48 +0900 | [diff] [blame] | 239 | wm.busyWorkers = make(map[*worker]bool) |
| 240 | for i := 0; i < numJobs; i++ { |
| 241 | w := newWorker(wm) |
| 242 | wm.freeWorkers = append(wm.freeWorkers, w) |
| 243 | go w.Run() |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 244 | } |
Shinichiro Hamaji | cedc5c8 | 2015-05-13 17:03:20 +0900 | [diff] [blame] | 245 | heap.Init(&wm.readyQueue) |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 246 | go wm.Run() |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 247 | return wm, nil |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 248 | } |
| 249 | |
| 250 | func exitStatus(err error) int { |
| 251 | if err == nil { |
| 252 | return 0 |
| 253 | } |
| 254 | exit := 1 |
| 255 | if err, ok := err.(*exec.ExitError); ok { |
| 256 | if w, ok := err.ProcessState.Sys().(syscall.WaitStatus); ok { |
| 257 | return w.ExitStatus() |
| 258 | } |
| 259 | } |
| 260 | return exit |
| 261 | } |
| 262 | |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 263 | func (wm *workerManager) hasTodo() bool { |
Shinichiro Hamaji | dbc6c13 | 2015-04-28 18:26:36 +0900 | [diff] [blame] | 264 | return wm.finishCnt != len(wm.jobs) |
| 265 | } |
| 266 | |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 267 | func (wm *workerManager) maybePushToReadyQueue(j *job) { |
Shinichiro Hamaji | dbc6c13 | 2015-04-28 18:26:36 +0900 | [diff] [blame] | 268 | if j.numDeps != 0 { |
| 269 | return |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 270 | } |
Shinichiro Hamaji | dbc6c13 | 2015-04-28 18:26:36 +0900 | [diff] [blame] | 271 | heap.Push(&wm.readyQueue, j) |
Fumitoshi Ukai | 6450d0f | 2015-07-10 16:34:06 +0900 | [diff] [blame] | 272 | glog.V(1).Infof("ready: %s", j.n.Output) |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 273 | } |
| 274 | |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 275 | func (wm *workerManager) handleNewDep(j *job, neededBy *job) { |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 276 | if j.numDeps < 0 { |
| 277 | neededBy.numDeps-- |
Shinichiro Hamaji | 5180c97 | 2015-04-28 20:14:38 +0900 | [diff] [blame] | 278 | if neededBy.id > 0 { |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 279 | panic("FIXME: already in WM... can this happen?") |
Shinichiro Hamaji | 5180c97 | 2015-04-28 20:14:38 +0900 | [diff] [blame] | 280 | } |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 281 | } else { |
| 282 | j.parents = append(j.parents, neededBy) |
| 283 | } |
| 284 | } |
| 285 | |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 286 | func (wm *workerManager) Run() { |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 287 | done := false |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 288 | var err error |
| 289 | Loop: |
Shinichiro Hamaji | cedc5c8 | 2015-05-13 17:03:20 +0900 | [diff] [blame] | 290 | for wm.hasTodo() || len(wm.busyWorkers) > 0 || len(wm.runnings) > 0 || !done { |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 291 | select { |
| 292 | case j := <-wm.jobChan: |
Fumitoshi Ukai | 6450d0f | 2015-07-10 16:34:06 +0900 | [diff] [blame] | 293 | glog.V(1).Infof("wait: %s (%d)", j.n.Output, j.numDeps) |
Shinichiro Hamaji | 5180c97 | 2015-04-28 20:14:38 +0900 | [diff] [blame] | 294 | j.id = len(wm.jobs) + 1 |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 295 | wm.jobs = append(wm.jobs, j) |
Shinichiro Hamaji | dbc6c13 | 2015-04-28 18:26:36 +0900 | [diff] [blame] | 296 | wm.maybePushToReadyQueue(j) |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 297 | case jr := <-wm.resultChan: |
Fumitoshi Ukai | 6450d0f | 2015-07-10 16:34:06 +0900 | [diff] [blame] | 298 | glog.V(1).Infof("done: %s", jr.j.n.Output) |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 299 | delete(wm.busyWorkers, jr.w) |
| 300 | wm.freeWorkers = append(wm.freeWorkers, jr.w) |
| 301 | wm.updateParents(jr.j) |
Shinichiro Hamaji | dbc6c13 | 2015-04-28 18:26:36 +0900 | [diff] [blame] | 302 | wm.finishCnt++ |
Fumitoshi Ukai | a43b96f | 2015-07-17 19:54:21 +0900 | [diff] [blame] | 303 | if jr.err == errNothingDone { |
| 304 | wm.skipCnt++ |
| 305 | jr.err = nil |
| 306 | } |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 307 | if jr.err != nil { |
| 308 | err = jr.err |
Fumitoshi Ukai | 531c5d2 | 2015-06-27 00:58:56 +0900 | [diff] [blame] | 309 | close(wm.stopChan) |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 310 | break Loop |
| 311 | } |
Shinichiro Hamaji | 55c50bd | 2015-04-27 21:05:47 +0900 | [diff] [blame] | 312 | case af := <-wm.newDepChan: |
| 313 | wm.handleNewDep(af.j, af.neededBy) |
Fumitoshi Ukai | 6450d0f | 2015-07-10 16:34:06 +0900 | [diff] [blame] | 314 | glog.V(1).Infof("dep: %s (%d) %s", af.neededBy.n.Output, af.neededBy.numDeps, af.j.n.Output) |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 315 | case done = <-wm.waitChan: |
| 316 | } |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 317 | err = wm.handleJobs() |
| 318 | if err != nil { |
| 319 | break Loop |
| 320 | } |
Shinichiro Hamaji | cedc5c8 | 2015-05-13 17:03:20 +0900 | [diff] [blame] | 321 | |
Fumitoshi Ukai | 6450d0f | 2015-07-10 16:34:06 +0900 | [diff] [blame] | 322 | glog.V(1).Infof("job=%d ready=%d free=%d busy=%d", len(wm.jobs)-wm.finishCnt, wm.readyQueue.Len(), len(wm.freeWorkers), len(wm.busyWorkers)) |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 323 | } |
Fumitoshi Ukai | 531c5d2 | 2015-06-27 00:58:56 +0900 | [diff] [blame] | 324 | if !done { |
| 325 | <-wm.waitChan |
| 326 | } |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 327 | |
Shinichiro Hamaji | 18287f0 | 2015-07-06 14:48:48 +0900 | [diff] [blame] | 328 | for _, w := range wm.freeWorkers { |
| 329 | w.Wait() |
| 330 | } |
| 331 | for w := range wm.busyWorkers { |
| 332 | w.Wait() |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 333 | } |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 334 | wm.doneChan <- err |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 335 | } |
| 336 | |
Fumitoshi Ukai | 531c5d2 | 2015-06-27 00:58:56 +0900 | [diff] [blame] | 337 | func (wm *workerManager) PostJob(j *job) error { |
| 338 | select { |
| 339 | case wm.jobChan <- j: |
| 340 | return nil |
| 341 | case <-wm.stopChan: |
| 342 | return errors.New("worker manager stopped") |
| 343 | } |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 344 | } |
| 345 | |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 346 | func (wm *workerManager) ReportResult(w *worker, j *job, err error) { |
Fumitoshi Ukai | 531c5d2 | 2015-06-27 00:58:56 +0900 | [diff] [blame] | 347 | select { |
| 348 | case wm.resultChan <- jobResult{w: w, j: j, err: err}: |
| 349 | case <-wm.stopChan: |
| 350 | } |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 351 | } |
| 352 | |
Fumitoshi Ukai | dfb518b | 2015-06-25 13:19:55 +0900 | [diff] [blame] | 353 | func (wm *workerManager) ReportNewDep(j *job, neededBy *job) { |
Fumitoshi Ukai | 531c5d2 | 2015-06-27 00:58:56 +0900 | [diff] [blame] | 354 | select { |
| 355 | case wm.newDepChan <- newDep{j: j, neededBy: neededBy}: |
| 356 | case <-wm.stopChan: |
| 357 | } |
Shinichiro Hamaji | 69bb7e4 | 2015-04-27 17:54:42 +0900 | [diff] [blame] | 358 | } |
| 359 | |
Fumitoshi Ukai | a43b96f | 2015-07-17 19:54:21 +0900 | [diff] [blame] | 360 | func (wm *workerManager) Wait() (int, error) { |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 361 | wm.waitChan <- true |
Fumitoshi Ukai | a43b96f | 2015-07-17 19:54:21 +0900 | [diff] [blame] | 362 | err := <-wm.doneChan |
| 363 | return wm.finishCnt - wm.skipCnt, err |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 364 | } |