Eric Li | 6f27d4f | 2010-09-29 10:55:17 -0700 | [diff] [blame] | 1 | """ |
| 2 | Module with abstraction layers to revision control systems. |
| 3 | |
| 4 | With this library, autotest developers can handle source code checkouts and |
| 5 | updates on both client as well as server code. |
| 6 | """ |
| 7 | |
| 8 | import os, warnings, logging |
| 9 | import error, utils |
| 10 | from autotest_lib.client.bin import os_dep |
| 11 | |
| 12 | |
beeps | d9153b5 | 2013-01-23 20:52:46 -0800 | [diff] [blame] | 13 | class RevisionControlError(Exception): |
| 14 | """Local exception to be raised by code in this file.""" |
| 15 | |
| 16 | |
| 17 | class GitError(RevisionControlError): |
| 18 | """Exceptions raised for general git errors.""" |
| 19 | |
| 20 | |
| 21 | class GitCloneError(GitError): |
| 22 | """Exceptions raised for git clone errors.""" |
| 23 | |
| 24 | |
| 25 | class GitFetchError(GitError): |
| 26 | """Exception raised for git fetch errors.""" |
| 27 | |
| 28 | |
| 29 | class GitPullError(GitError): |
| 30 | """Exception raised for git pull errors.""" |
| 31 | |
| 32 | |
| 33 | class GitResetError(GitError): |
| 34 | """Exception raised for git reset errors.""" |
| 35 | |
| 36 | |
| 37 | class GitCommitError(GitError): |
| 38 | """Exception raised for git commit errors.""" |
| 39 | |
| 40 | |
Eric Li | 6f27d4f | 2010-09-29 10:55:17 -0700 | [diff] [blame] | 41 | class GitRepo(object): |
| 42 | """ |
| 43 | This class represents a git repo. |
| 44 | |
| 45 | It is used to pull down a local copy of a git repo, check if the local |
| 46 | repo is up-to-date, if not update. It delegates the install to |
| 47 | implementation classes. |
| 48 | """ |
| 49 | |
beeps | 98365d8 | 2013-02-20 20:08:07 -0800 | [diff] [blame^] | 50 | def __init__(self, repodir, giturl=None, weburl=None, abs_work_tree=None): |
beeps | d9153b5 | 2013-01-23 20:52:46 -0800 | [diff] [blame] | 51 | """ |
| 52 | Initialized reposotory. |
| 53 | |
| 54 | @param repodir: destination repo directory. |
| 55 | @param giturl: master repo git url. |
| 56 | @param weburl: a web url for the master repo. |
| 57 | @param abs_work_tree: work tree of the git repo. In the |
| 58 | absence of a work tree git manipulations will occur |
| 59 | in the current working directory for non bare repos. |
| 60 | In such repos the -git-dir option should point to |
| 61 | the .git directory and -work-tree should point to |
| 62 | the repos working tree. |
| 63 | Note: a bare reposotory is one which contains all the |
| 64 | working files (the tree) and the other wise hidden files |
| 65 | (.git) in the same directory. This class assumes non-bare |
| 66 | reposotories. |
| 67 | """ |
Eric Li | 6f27d4f | 2010-09-29 10:55:17 -0700 | [diff] [blame] | 68 | if repodir is None: |
| 69 | raise ValueError('You must provide a path that will hold the' |
| 70 | 'git repository') |
| 71 | self.repodir = utils.sh_escape(repodir) |
beeps | 98365d8 | 2013-02-20 20:08:07 -0800 | [diff] [blame^] | 72 | self._giturl = giturl |
Eric Li | 6f27d4f | 2010-09-29 10:55:17 -0700 | [diff] [blame] | 73 | if weburl is not None: |
| 74 | warnings.warn("Param weburl: You are no longer required to provide " |
| 75 | "a web URL for your git repos", DeprecationWarning) |
| 76 | |
| 77 | # path to .git dir |
| 78 | self.gitpath = utils.sh_escape(os.path.join(self.repodir,'.git')) |
| 79 | |
| 80 | # Find git base command. If not found, this will throw an exception |
beeps | d9153b5 | 2013-01-23 20:52:46 -0800 | [diff] [blame] | 81 | self.git_base_cmd = os_dep.command('git') |
beeps | d9153b5 | 2013-01-23 20:52:46 -0800 | [diff] [blame] | 82 | self.work_tree = abs_work_tree |
Eric Li | 6f27d4f | 2010-09-29 10:55:17 -0700 | [diff] [blame] | 83 | |
| 84 | # default to same remote path as local |
| 85 | self._build = os.path.dirname(self.repodir) |
| 86 | |
| 87 | |
beeps | 98365d8 | 2013-02-20 20:08:07 -0800 | [diff] [blame^] | 88 | @property |
| 89 | def giturl(self): |
| 90 | """ |
| 91 | A giturl is necessary to perform certain actions (clone, pull, fetch) |
| 92 | but not others (like diff). |
| 93 | """ |
| 94 | if self._giturl is None: |
| 95 | raise ValueError('Unsupported operation -- this object was not' |
| 96 | 'constructed with a git URL.') |
| 97 | return self._giturl |
| 98 | |
| 99 | |
beeps | d9153b5 | 2013-01-23 20:52:46 -0800 | [diff] [blame] | 100 | def gen_git_cmd_base(self): |
| 101 | """ |
| 102 | The command we use to run git cannot be set. It is reconstructed |
| 103 | on each access from it's component variables. This is it's getter. |
| 104 | """ |
| 105 | # base git command , pointing to gitpath git dir |
| 106 | gitcmdbase = '%s --git-dir=%s' % (self.git_base_cmd, |
| 107 | self.gitpath) |
| 108 | if self.work_tree: |
| 109 | gitcmdbase += ' --work-tree=%s' % self.work_tree |
| 110 | return gitcmdbase |
| 111 | |
| 112 | |
Eric Li | 6f27d4f | 2010-09-29 10:55:17 -0700 | [diff] [blame] | 113 | def _run(self, command, timeout=None, ignore_status=False): |
| 114 | """ |
| 115 | Auxiliary function to run a command, with proper shell escaping. |
| 116 | |
| 117 | @param timeout: Timeout to run the command. |
| 118 | @param ignore_status: Whether we should supress error.CmdError |
| 119 | exceptions if the command did return exit code !=0 (True), or |
| 120 | not supress them (False). |
| 121 | """ |
| 122 | return utils.run(r'%s' % (utils.sh_escape(command)), |
| 123 | timeout, ignore_status) |
| 124 | |
| 125 | |
| 126 | def gitcmd(self, cmd, ignore_status=False): |
| 127 | """ |
| 128 | Wrapper for a git command. |
| 129 | |
| 130 | @param cmd: Git subcommand (ex 'clone'). |
| 131 | @param ignore_status: Whether we should supress error.CmdError |
| 132 | exceptions if the command did return exit code !=0 (True), or |
| 133 | not supress them (False). |
| 134 | """ |
beeps | d9153b5 | 2013-01-23 20:52:46 -0800 | [diff] [blame] | 135 | cmd = '%s %s' % (self.gen_git_cmd_base(), cmd) |
Eric Li | 6f27d4f | 2010-09-29 10:55:17 -0700 | [diff] [blame] | 136 | return self._run(cmd, ignore_status=ignore_status) |
| 137 | |
| 138 | |
beeps | d9153b5 | 2013-01-23 20:52:46 -0800 | [diff] [blame] | 139 | def clone(self): |
| 140 | """ |
| 141 | Clones a repo using giturl and repodir. |
| 142 | |
| 143 | Since we're cloning the master repo we don't have a work tree yet, |
| 144 | make sure the getter of the gitcmd doesn't think we do by setting |
| 145 | work_tree to None. |
| 146 | |
| 147 | @raises GitCloneError: if cloning the master repo fails. |
| 148 | """ |
| 149 | logging.info('Cloning git repo %s', self.giturl) |
| 150 | cmd = 'clone %s %s ' % (self.giturl, self.repodir) |
| 151 | abs_work_tree = self.work_tree |
| 152 | self.work_tree = None |
| 153 | try: |
| 154 | rv = self.gitcmd(cmd, True) |
| 155 | if rv.exit_status != 0: |
| 156 | logging.error(rv.stderr) |
| 157 | raise GitCloneError('Failed to clone git url', rv) |
| 158 | else: |
| 159 | logging.info(rv.stdout) |
| 160 | finally: |
| 161 | self.work_tree = abs_work_tree |
| 162 | |
| 163 | |
beeps | aae3f1c | 2013-03-19 15:49:14 -0700 | [diff] [blame] | 164 | def pull(self, rebase=False): |
beeps | d9153b5 | 2013-01-23 20:52:46 -0800 | [diff] [blame] | 165 | """ |
| 166 | Pulls into repodir using giturl. |
| 167 | |
beeps | aae3f1c | 2013-03-19 15:49:14 -0700 | [diff] [blame] | 168 | @param rebase: If true forces git pull to perform a rebase instead of a |
| 169 | merge. |
beeps | d9153b5 | 2013-01-23 20:52:46 -0800 | [diff] [blame] | 170 | @raises GitPullError: if pulling from giturl fails. |
| 171 | """ |
| 172 | logging.info('Updating git repo %s', self.giturl) |
beeps | aae3f1c | 2013-03-19 15:49:14 -0700 | [diff] [blame] | 173 | cmd = 'pull ' |
| 174 | if rebase: |
| 175 | cmd += '--rebase ' |
| 176 | cmd += self.giturl |
| 177 | |
beeps | d9153b5 | 2013-01-23 20:52:46 -0800 | [diff] [blame] | 178 | rv = self.gitcmd(cmd, True) |
| 179 | if rv.exit_status != 0: |
| 180 | logging.error(rv.stderr) |
| 181 | e_msg = 'Failed to pull git repo data' |
| 182 | raise GitPullError(e_msg, rv) |
| 183 | |
| 184 | |
| 185 | def commit(self, msg='default'): |
| 186 | """ |
| 187 | Commit changes to repo with the supplied commit msg. |
| 188 | |
| 189 | @param msg: A message that goes with the commit. |
| 190 | """ |
| 191 | rv = self.gitcmd('commit -a -m %s' % msg) |
| 192 | if rv.exit_status != 0: |
| 193 | logging.error(rv.stderr) |
| 194 | raise revision_control.GitCommitError('Unable to commit', rv) |
| 195 | |
| 196 | |
| 197 | def reset_head(self): |
| 198 | """ |
| 199 | Reset repo to HEAD@{0} by running git reset --hard HEAD. |
| 200 | |
| 201 | @raises GitResetError: if we fails to reset HEAD. |
| 202 | """ |
| 203 | logging.info('Resetting head on repo %s', self.repodir) |
| 204 | rv = self.gitcmd('reset --hard HEAD') |
| 205 | if rv.exit_status != 0: |
| 206 | logging.error(rv.stderr) |
| 207 | e_msg = 'Failed to reset HEAD' |
| 208 | raise GitResetError(e_msg, rv) |
| 209 | |
| 210 | |
| 211 | def fetch_remote(self): |
| 212 | """ |
| 213 | Fetches all files from the remote but doesn't reset head. |
| 214 | |
| 215 | @raises GitFetchError: if we fail to fetch all files from giturl. |
| 216 | """ |
| 217 | logging.info('fetching from repo %s', self.giturl) |
| 218 | rv = self.gitcmd('fetch --all') |
| 219 | if rv.exit_status != 0: |
| 220 | logging.error(rv.stderr) |
| 221 | e_msg = 'Failed to fetch from %s' % self.giturl |
| 222 | raise GitFetchError(e_msg, rv) |
| 223 | |
| 224 | |
beeps | aae3f1c | 2013-03-19 15:49:14 -0700 | [diff] [blame] | 225 | def pull_or_clone(self): |
beeps | d9153b5 | 2013-01-23 20:52:46 -0800 | [diff] [blame] | 226 | """ |
beeps | aae3f1c | 2013-03-19 15:49:14 -0700 | [diff] [blame] | 227 | Pulls if the repo is already initialized, clones if it isn't. |
beeps | d9153b5 | 2013-01-23 20:52:46 -0800 | [diff] [blame] | 228 | """ |
beeps | aae3f1c | 2013-03-19 15:49:14 -0700 | [diff] [blame] | 229 | # TODO beeps: if the user has local changes in the repo they're |
| 230 | # pulling into, this could fail on rebase. Currently the only consumer |
| 231 | # of this method is external_packages and it makes sense for |
| 232 | # build_externals to fail in such a scenario. Investigate ways to get |
| 233 | # this to squash local changes using git rev-parse to get the upstream |
| 234 | # tracking branch name, and then do a fetch + reset head. |
beeps | d9153b5 | 2013-01-23 20:52:46 -0800 | [diff] [blame] | 235 | if self.is_repo_initialized(): |
beeps | aae3f1c | 2013-03-19 15:49:14 -0700 | [diff] [blame] | 236 | self.pull(rebase=True) |
beeps | d9153b5 | 2013-01-23 20:52:46 -0800 | [diff] [blame] | 237 | else: |
| 238 | self.clone() |
| 239 | |
| 240 | |
Eric Li | 6f27d4f | 2010-09-29 10:55:17 -0700 | [diff] [blame] | 241 | def get(self, **kwargs): |
| 242 | """ |
| 243 | This method overrides baseclass get so we can do proper git |
| 244 | clone/pulls, and check for updated versions. The result of |
| 245 | this method will leave an up-to-date version of git repo at |
| 246 | 'giturl' in 'repodir' directory to be used by build/install |
| 247 | methods. |
| 248 | |
beeps | 98365d8 | 2013-02-20 20:08:07 -0800 | [diff] [blame^] | 249 | @param kwargs: Dictionary of parameters to the method get. |
Eric Li | 6f27d4f | 2010-09-29 10:55:17 -0700 | [diff] [blame] | 250 | """ |
| 251 | if not self.is_repo_initialized(): |
| 252 | # this is your first time ... |
beeps | d9153b5 | 2013-01-23 20:52:46 -0800 | [diff] [blame] | 253 | self.clone() |
| 254 | elif self.is_out_of_date(): |
Eric Li | 6f27d4f | 2010-09-29 10:55:17 -0700 | [diff] [blame] | 255 | # exiting repo, check if we're up-to-date |
beeps | d9153b5 | 2013-01-23 20:52:46 -0800 | [diff] [blame] | 256 | self.pull() |
| 257 | else: |
| 258 | logging.info('repo up-to-date') |
Eric Li | 6f27d4f | 2010-09-29 10:55:17 -0700 | [diff] [blame] | 259 | |
| 260 | # remember where the source is |
| 261 | self.source_material = self.repodir |
| 262 | |
| 263 | |
| 264 | def get_local_head(self): |
| 265 | """ |
| 266 | Get the top commit hash of the current local git branch. |
| 267 | |
| 268 | @return: Top commit hash of local git branch |
| 269 | """ |
| 270 | cmd = 'log --pretty=format:"%H" -1' |
| 271 | l_head_cmd = self.gitcmd(cmd) |
| 272 | return l_head_cmd.stdout.strip() |
| 273 | |
| 274 | |
| 275 | def get_remote_head(self): |
| 276 | """ |
| 277 | Get the top commit hash of the current remote git branch. |
| 278 | |
| 279 | @return: Top commit hash of remote git branch |
| 280 | """ |
| 281 | cmd1 = 'remote show' |
| 282 | origin_name_cmd = self.gitcmd(cmd1) |
| 283 | cmd2 = 'log --pretty=format:"%H" -1 ' + origin_name_cmd.stdout.strip() |
| 284 | r_head_cmd = self.gitcmd(cmd2) |
| 285 | return r_head_cmd.stdout.strip() |
| 286 | |
| 287 | |
| 288 | def is_out_of_date(self): |
| 289 | """ |
| 290 | Return whether this branch is out of date with regards to remote branch. |
| 291 | |
| 292 | @return: False, if the branch is outdated, True if it is current. |
| 293 | """ |
| 294 | local_head = self.get_local_head() |
| 295 | remote_head = self.get_remote_head() |
| 296 | |
| 297 | # local is out-of-date, pull |
| 298 | if local_head != remote_head: |
| 299 | return True |
| 300 | |
| 301 | return False |
| 302 | |
| 303 | |
| 304 | def is_repo_initialized(self): |
| 305 | """ |
beeps | d9153b5 | 2013-01-23 20:52:46 -0800 | [diff] [blame] | 306 | Return whether the git repo was already initialized. |
Eric Li | 6f27d4f | 2010-09-29 10:55:17 -0700 | [diff] [blame] | 307 | |
beeps | d9153b5 | 2013-01-23 20:52:46 -0800 | [diff] [blame] | 308 | Counts objects in .git directory, since these will exist even if the |
| 309 | repo is empty. Assumes non-bare reposotories like the rest of this file. |
| 310 | |
| 311 | @return: True if the repo is initialized. |
Eric Li | 6f27d4f | 2010-09-29 10:55:17 -0700 | [diff] [blame] | 312 | """ |
beeps | d9153b5 | 2013-01-23 20:52:46 -0800 | [diff] [blame] | 313 | cmd = 'count-objects' |
Eric Li | 6f27d4f | 2010-09-29 10:55:17 -0700 | [diff] [blame] | 314 | rv = self.gitcmd(cmd, True) |
| 315 | if rv.exit_status == 0: |
| 316 | return True |
| 317 | |
| 318 | return False |
| 319 | |
| 320 | |
beeps | d9153b5 | 2013-01-23 20:52:46 -0800 | [diff] [blame] | 321 | def get_latest_commit_hash(self): |
| 322 | """ |
| 323 | Get the commit hash of the latest commit in the repo. |
| 324 | |
| 325 | We don't raise an exception if no commit hash was found as |
| 326 | this could be an empty repository. The caller should notice this |
| 327 | methods return value and raise one appropriately. |
| 328 | |
| 329 | @return: The first commit hash if anything has been committed. |
| 330 | """ |
| 331 | cmd = 'rev-list -n 1 --all' |
| 332 | rv = self.gitcmd(cmd, True) |
| 333 | if rv.exit_status == 0: |
| 334 | return rv.stdout |
| 335 | return None |
| 336 | |
| 337 | |
| 338 | def is_repo_empty(self): |
| 339 | """ |
| 340 | Checks for empty but initialized repos. |
| 341 | |
| 342 | eg: we clone an empty master repo, then don't pull |
| 343 | after the master commits. |
| 344 | |
| 345 | @return True if the repo has no commits. |
| 346 | """ |
| 347 | if self.get_latest_commit_hash(): |
| 348 | return False |
| 349 | return True |
| 350 | |
| 351 | |
Eric Li | 6f27d4f | 2010-09-29 10:55:17 -0700 | [diff] [blame] | 352 | def get_revision(self): |
| 353 | """ |
| 354 | Return current HEAD commit id |
| 355 | """ |
| 356 | if not self.is_repo_initialized(): |
| 357 | self.get() |
| 358 | |
| 359 | cmd = 'rev-parse --verify HEAD' |
| 360 | gitlog = self.gitcmd(cmd, True) |
| 361 | if gitlog.exit_status != 0: |
| 362 | logging.error(gitlog.stderr) |
| 363 | raise error.CmdError('Failed to find git sha1 revision', gitlog) |
| 364 | else: |
| 365 | return gitlog.stdout.strip('\n') |
| 366 | |
| 367 | |
| 368 | def checkout(self, remote, local=None): |
| 369 | """ |
| 370 | Check out the git commit id, branch, or tag given by remote. |
| 371 | |
| 372 | Optional give the local branch name as local. |
| 373 | |
| 374 | @param remote: Remote commit hash |
| 375 | @param local: Local commit hash |
| 376 | @note: For git checkout tag git version >= 1.5.0 is required |
| 377 | """ |
| 378 | if not self.is_repo_initialized(): |
| 379 | self.get() |
| 380 | |
| 381 | assert(isinstance(remote, basestring)) |
| 382 | if local: |
| 383 | cmd = 'checkout -b %s %s' % (local, remote) |
| 384 | else: |
| 385 | cmd = 'checkout %s' % (remote) |
| 386 | gitlog = self.gitcmd(cmd, True) |
| 387 | if gitlog.exit_status != 0: |
| 388 | logging.error(gitlog.stderr) |
| 389 | raise error.CmdError('Failed to checkout git branch', gitlog) |
| 390 | else: |
| 391 | logging.info(gitlog.stdout) |
| 392 | |
| 393 | |
| 394 | def get_branch(self, all=False, remote_tracking=False): |
| 395 | """ |
| 396 | Show the branches. |
| 397 | |
| 398 | @param all: List both remote-tracking branches and local branches (True) |
| 399 | or only the local ones (False). |
| 400 | @param remote_tracking: Lists the remote-tracking branches. |
| 401 | """ |
| 402 | if not self.is_repo_initialized(): |
| 403 | self.get() |
| 404 | |
| 405 | cmd = 'branch --no-color' |
| 406 | if all: |
| 407 | cmd = " ".join([cmd, "-a"]) |
| 408 | if remote_tracking: |
| 409 | cmd = " ".join([cmd, "-r"]) |
| 410 | |
| 411 | gitlog = self.gitcmd(cmd, True) |
| 412 | if gitlog.exit_status != 0: |
| 413 | logging.error(gitlog.stderr) |
| 414 | raise error.CmdError('Failed to get git branch', gitlog) |
| 415 | elif all or remote_tracking: |
| 416 | return gitlog.stdout.strip('\n') |
| 417 | else: |
| 418 | branch = [b[2:] for b in gitlog.stdout.split('\n') |
| 419 | if b.startswith('*')][0] |
| 420 | return branch |