Merge remote branch 'cros/upstream' into autotest-rebase
Merged to upstream trunk@5066, from trunk@4749.
There is no way I could enlist each individual CL from the upstream here since it will blow up the changelist description field.
BUG=
TEST=
Had patched this CL into a fresh cut client to avoid any side effect.
run_remote_test bvt from both emerged location and third_party/autotest/file.
Both test passed!
We should also keep any eye on this to see how it gets propagated into cautotest server.
TBR=dalecurtis
Change-Id: I72f2bc7a9de530178484aea1bfb5ace68bcad029
diff --git a/utils/check_patch.py b/utils/check_patch.py
index 576b97d..78af6b9 100755
--- a/utils/check_patch.py
+++ b/utils/check_patch.py
@@ -20,7 +20,7 @@
@author: Lucas Meneghel Rodrigues <lmr@redhat.com>
"""
-import os, stat, logging, sys, optparse
+import os, stat, logging, sys, optparse, time
import common
from autotest_lib.client.common_lib import utils, error, logging_config
from autotest_lib.client.common_lib import logging_manager
@@ -32,6 +32,20 @@
verbose=verbose)
+def ask(question, auto=False):
+ """
+ Raw input with a prompt that emulates logging.
+
+ @param question: Question to be asked
+ @param auto: Whether to return "y" instead of asking the question
+ """
+ if auto:
+ logging.info("%s (y/n) y" % question)
+ return "y"
+ return raw_input("%s INFO | %s (y/n) " %
+ (time.strftime("%H:%M:%S", time.localtime()), question))
+
+
class VCS(object):
"""
Abstraction layer to the version control system.
@@ -104,6 +118,7 @@
"""
def __init__(self):
logging.debug("Subversion VCS backend initialized.")
+ self.ignored_extension_list = ['.orig', '.bak']
def get_unknown_files(self):
@@ -112,7 +127,9 @@
for line in status.split("\n"):
status_flag = line[0]
if line and status_flag == "?":
- unknown_files.append(line[1:].strip())
+ for extension in self.ignored_extension_list:
+ if not line.endswith(extension):
+ unknown_files.append(line[1:].strip())
return unknown_files
@@ -181,13 +198,16 @@
Picks up a given file and performs various checks, looking after problems
and eventually suggesting solutions.
"""
- def __init__(self, path):
+ def __init__(self, path, confirm=False):
"""
Class constructor, sets the path attribute.
@param path: Path to the file that will be checked.
+ @param confirm: Whether to answer yes to all questions asked without
+ prompting the user.
"""
self.path = path
+ self.confirm = confirm
self.basename = os.path.basename(self.path)
if self.basename.endswith('.py'):
self.is_python = True
@@ -204,7 +224,7 @@
self.first_line = checked_file.readline()
checked_file.close()
self.corrective_actions = []
- self.indentation_exceptions = ['cli/job_unittest.py']
+ self.indentation_exceptions = ['job_unittest.py']
def _check_indent(self):
@@ -226,8 +246,6 @@
reindent_results = reindent_raw.split(" ")[-1].strip(".")
if reindent_results == "changed":
if self.basename not in self.indentation_exceptions:
- logging.error("Possible indentation and spacing issues on "
- "file %s" % self.path)
self.corrective_actions.append("reindent.py -v %s" % self.path)
@@ -242,8 +260,7 @@
c_cmd = 'run_pylint.py %s' % self.path
rc = utils.system(c_cmd, ignore_status=True)
if rc != 0:
- logging.error("Possible syntax problems on file %s", self.path)
- logging.error("You might want to rerun '%s'", c_cmd)
+ logging.error("Syntax issues found during '%s'", c_cmd)
def _check_unittest(self):
@@ -260,9 +277,8 @@
unittest_cmd = 'python %s' % unittest_path
rc = utils.system(unittest_cmd, ignore_status=True)
if rc != 0:
- logging.error("Problems during unit test execution "
- "for file %s", self.path)
- logging.error("You might want to rerun '%s'", unittest_cmd)
+ logging.error("Unittest issues found during '%s'",
+ unittest_cmd)
def _check_permissions(self):
@@ -273,14 +289,10 @@
"""
if self.first_line.startswith("#!"):
if not self.is_executable:
- logging.info("File %s seems to require execution "
- "permissions. ", self.path)
- self.corrective_actions.append("chmod +x %s" % self.path)
+ self.corrective_actions.append("svn propset svn:executable ON %s" % self.path)
else:
if self.is_executable:
- logging.info("File %s does not seem to require execution "
- "permissions. ", self.path)
- self.corrective_actions.append("chmod -x %s" % self.path)
+ self.corrective_actions.append("svn propdel svn:executable %s" % self.path)
def report(self):
@@ -294,10 +306,9 @@
self._check_code()
self._check_unittest()
if self.corrective_actions:
- logging.info("The following corrective actions are suggested:")
for action in self.corrective_actions:
- logging.info(action)
- answer = raw_input("Would you like to apply it? (y/n) ")
+ answer = ask("Would you like to execute %s?" % action,
+ auto=self.confirm)
if answer == "y":
rc = utils.system(action, ignore_status=True)
if rc != 0:
@@ -305,7 +316,8 @@
class PatchChecker(object):
- def __init__(self, patch=None, patchwork_id=None):
+ def __init__(self, patch=None, patchwork_id=None, confirm=False):
+ self.confirm = confirm
self.base_dir = os.getcwd()
if patch:
self.patch = os.path.abspath(patch)
@@ -322,7 +334,7 @@
if changed_files_before:
logging.error("Repository has changed files prior to patch "
"application. ")
- answer = raw_input("Would you like to revert them? (y/n) ")
+ answer = ask("Would you like to revert them?", auto=self.confirm)
if answer == "n":
logging.error("Not safe to proceed without reverting files.")
sys.exit(1)
@@ -370,20 +382,20 @@
for untracked_file in add_to_vcs:
logging.info(untracked_file)
logging.info("Might need to be added to VCS")
- logging.info("Would you like to add them to VCS ? (y/n/abort) ")
- answer = raw_input()
+ answer = ask("Would you like to add them to VCS ?")
if answer == "y":
for untracked_file in add_to_vcs:
self.vcs.add_untracked_file(untracked_file)
modified_files_after.append(untracked_file)
elif answer == "n":
pass
- elif answer == "abort":
- sys.exit(1)
for modified_file in modified_files_after:
- file_checker = FileChecker(modified_file)
- file_checker.report()
+ # Additional safety check, new commits might introduce
+ # new directories
+ if os.path.isfile(modified_file):
+ file_checker = FileChecker(modified_file)
+ file_checker.report()
def check(self):
@@ -399,20 +411,37 @@
help='id of a given patchwork patch')
parser.add_option('--verbose', dest="debug", action='store_true',
help='include debug messages in console output')
+ parser.add_option('-f', '--full-check', dest="full_check",
+ action='store_true',
+ help='check the full tree for corrective actions')
+ parser.add_option('-y', '--yes', dest="confirm",
+ action='store_true',
+ help='Answer yes to all questions')
options, args = parser.parse_args()
local_patch = options.local_patch
id = options.id
debug = options.debug
+ full_check = options.full_check
+ confirm = options.confirm
logging_manager.configure_logging(CheckPatchLoggingConfig(), verbose=debug)
- if local_patch:
- patch_checker = PatchChecker(patch=local_patch)
- elif id:
- patch_checker = PatchChecker(patchwork_id=id)
+ ignore_file_list = ['common.py']
+ if full_check:
+ for root, dirs, files in os.walk('.'):
+ if not '.svn' in root:
+ for file in files:
+ if file not in ignore_file_list:
+ path = os.path.join(root, file)
+ file_checker = FileChecker(path, confirm=confirm)
+ file_checker.report()
else:
- logging.error('No patch or patchwork id specified. Aborting.')
- sys.exit(1)
-
- patch_checker.check()
+ if local_patch:
+ patch_checker = PatchChecker(patch=local_patch, confirm=confirm)
+ elif id:
+ patch_checker = PatchChecker(patchwork_id=id, confirm=confirm)
+ else:
+ logging.error('No patch or patchwork id specified. Aborting.')
+ sys.exit(1)
+ patch_checker.check()
diff --git a/utils/external_packages.py b/utils/external_packages.py
old mode 100644
new mode 100755
index d505eef..bcfd15d
--- a/utils/external_packages.py
+++ b/utils/external_packages.py
@@ -613,7 +613,8 @@
class ParamikoPackage(ExternalPackage):
version = '1.7.5'
local_filename = 'paramiko-%s.tar.gz' % version
- urls = ('http://www.lag.net/paramiko/download/' + local_filename,)
+ urls = ('http://www.lag.net/paramiko/download/' + local_filename,
+ 'ftp://mirrors.kernel.org/gentoo/distfiles/' + local_filename,)
hex_sum = '592be7a08290070b71da63a8e6f28a803399e5c5'
diff --git a/utils/run_pylint.py b/utils/run_pylint.py
index 0bf5f95..3c3e225 100755
--- a/utils/run_pylint.py
+++ b/utils/run_pylint.py
@@ -6,10 +6,13 @@
# do a basic check to see if pylint is even installed
try:
import pylint
+ from pylint.__pkginfo__ import version as pylint_version
except ImportError:
print "Unable to import pylint, it may need to be installed"
sys.exit(1)
+major, minor, release = pylint_version.split('.')
+pylint_version = float("%s.%s" % (major, minor))
pylintrc_path = os.path.expanduser('~/.pylintrc')
if not os.path.exists(pylintrc_path):
open(pylintrc_path, 'w').close()
@@ -54,10 +57,13 @@
# * common_lib.enum.Enum objects
# * DB model objects (scheduler models are the worst, but Django models also
# generate some errors)
-pylint_base_opts = ['--disable-msg-cat=warning,refactor,convention',
- '--disable-msg=E1101,E1103',
- '--reports=no',
- '--include-ids=y']
+if pylint_version >= 0.21:
+ pylint_base_opts = ['--disable=W,R,C,E1101,E1103']
+else:
+ pylint_base_opts = ['--disable-msg-cat=warning,refactor,convention',
+ '--disable-msg=E1101,E1103']
+pylint_base_opts += ['--reports=no',
+ '--include-ids=y']
file_list = sys.argv[1:]
if '--' in file_list:
diff --git a/utils/site_test_importer_attributes.py b/utils/site_test_importer_attributes.py
index f49aedb..a2a1b50 100755
--- a/utils/site_test_importer_attributes.py
+++ b/utils/site_test_importer_attributes.py
@@ -8,7 +8,6 @@
import common, re
-from autotest_lib.frontend.afe import models
def _set_attributes_custom(test, data):
@@ -24,9 +23,3 @@
# We set verify to always False (0).
test.run_verify = 0
-
- if hasattr(data, 'test_parameters'):
- for para_name in data.test_parameters:
- test_parameter = models.TestParameter.objects.get_or_create(
- test=test, name=para_name)[0]
- test_parameter.save()
diff --git a/utils/test_importer.py b/utils/test_importer.py
index 00b912c..6a63b1e 100755
--- a/utils/test_importer.py
+++ b/utils/test_importer.py
@@ -241,6 +241,12 @@
_log_or_execute(repr(new_test), new_test.save)
add_label_dependencies(new_test)
+
+ # save TestParameter
+ for para_name in data.test_parameters:
+ test_parameter = models.TestParameter.objects.get_or_create(
+ test=new_test, name=para_name)[0]
+ test_parameter.save()
def _set_attributes_clean(test, data):