utils/ccc-analyzer

#!/usr/bin/env perl
#
#                     The LLVM Compiler Infrastructure
#
# This file is distributed under the University of Illinois Open Source
# License. See LICENSE.TXT for details.
#
##===----------------------------------------------------------------------===##
#
#  A script designed to interpose between the build system and gcc.  It invokes
#  both gcc and the static analyzer.
#
##===----------------------------------------------------------------------===##

use strict;
use warnings;
use Cwd qw/ getcwd abs_path /;
use File::Temp qw/ tempfile /;
use File::Path qw / mkpath /;
use File::Basename;
use Text::ParseWords;

my $CC = $ENV{'CCC_CC'};
if (!defined $CC) { $CC = "gcc"; }
my $CleanupFile;
my $ResultFile;

# Remove any stale files at exit.
END { 
  if (defined $CleanupFile && -z $CleanupFile) {
    `rm -f $CleanupFile`;
  }
}

##----------------------------------------------------------------------------##
#  Process Clang Crashes.
##----------------------------------------------------------------------------##

sub GetPPExt {
  my $Lang = shift;
  if ($Lang =~ /objective-c/) { return ".mi"; }
  return ".i";
}

# Set this to 1 if we want to include 'parser rejects' files.
my $IncludeParserRejects = 0;
my $ParserRejects = "Parser Rejects";

my $AttributeIgnored = "Attribute Ignored";

sub ProcessClangFailure {
  my ($ClangCC, $Lang, $file, $Args, $HtmlDir, $ErrorType, $ofile) = @_;
  my $Dir = "$HtmlDir/failures";
  mkpath $Dir;
  
  my $prefix = "clang_crash";
  if ($ErrorType eq $ParserRejects) {
    $prefix = "clang_parser_rejects";
  }
  elsif ($ErrorType eq $AttributeIgnored) {
    $prefix = "clang_attribute_ignored";
  }

  # Generate the preprocessed file with cc (i.e., gcc).
  my ($PPH, $PPFile) = tempfile( $prefix . "_XXXXXX",
                                 SUFFIX => GetPPExt($Lang),
                                 DIR => $Dir);

  system $CC, @$Args, "-E", "-o", $PPFile;
  close ($PPH);

  # Generate the preprocessed file with clang.
  my $PPFile_Clang = $PPFile;
  $PPFile_Clang =~ s/[.](.+)$/.clang.$1/;  
  system $ClangCC, @$Args, "-E", "-o", "$PPFile_Clang";
  
  # Create the info file.
  open (OUT, ">", "$PPFile.info.txt") or die "Cannot open $PPFile.info.txt\n";
  print OUT abs_path($file), "\n";
  print OUT "$ErrorType\n";
  print OUT "@$Args\n";
  close OUT;
  `uname -a >> $PPFile.info.txt 2>&1`;
  `$CC -v >> $PPFile.info.txt 2>&1`;
  system 'mv',$ofile,"$PPFile.stderr.txt";
  return (basename $PPFile);
}

##----------------------------------------------------------------------------##
#  Running the analyzer.
##----------------------------------------------------------------------------##

# Determine what clang executable to use.
my $Clang = $ENV{'CLANG'};
if (!defined $Clang) { $Clang = 'clang'; }

sub GetCCArgs {
  my $Args = shift;
  
  pipe (FROM_CHILD, TO_PARENT);
  my $pid = fork();
  if ($pid == 0) {
    close FROM_CHILD;
    open(STDOUT,">&", \*TO_PARENT);
    open(STDERR,">&", \*TO_PARENT);
    exec $Clang, "-###", "-fsyntax-only", @$Args;
  }  
  close(TO_PARENT);
  my $line;
  while (<FROM_CHILD>) {
    next if (!/clang-cc/);
    $line = $_;
  }

  waitpid($pid,0);
  close(FROM_CHILD);
  
  die "could not find clang-cc line\n" if (!defined $line);
  # Strip the newline and initial whitspace
  chomp $line;  
  $line =~ s/^\s+//;
  
  my @items = quotewords('\s+', 1, $line);
  for (my $i = 0 ; $ i < scalar(@items); ++$i) {
    $items[$i] =~ s/^\"//;
    $items[$i] =~ s/\"$//;
  }
  my $cmd = shift @items;
  die "cannot find 'clang-cc' in 'clang' command\n" if (!($cmd =~ /clang-cc/));
  return \@items;
}

sub Analyze {
  my ($ClangCC, $Args, $AnalyzeArgs, $Lang, $Output, $Verbose, $HtmlDir,
      $file, $Analyses) = @_;
  
  $Args = GetCCArgs($Args);

  # Skip anything related to C++.
  return if ($Lang =~ /c[+][+]/);

  my $RunAnalyzer = 0;
  my $Cmd;
  my @CmdArgs;
  my @CmdArgsSansAnalyses;
  
  if ($Lang =~ /header/) {
    exit 0 if (!defined ($Output));
    $Cmd = 'cp';
    push @CmdArgs,$file;
    # Remove the PCH extension.
    $Output =~ s/[.]gch$//;
    push @CmdArgs,$Output;
    @CmdArgsSansAnalyses = @CmdArgs;    
  }
  else {
    $Cmd = $ClangCC;
    push @CmdArgs,'-DIBOutlet=__attribute__((iboutlet))';
    push @CmdArgs,@$Args;
    @CmdArgsSansAnalyses = @CmdArgs;
    push @CmdArgs,'-analyze';
    push @CmdArgs,"-analyzer-display-progress";
    push @CmdArgs,"-analyzer-eagerly-assume";
    push @CmdArgs,(split /\s/,$Analyses);
    $RunAnalyzer = 1;
  }
  
  # Add the analysis arguments passed down from scan-build.
  foreach my $Arg (@$AnalyzeArgs) {
    push @CmdArgs, $Arg;
  }
  
  my @PrintArgs;
  my $dir;
  
  if ($Verbose) {
    $dir = getcwd();
    print STDERR "\n[LOCATION]: $dir\n";
    push @PrintArgs,"'$Cmd'";
    foreach my $arg (@CmdArgs) { push @PrintArgs,"\'$arg\'"; }
  }
  
  if ($Verbose == 1) {
    # We MUST print to stderr.  Some clients use the stdout output of
    # gcc for various purposes. 
    print STDERR join(' ',@PrintArgs);
    print STDERR "\n";
  }
  elsif ($Verbose == 2) {
    print STDERR "#SHELL (cd '$dir' && @PrintArgs)\n";
  }
  
  if ($RunAnalyzer) {
    if (defined $ResultFile) {
      push @CmdArgs,'-o';
      push @CmdArgs, $ResultFile;
    }
    elsif (defined $HtmlDir) {
      push @CmdArgs,'-o';
      push @CmdArgs, $HtmlDir;
    }
  }
  
  if (defined $ENV{'CCC_UBI'}) {   
    push @CmdArgs,"--analyzer-viz-egraph-ubigraph";
  }
  
  # Capture the STDERR of clang and send it to a temporary file.
  # Capture the STDOUT of clang and reroute it to ccc-analyzer's STDERR.
  # We save the output file in the 'crashes' directory if clang encounters
  # any problems with the file.  
  pipe (FROM_CHILD, TO_PARENT);
  my $pid = fork();
  if ($pid == 0) {
    close FROM_CHILD;
    open(STDOUT,">&", \*TO_PARENT);
    open(STDERR,">&", \*TO_PARENT);
    exec $Cmd, @CmdArgs;
  }
  
  close TO_PARENT;
  my ($ofh, $ofile) = tempfile("clang_output_XXXXXX", DIR => $HtmlDir);
  
  while (<FROM_CHILD>) {
    print $ofh $_;
    print STDERR $_;    
  }

  waitpid($pid,0);
  close(FROM_CHILD);
  my $Result = $?;

  # Did the command die because of a signal?
  if ($Result & 127 and $Cmd eq $ClangCC and defined $HtmlDir) {
    ProcessClangFailure($ClangCC, $Lang, $file, \@CmdArgsSansAnalyses, $HtmlDir,
                        "Crash", $ofile);
  }
  elsif ($Result) {
    if ($IncludeParserRejects && !($file =~/conftest/)) {
      ProcessClangFailure($ClangCC, $Lang, $file, \@CmdArgsSansAnalyses, $HtmlDir,
                          $ParserRejects, $ofile);
    }
  }
  else {
    # Check if there were any unhandled attributes.
    if (open(CHILD, $ofile)) {
      my %attributes_not_handled;
      
      # Don't flag warnings about the following attributes that we
      # know are currently not supported by Clang.
      $attributes_not_handled{"cdecl"} = 1;
      
      my $ppfile;
      while (<CHILD>) {
        next if (! /warning: '([^\']+)' attribute ignored/);

        # Have we already spotted this unhandled attribute?
        next if (defined $attributes_not_handled{$1});
        $attributes_not_handled{$1} = 1;
        
        # Get the name of the attribute file.
        my $dir = "$HtmlDir/failures";
        my $afile = "$dir/attribute_ignored_$1.txt";
        
        # Only create another preprocessed file if the attribute file
        # doesn't exist yet.
        next if (-e $afile);
        
        # Add this file to the list of files that contained this attribute.
        # Generate a preprocessed file if we haven't already.
        if (!(defined $ppfile)) {
          $ppfile = ProcessClangFailure($ClangCC, $Lang, $file,
                                        \@CmdArgsSansAnalyses,
                                        $HtmlDir, $AttributeIgnored, $ofile);
        }

        mkpath $dir;
        open(AFILE, ">$afile");
        print AFILE "$ppfile\n";
        close(AFILE);
      }
      close CHILD;
    }
  }
  
  `rm -f $ofile`;
}

##----------------------------------------------------------------------------##
#  Lookup tables.
##----------------------------------------------------------------------------##

my %CompileOptionMap = (
  '-nostdinc' => 0,
  '-fblocks' => 0,
  '-fobjc-gc-only' => 0,
  '-fobjc-gc' => 0,
  '-ffreestanding' => 0,
  '-include' => 1,
  '-idirafter' => 1,
  '-iprefix' => 1,
  '-iquote' => 1,
  '-isystem' => 1,
  '-iwithprefix' => 1,
  '-iwithprefixbefore' => 1
);

my %LinkerOptionMap = (
  '-framework' => 1
);

my %CompilerLinkerOptionMap = (
  '-isysroot' => 1,
  '-arch' => 1,
  '-v' => 0,
  '-fpascal-strings' => 0,
  '-mmacosx-version-min' => 0, # This is really a 1 argument, but always has '='
  '-miphoneos-version-min' => 0 # This is really a 1 argument, but always has '='
);

my %IgnoredOptionMap = (
  '-MT' => 1,  # Ignore these preprocessor options.
  '-MF' => 1,

  '-fsyntax-only' => 0,
  '-save-temps' => 0,
  '-install_name' => 1,
  '-exported_symbols_list' => 1,
  '-current_version' => 1,
  '-compatibility_version' => 1,
  '-init' => 1,
  '-e' => 1,
  '-seg1addr' => 1,
  '-bundle_loader' => 1,
  '-multiply_defined' => 1,
  '-sectorder' => 3,
  '--param' => 1,
  '-u' => 1
);

my %LangMap = (
  'c'   => 'c',
  'cpp' => 'c++',
  'cc'  => 'c++',
  'i'   => 'c-cpp-output',
  'm'   => 'objective-c',
  'mi'  => 'objective-c-cpp-output'
);

my %UniqueOptions = (
  '-isysroot' => 0  
);

my %LangsAccepted = (
  "objective-c" => 1,
  "c" => 1
);

##----------------------------------------------------------------------------##
#  Main Logic.
##----------------------------------------------------------------------------##

my $Action = 'link';
my @CompileOpts;
my @LinkOpts;
my @Files;
my $Lang;
my $Output;
my %Uniqued;

# Forward arguments to gcc.
my $Status = system($CC,@ARGV);
if ($Status) { exit($Status >> 8); }

# Get the analysis options.
my $Analyses = $ENV{'CCC_ANALYZER_ANALYSIS'};
if (!defined($Analyses)) { $Analyses = '-checker-cfref'; }

# Get the store model.
my $StoreModel = $ENV{'CCC_ANALYZER_STORE_MODEL'};
if (!defined $StoreModel) { $StoreModel = "basic"; }

# Get the constraints engine.
my $ConstraintsModel = $ENV{'CCC_ANALYZER_CONSTRAINTS_MODEL'};
if (!defined $ConstraintsModel) { $ConstraintsModel = "range"; }

# Get the output format.
my $OutputFormat = $ENV{'CCC_ANALYZER_OUTPUT_FORMAT'};
if (!defined $OutputFormat) { $OutputFormat = "html"; }

# Determine the level of verbosity.
my $Verbose = 0;
if (defined $ENV{CCC_ANALYZER_VERBOSE}) { $Verbose = 1; }
if (defined $ENV{CCC_ANALYZER_LOG}) { $Verbose = 2; }

# Determine what clang-cc executable to use.
my $ClangCC = $ENV{'CLANG_CC'};
if (!defined $ClangCC) { $ClangCC = 'clang-cc'; }

# Get the HTML output directory.
my $HtmlDir = $ENV{'CCC_ANALYZER_HTML'};

my %DisabledArchs = ('ppc' => 1, 'ppc64' => 1);
my %ArchsSeen;
my $HadArch = 0;

# Process the arguments.
foreach (my $i = 0; $i < scalar(@ARGV); ++$i) {
  my $Arg = $ARGV[$i];  
  my ($ArgKey) = split /=/,$Arg,2;

  # Modes ccc-analyzer supports
  if ($Arg eq '-E') { $Action = 'preprocess'; }
  elsif ($Arg eq '-c') { $Action = 'compile'; }
  elsif ($Arg =~ /^-print-prog-name/) { exit 0; }

  # Specially handle duplicate cases of -arch
  if ($Arg eq "-arch") {
    my $arch = $ARGV[$i+1];
    # We don't want to process 'ppc' because of Clang's lack of support
    # for Altivec (also some #defines won't likely be defined correctly, etc.)
    if (!(defined $DisabledArchs{$arch})) { $ArchsSeen{$arch} = 1; }
    $HadArch = 1;
    ++$i;
    next;
  }

  # Options with possible arguments that should pass through to compiler.
  if (defined $CompileOptionMap{$ArgKey}) {
    my $Cnt = $CompileOptionMap{$ArgKey};
    push @CompileOpts,$Arg;
    while ($Cnt > 0) { ++$i; --$Cnt; push @CompileOpts, $ARGV[$i]; }
    next;
  }

  # Options with possible arguments that should pass through to linker.
  if (defined $LinkerOptionMap{$ArgKey}) {
    my $Cnt = $LinkerOptionMap{$ArgKey};
    push @LinkOpts,$Arg;
    while ($Cnt > 0) { ++$i; --$Cnt; push @LinkOpts, $ARGV[$i]; }
    next;
  }

  # Options with possible arguments that should pass through to both compiler
  # and the linker.
  if (defined $CompilerLinkerOptionMap{$ArgKey}) {
    my $Cnt = $CompilerLinkerOptionMap{$ArgKey};
    
    # Check if this is an option that should have a unique value, and if so
    # determine if the value was checked before.
    if ($UniqueOptions{$Arg}) {
      if (defined $Uniqued{$Arg}) {
        $i += $Cnt;
        next;
      }
      $Uniqued{$Arg} = 1;
    }
    
    push @CompileOpts,$Arg;    
    push @LinkOpts,$Arg;

    while ($Cnt > 0) {
      ++$i; --$Cnt;
      push @CompileOpts, $ARGV[$i];
      push @LinkOpts, $ARGV[$i];
    }
    next;
  }
  
  # Ignored options.
  if (defined $IgnoredOptionMap{$ArgKey}) {
    my $Cnt = $IgnoredOptionMap{$ArgKey};
    while ($Cnt > 0) {
      ++$i; --$Cnt;
    }
    next;
  }
  
  # Compile mode flags.
  if ($Arg =~ /^-[D,I,U](.*)$/) {
    my $Tmp = $Arg;    
    if ($1 eq '') {
      # FIXME: Check if we are going off the end.
      ++$i;
      $Tmp = $Arg . $ARGV[$i];
    }
    push @CompileOpts,$Tmp;
    next;
  }
  
  # Language.
  if ($Arg eq '-x') {
    $Lang = $ARGV[$i+1];
    ++$i; next;
  }

  # Output file.
  if ($Arg eq '-o') {
    ++$i;
    $Output = $ARGV[$i];
    next;
  }
  
  # Get the link mode.
  if ($Arg =~ /^-[l,L,O]/) {
    if ($Arg eq '-O') { push @LinkOpts,'-O1'; }
    elsif ($Arg eq '-Os') { push @LinkOpts,'-O2'; }
    else { push @LinkOpts,$Arg; }
    next;
  }
  
  if ($Arg =~ /^-std=/) {
    push @CompileOpts,$Arg;
    next;
  }
  
#  if ($Arg =~ /^-f/) {
#    # FIXME: Not sure if the remaining -fxxxx options have no arguments.
#    push @CompileOpts,$Arg;
#    push @LinkOpts,$Arg;  # FIXME: Not sure if these are link opts.
#  }
  
  # Get the compiler/link mode.
  if ($Arg =~ /^-F(.+)$/) {
    my $Tmp = $Arg;
    if ($1 eq '') {
      # FIXME: Check if we are going off the end.
      ++$i;
      $Tmp = $Arg . $ARGV[$i];
    }
    push @CompileOpts,$Tmp;
    push @LinkOpts,$Tmp;
    next;
  }

  # Input files.
  if ($Arg eq '-filelist') {
    # FIXME: Make sure we aren't walking off the end.
    open(IN, $ARGV[$i+1]);
    while (<IN>) { s/\015?\012//; push @Files,$_; }
    close(IN);
    ++$i; next;
  }
  
  if (!($Arg =~ /^-/)) {
    push @Files,$Arg; next;
  }
}

if ($Action eq 'compile' or $Action eq 'link') {
  my @Archs = keys %ArchsSeen;
  # Skip the file if we don't support the architectures specified.
  exit 0 if ($HadArch && scalar(@Archs) == 0);
  
  foreach my $file (@Files) {
    # Determine the language for the file.
    my $FileLang = $Lang;

    if (!defined($FileLang)) {
      # Infer the language from the extension.
      if ($file =~ /[.]([^.]+)$/) {
        $FileLang = $LangMap{$1};
      }
    }
    
    next if (!defined $FileLang);
    next if (!defined $LangsAccepted{$FileLang});
    
    my @CmdArgs;
    my @AnalyzeArgs;    
    
    if ($FileLang ne 'unknown') {
      push @CmdArgs,'-x';
      push @CmdArgs,$FileLang;
    }

    if (defined $StoreModel) {
      push @AnalyzeArgs, "-analyzer-store=$StoreModel";
    }

    if (defined $ConstraintsModel) {
      push @AnalyzeArgs, "-analyzer-constraints=$ConstraintsModel";
    }

    if (defined $OutputFormat) {
      push @AnalyzeArgs, "-analyzer-output=" . $OutputFormat;
      if ($OutputFormat eq "plist") {
        # Change "Output" to be a file.
        my ($h, $f) = tempfile("report-XXXXXX", SUFFIX => ".plist",
                               DIR => $HtmlDir);
        $ResultFile = $f;
        $CleanupFile = $f;
      }
    }

    push @CmdArgs,@CompileOpts;
    push @CmdArgs,$file;

    if (scalar @Archs) {
      foreach my $arch (@Archs) {
        my @NewArgs;
        push @NewArgs, '-arch';
        push @NewArgs, $arch;
        push @NewArgs, @CmdArgs;
        Analyze($ClangCC, \@NewArgs, \@AnalyzeArgs, $FileLang, $Output,
                $Verbose, $HtmlDir, $file, $Analyses);
      }
    }
    else {
      Analyze($ClangCC, \@CmdArgs, \@AnalyzeArgs, $FileLang, $Output,
              $Verbose, $HtmlDir, $file, $Analyses);
    }
  }
}

exit($Status >> 8);