Gitiles
Code Review Sign In
gerrit-public.fairphone.software / platform / external / python / cpython3 / 292d351fc1a7dc9feccd412843832808680a631f / . / Misc / build.sh
blob: 8eca6bafb82d545b5891d7066129b845b8ba7b1b [file] [log] [blame]
Neal Norwitz461aa5b2006-01-02 20:07:16 +0000[diff] [blame]1#!/bin/sh
2
3## Script to build and test the latest python from svn. It basically
4## does this:
5## svn up ; ./configure ; make ; make test ; make install ; cd Doc ; make
6##
7## Logs are kept and rsync'ed to the host. If there are test failure(s),
8## information about the failure(s) is mailed.
9##
10## This script is run on the PSF's machine as user neal via crontab.
11##
12## Yes, this script would probably be easier in python, but then
13## there's a bootstrap problem. What if Python doesn't build?
14##
15## This script should be fairly clean Bourne shell, ie not too many
16## bash-isms. We should try to keep it portable to other Unixes.
17## Even though it will probably only run on Linux. I'm sure there are
18## several GNU-isms currently (date +%s and readlink).
19##
20## Perhaps this script should be broken up into 2 (or more) components.
21## Building doc is orthogonal to the rest of the python build/test.
22##
23
24## FIXME: we should detect test hangs (eg, if they take more than 45 minutes)
25
26## FIXME: we should run valgrind
27## FIXME: we should run code coverage
28
29## Utilities invoked in this script include:
Neal Norwitzd3a58672006-01-02 23:22:41 +0000[diff] [blame]30## basename, date, dirname, expr, grep, readlink, uname
Neal Norwitz461aa5b2006-01-02 20:07:16 +0000[diff] [blame]31## cksum, make, mutt, rsync, svn
32
Neal Norwitz461aa5b2006-01-02 20:07:16 +0000[diff] [blame]33## remember where did we started from
34DIR=`dirname $0`
35if [ "$DIR" = "" ]; then
36 DIR="."
37fi
38
39## make directory absolute
40DIR=`readlink -f $DIR`
Neal Norwitzd3a58672006-01-02 23:22:41 +0000[diff] [blame]41FULLPATHNAME="$DIR/`basename $0`"
Neal Norwitz461aa5b2006-01-02 20:07:16 +0000[diff] [blame]42## we want Misc/..
43DIR=`dirname $DIR`
44
45## Configurable options
46
47FAILURE_SUBJECT="Python Regression Test Failures"
Neal Norwitz461aa5b2006-01-02 20:07:16 +0000[diff] [blame]48#FAILURE_MAILTO="YOUR_ACCOUNT@gmail.com"
Guido van Rossum52056532007-05-10 14:04:07 +0000[diff] [blame]49FAILURE_MAILTO="python-3000-checkins@python.org"
Guido van Rossumd8faa362007-04-27 19:54:29 +0000[diff] [blame]50#FAILURE_CC="optional--uncomment and set to desired address"
Neal Norwitz461aa5b2006-01-02 20:07:16 +0000[diff] [blame]51
52REMOTE_SYSTEM="neal@dinsdale.python.org"
Guido van Rossum52056532007-05-10 14:04:07 +0000[diff] [blame]53REMOTE_DIR="/data/ftp.python.org/pub/docs.python.org/dev/3.0"
Neal Norwitz461aa5b2006-01-02 20:07:16 +0000[diff] [blame]54RESULT_FILE="$DIR/build/index.html"
Guido van Rossum52056532007-05-10 14:04:07 +0000[diff] [blame]55INSTALL_DIR="/tmp/python-test-3.0/local"
Neal Norwitz461aa5b2006-01-02 20:07:16 +0000[diff] [blame]56RSYNC_OPTS="-aC -e ssh"
57
Thomas Wouters0e3f5912006-08-11 14:57:12 +0000[diff] [blame]58# Always run the installed version of Python.
59PYTHON=$INSTALL_DIR/bin/python
60
61# Python options and regression test program that should always be run.
Guido van Rossum52056532007-05-10 14:04:07 +0000[diff] [blame]62REGRTEST_ARGS="-E -tt $INSTALL_DIR/lib/python3.0/test/regrtest.py"
Thomas Wouters0e3f5912006-08-11 14:57:12 +0000[diff] [blame]63
Neal Norwitz461aa5b2006-01-02 20:07:16 +0000[diff] [blame]64REFLOG="build/reflog.txt.out"
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000[diff] [blame]65# These tests are not stable and falsely report leaks sometimes.
Neal Norwitzf415d5f2006-02-19 18:48:19 +0000[diff] [blame]66# The entire leak report will be mailed if any test not in this list leaks.
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000[diff] [blame]67# Note: test_XXX (none currently) really leak, but are disabled
68# so we don't send spam. Any test which really leaks should only
69# be listed here if there are also test cases under Lib/test/leakers.
Christian Heimes292d3512008-02-03 16:51:08 +0000[diff] [blame^]70LEAKY_TESTS="test_(cmd_line|popen2|socket|sys|threadsignals|urllib2_localnet)"
Guido van Rossum52056532007-05-10 14:04:07 +0000[diff] [blame]71
72# These tests always fail, so skip them so we don't get false positives.
Guido van Rossumcd16bf62007-06-13 18:07:49 +0000[diff] [blame]73_ALWAYS_SKIP=""
Guido van Rossum52056532007-05-10 14:04:07 +0000[diff] [blame]74ALWAYS_SKIP="-x $_ALWAYS_SKIP"
Neal Norwitzf415d5f2006-02-19 18:48:19 +0000[diff] [blame]75
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000[diff] [blame]76# Skip these tests altogether when looking for leaks. These tests
77# do not need to be stored above in LEAKY_TESTS too.
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000[diff] [blame]78# test_logging causes hangs, skip it.
Neal Norwitzcb44f262007-08-16 05:15:20 +0000[diff] [blame]79LEAKY_SKIPS="-x test_logging $_ALWAYS_SKIP"
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000[diff] [blame]80
81# Change this flag to "yes" for old releases to only update/build the docs.
Neal Norwitz88b78d82006-02-14 08:14:16 +0000[diff] [blame]82BUILD_DISABLED="no"
Neal Norwitz461aa5b2006-01-02 20:07:16 +0000[diff] [blame]83
84## utility functions
85current_time() {
86 date +%s
87}
88
89update_status() {
90 now=`current_time`
91 time=`expr $now - $3`
92 echo "<li><a href=\"$2\">$1</a> <font size=\"-1\">($time seconds)</font></li>" >> $RESULT_FILE
93}
94
95mail_on_failure() {
96 if [ "$NUM_FAILURES" != "0" ]; then
Guido van Rossumd8faa362007-04-27 19:54:29 +0000[diff] [blame]97 dest=$FAILURE_MAILTO
98 # FAILURE_CC is optional.
99 if [ "$FAILURE_CC" != "" ]; then
100 dest="$dest -c $FAILURE_CC"
101 fi
Christian Heimes292d3512008-02-03 16:51:08 +0000[diff] [blame^]102 if [ "x$3" != "x" ] ; then
103 (echo "More important issues:"
104 echo "----------------------"
105 egrep -v "$3" < $2
106 echo ""
107 echo "Less important issues:"
108 echo "----------------------"
109 egrep "$3" < $2)
110 else
111 cat $2
112 fi | mutt -s "$FAILURE_SUBJECT $1 ($NUM_FAILURES)" $dest
Neal Norwitz461aa5b2006-01-02 20:07:16 +0000[diff] [blame]113 fi
114}
115
116## setup
117cd $DIR
118mkdir -p build
119rm -f $RESULT_FILE build/*.out
120rm -rf $INSTALL_DIR
121
122## create results file
123TITLE="Automated Python Build Results"
Neal Norwitzb8967592006-01-16 04:37:22 +0000[diff] [blame]124echo "<html>" >> $RESULT_FILE
125echo " <head>" >> $RESULT_FILE
126echo " <title>$TITLE</title>" >> $RESULT_FILE
127echo " <meta http-equiv=\"refresh\" content=\"43200\">" >> $RESULT_FILE
128echo " </head>" >> $RESULT_FILE
Neal Norwitz461aa5b2006-01-02 20:07:16 +0000[diff] [blame]129echo "<body>" >> $RESULT_FILE
130echo "<h2>Automated Python Build Results</h2>" >> $RESULT_FILE
131echo "<table>" >> $RESULT_FILE
132echo " <tr>" >> $RESULT_FILE
133echo " <td>Built on:</td><td>`date`</td>" >> $RESULT_FILE
134echo " </tr><tr>" >> $RESULT_FILE
135echo " <td>Hostname:</td><td>`uname -n`</td>" >> $RESULT_FILE
136echo " </tr><tr>" >> $RESULT_FILE
137echo " <td>Platform:</td><td>`uname -srmpo`</td>" >> $RESULT_FILE
138echo " </tr>" >> $RESULT_FILE
139echo "</table>" >> $RESULT_FILE
140echo "<ul>" >> $RESULT_FILE
141
142## update, build, and test
143ORIG_CHECKSUM=`cksum $FULLPATHNAME`
144F=svn-update.out
145start=`current_time`
146svn update >& build/$F
147err=$?
148update_status "Updating" "$F" $start
Neal Norwitz88b78d82006-02-14 08:14:16 +0000[diff] [blame]149if [ $err = 0 -a "$BUILD_DISABLED" != "yes" ]; then
Neal Norwitz461aa5b2006-01-02 20:07:16 +0000[diff] [blame]150 ## FIXME: we should check if this file has changed.
151 ## If it has changed, we should re-run the script to pick up changes.
152 if [ "$ORIG_CHECKSUM" != "$ORIG_CHECKSUM" ]; then
153 exec $FULLPATHNAME $@
154 fi
155
156 F=svn-stat.out
157 start=`current_time`
158 svn stat >& build/$F
159 ## ignore some of the diffs
160 NUM_DIFFS=`egrep -vc '^. (@test|db_home|Lib/test/(regrtest\.py|db_home))$' build/$F`
161 update_status "svn stat ($NUM_DIFFS possibly important diffs)" "$F" $start
162
163 F=configure.out
164 start=`current_time`
165 ./configure --prefix=$INSTALL_DIR --with-pydebug >& build/$F
166 err=$?
167 update_status "Configuring" "$F" $start
168 if [ $err = 0 ]; then
169 F=make.out
170 start=`current_time`
171 make >& build/$F
172 err=$?
173 warnings=`grep warning build/$F | egrep -vc "te?mpnam(_r|)' is dangerous,"`
174 update_status "Building ($warnings warnings)" "$F" $start
175 if [ $err = 0 ]; then
Neal Norwitzd19a4d42006-01-02 22:10:10 +0000[diff] [blame]176 ## make install
Neal Norwitz461aa5b2006-01-02 20:07:16 +0000[diff] [blame]177 F=make-install.out
178 start=`current_time`
179 make install >& build/$F
180 update_status "Installing" "$F" $start
181
Thomas Wouters0e3f5912006-08-11 14:57:12 +0000[diff] [blame]182 if [ ! -x $PYTHON ]; then
Guido van Rossum52056532007-05-10 14:04:07 +0000[diff] [blame]183 ln -s ${PYTHON}3.* $PYTHON
Thomas Wouters0e3f5912006-08-11 14:57:12 +0000[diff] [blame]184 fi
185
Neal Norwitzd19a4d42006-01-02 22:10:10 +0000[diff] [blame]186 ## make and run basic tests
Neal Norwitz461aa5b2006-01-02 20:07:16 +0000[diff] [blame]187 F=make-test.out
188 start=`current_time`
Guido van Rossum52056532007-05-10 14:04:07 +0000[diff] [blame]189 $PYTHON $REGRTEST_ARGS $ALWAYS_SKIP >& build/$F
Neal Norwitz79415522006-02-15 06:07:32 +0000[diff] [blame]190 NUM_FAILURES=`grep -ic " failed:" build/$F`
Neal Norwitz461aa5b2006-01-02 20:07:16 +0000[diff] [blame]191 update_status "Testing basics ($NUM_FAILURES failures)" "$F" $start
Neal Norwitz02c408d2006-01-03 02:18:01 +0000[diff] [blame]192 mail_on_failure "basics" build/$F
Neal Norwitz461aa5b2006-01-02 20:07:16 +0000[diff] [blame]193
Thomas Wouters0e3f5912006-08-11 14:57:12 +0000[diff] [blame]194 F=make-test-opt.out
195 start=`current_time`
Guido van Rossum52056532007-05-10 14:04:07 +0000[diff] [blame]196 $PYTHON -O $REGRTEST_ARGS $ALWAYS_SKIP >& build/$F
Thomas Wouters0e3f5912006-08-11 14:57:12 +0000[diff] [blame]197 NUM_FAILURES=`grep -ic " failed:" build/$F`
198 update_status "Testing opt ($NUM_FAILURES failures)" "$F" $start
199 mail_on_failure "opt" build/$F
200
Neal Norwitzd19a4d42006-01-02 22:10:10 +0000[diff] [blame]201 ## run the tests looking for leaks
Neal Norwitz461aa5b2006-01-02 20:07:16 +0000[diff] [blame]202 F=make-test-refleak.out
203 start=`current_time`
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000[diff] [blame]204 ## ensure that the reflog exists so the grep doesn't fail
205 touch $REFLOG
Thomas Wouters0e3f5912006-08-11 14:57:12 +0000[diff] [blame]206 $PYTHON $REGRTEST_ARGS -R 4:3:$REFLOG -u network $LEAKY_SKIPS >& build/$F
Christian Heimes292d3512008-02-03 16:51:08 +0000[diff] [blame^]207 LEAK_PAT="($LEAKY_TESTS|sum=0)"
208 NUM_FAILURES=`egrep -vc "$LEAK_PAT" $REFLOG`
Neal Norwitz461aa5b2006-01-02 20:07:16 +0000[diff] [blame]209 update_status "Testing refleaks ($NUM_FAILURES failures)" "$F" $start
Christian Heimes292d3512008-02-03 16:51:08 +0000[diff] [blame^]210 mail_on_failure "refleak" $REFLOG "$LEAK_PAT"
Neal Norwitz461aa5b2006-01-02 20:07:16 +0000[diff] [blame]211
Neal Norwitzd19a4d42006-01-02 22:10:10 +0000[diff] [blame]212 ## now try to run all the tests
Neal Norwitz461aa5b2006-01-02 20:07:16 +0000[diff] [blame]213 F=make-testall.out
214 start=`current_time`
Neal Norwitzd19a4d42006-01-02 22:10:10 +0000[diff] [blame]215 ## skip curses when running from cron since there's no terminal
216 ## skip sound since it's not setup on the PSF box (/dev/dsp)
Guido van Rossum52056532007-05-10 14:04:07 +0000[diff] [blame]217 $PYTHON $REGRTEST_ARGS -uall -x test_curses test_linuxaudiodev test_ossaudiodev $_ALWAYS_SKIP >& build/$F
Neal Norwitz79415522006-02-15 06:07:32 +0000[diff] [blame]218 NUM_FAILURES=`grep -ic " failed:" build/$F`
Neal Norwitz461aa5b2006-01-02 20:07:16 +0000[diff] [blame]219 update_status "Testing all except curses and sound ($NUM_FAILURES failures)" "$F" $start
Neal Norwitzd3a58672006-01-02 23:22:41 +0000[diff] [blame]220 mail_on_failure "all" build/$F
Neal Norwitz461aa5b2006-01-02 20:07:16 +0000[diff] [blame]221 fi
222 fi
223fi
224
225
226## make doc
Thomas Wouters0e3f5912006-08-11 14:57:12 +0000[diff] [blame]227cd $DIR/Doc
Neal Norwitz461aa5b2006-01-02 20:07:16 +0000[diff] [blame]228F="make-doc.out"
229start=`current_time`
Neal Norwitzcb44f262007-08-16 05:15:20 +0000[diff] [blame]230# XXX(nnorwitz): For now, keep the code that checks for a conflicted file until
231# after the first release of 2.6a1 or 3.0a1. At that point, it will be clear
232# if there will be a similar problem with the new doc system.
233
Guido van Rossumd8faa362007-04-27 19:54:29 +0000[diff] [blame]234# Doc/commontex/boilerplate.tex is expected to always have an outstanding
235# modification for the date. When a release is cut, a conflict occurs.
236# This allows us to detect this problem and not try to build the docs
237# which will definitely fail with a conflict.
Neal Norwitzcb44f262007-08-16 05:15:20 +0000[diff] [blame]238#CONFLICTED_FILE=commontex/boilerplate.tex
239#conflict_count=`grep -c "<<<" $CONFLICTED_FILE`
240conflict_count=0
Guido van Rossumd8faa362007-04-27 19:54:29 +0000[diff] [blame]241if [ $conflict_count != 0 ]; then
242 echo "Conflict detected in $CONFLICTED_FILE. Doc build skipped." > ../build/$F
243 err=1
244else
Neal Norwitzed44a1a2007-08-17 04:19:37 +0000[diff] [blame]245 make update html >& ../build/$F
Guido van Rossumd8faa362007-04-27 19:54:29 +0000[diff] [blame]246 err=$?
247fi
Neal Norwitz461aa5b2006-01-02 20:07:16 +0000[diff] [blame]248update_status "Making doc" "$F" $start
249if [ $err != 0 ]; then
250 NUM_FAILURES=1
251 mail_on_failure "doc" ../build/$F
252fi
253
254echo "</ul>" >> $RESULT_FILE
255echo "</body>" >> $RESULT_FILE
256echo "</html>" >> $RESULT_FILE
257
258## copy results
Neal Norwitzcb44f262007-08-16 05:15:20 +0000[diff] [blame]259rsync $RSYNC_OPTS build/html/* $REMOTE_SYSTEM:$REMOTE_DIR
Neal Norwitz461aa5b2006-01-02 20:07:16 +0000[diff] [blame]260cd ../build
261rsync $RSYNC_OPTS index.html *.out $REMOTE_SYSTEM:$REMOTE_DIR/results/