Gitiles
Code Review Sign In
gerrit-public.fairphone.software / platform / external / python / cpython3 / b94ed71bd157bbcb25a2e82c4c0bc238a7647886 / . / Misc / build.sh
blob: d5079ca582cc039b3113ded5d4726cc3aca81ae9 [file] [log] [blame]
Neal Norwitz461aa5b2006-01-02 20:07:16 +0000[diff] [blame]1#!/bin/sh
2
3## Script to build and test the latest python from svn. It basically
4## does this:
5## svn up ; ./configure ; make ; make test ; make install ; cd Doc ; make
6##
7## Logs are kept and rsync'ed to the host. If there are test failure(s),
8## information about the failure(s) is mailed.
9##
10## This script is run on the PSF's machine as user neal via crontab.
11##
12## Yes, this script would probably be easier in python, but then
13## there's a bootstrap problem. What if Python doesn't build?
14##
15## This script should be fairly clean Bourne shell, ie not too many
16## bash-isms. We should try to keep it portable to other Unixes.
17## Even though it will probably only run on Linux. I'm sure there are
18## several GNU-isms currently (date +%s and readlink).
19##
20## Perhaps this script should be broken up into 2 (or more) components.
21## Building doc is orthogonal to the rest of the python build/test.
22##
23
24## FIXME: we should detect test hangs (eg, if they take more than 45 minutes)
25
26## FIXME: we should run valgrind
27## FIXME: we should run code coverage
28
29## Utilities invoked in this script include:
Neal Norwitzd3a58672006-01-02 23:22:41 +0000[diff] [blame]30## basename, date, dirname, expr, grep, readlink, uname
Neal Norwitz461aa5b2006-01-02 20:07:16 +0000[diff] [blame]31## cksum, make, mutt, rsync, svn
32
Neal Norwitz461aa5b2006-01-02 20:07:16 +0000[diff] [blame]33## remember where did we started from
34DIR=`dirname $0`
35if [ "$DIR" = "" ]; then
36 DIR="."
37fi
38
39## make directory absolute
40DIR=`readlink -f $DIR`
Neal Norwitzd3a58672006-01-02 23:22:41 +0000[diff] [blame]41FULLPATHNAME="$DIR/`basename $0`"
Neal Norwitz461aa5b2006-01-02 20:07:16 +0000[diff] [blame]42## we want Misc/..
43DIR=`dirname $DIR`
44
45## Configurable options
46
47FAILURE_SUBJECT="Python Regression Test Failures"
Neal Norwitz461aa5b2006-01-02 20:07:16 +0000[diff] [blame]48#FAILURE_MAILTO="YOUR_ACCOUNT@gmail.com"
Neal Norwitzf415d5f2006-02-19 18:48:19 +0000[diff] [blame]49FAILURE_MAILTO="python-checkins@python.org"
Neal Norwitz461aa5b2006-01-02 20:07:16 +0000[diff] [blame]50
51REMOTE_SYSTEM="neal@dinsdale.python.org"
52REMOTE_DIR="/data/ftp.python.org/pub/docs.python.org/dev/"
53RESULT_FILE="$DIR/build/index.html"
54INSTALL_DIR="/tmp/python-test/local"
55RSYNC_OPTS="-aC -e ssh"
56
Neal Norwitz524b59b2006-06-27 04:09:13 +0000[diff] [blame]57# Always run the installed version of Python.
58PYTHON=$INSTALL_DIR/bin/python
59
60# Python options and regression test program that should always be run.
Neal Norwitzf2fcfa32006-08-18 06:14:52 +0000[diff] [blame]61REGRTEST_ARGS="-E -tt $INSTALL_DIR/lib/python2.6/test/regrtest.py"
Neal Norwitz524b59b2006-06-27 04:09:13 +0000[diff] [blame]62
Neal Norwitz461aa5b2006-01-02 20:07:16 +0000[diff] [blame]63REFLOG="build/reflog.txt.out"
Neal Norwitzee6d23e2006-04-12 05:56:00 +0000[diff] [blame]64# These tests are not stable and falsely report leaks sometimes.
Neal Norwitzf415d5f2006-02-19 18:48:19 +0000[diff] [blame]65# The entire leak report will be mailed if any test not in this list leaks.
Neal Norwitza1f9b7f2006-04-06 07:58:59 +0000[diff] [blame]66# Note: test_XXX (none currently) really leak, but are disabled
Neal Norwitz770a8002006-03-17 04:52:38 +0000[diff] [blame]67# so we don't send spam. Any test which really leaks should only
68# be listed here if there are also test cases under Lib/test/leakers.
Neal Norwitz9602cc22006-06-18 19:35:01 +0000[diff] [blame]69LEAKY_TESTS="test_(XXX)" # Currently no tests should report spurious leaks.
Neal Norwitzee6d23e2006-04-12 05:56:00 +0000[diff] [blame]70
71# Skip these tests altogether when looking for leaks. These tests
72# do not need to be stored above in LEAKY_TESTS too.
73# test_compiler almost never finishes with the same number of refs
74# since it depends on other modules, skip it.
75# test_logging causes hangs, skip it.
76LEAKY_SKIPS="-x test_compiler test_logging"
Neal Norwitzf415d5f2006-02-19 18:48:19 +0000[diff] [blame]77
Neal Norwitz770a8002006-03-17 04:52:38 +0000[diff] [blame]78# Change this flag to "yes" for old releases to only update/build the docs.
Neal Norwitz88b78d82006-02-14 08:14:16 +0000[diff] [blame]79BUILD_DISABLED="no"
Neal Norwitz461aa5b2006-01-02 20:07:16 +0000[diff] [blame]80
81## utility functions
82current_time() {
83 date +%s
84}
85
86update_status() {
87 now=`current_time`
88 time=`expr $now - $3`
89 echo "<li><a href=\"$2\">$1</a> <font size=\"-1\">($time seconds)</font></li>" >> $RESULT_FILE
90}
91
92mail_on_failure() {
93 if [ "$NUM_FAILURES" != "0" ]; then
94 mutt -s "$FAILURE_SUBJECT $1 ($NUM_FAILURES)" $FAILURE_MAILTO < $2
95 fi
96}
97
98## setup
99cd $DIR
100mkdir -p build
101rm -f $RESULT_FILE build/*.out
102rm -rf $INSTALL_DIR
103
104## create results file
105TITLE="Automated Python Build Results"
Neal Norwitzb8967592006-01-16 04:37:22 +0000[diff] [blame]106echo "<html>" >> $RESULT_FILE
107echo " <head>" >> $RESULT_FILE
108echo " <title>$TITLE</title>" >> $RESULT_FILE
109echo " <meta http-equiv=\"refresh\" content=\"43200\">" >> $RESULT_FILE
110echo " </head>" >> $RESULT_FILE
Neal Norwitz461aa5b2006-01-02 20:07:16 +0000[diff] [blame]111echo "<body>" >> $RESULT_FILE
112echo "<h2>Automated Python Build Results</h2>" >> $RESULT_FILE
113echo "<table>" >> $RESULT_FILE
114echo " <tr>" >> $RESULT_FILE
115echo " <td>Built on:</td><td>`date`</td>" >> $RESULT_FILE
116echo " </tr><tr>" >> $RESULT_FILE
117echo " <td>Hostname:</td><td>`uname -n`</td>" >> $RESULT_FILE
118echo " </tr><tr>" >> $RESULT_FILE
119echo " <td>Platform:</td><td>`uname -srmpo`</td>" >> $RESULT_FILE
120echo " </tr>" >> $RESULT_FILE
121echo "</table>" >> $RESULT_FILE
122echo "<ul>" >> $RESULT_FILE
123
124## update, build, and test
125ORIG_CHECKSUM=`cksum $FULLPATHNAME`
126F=svn-update.out
127start=`current_time`
128svn update >& build/$F
129err=$?
130update_status "Updating" "$F" $start
Neal Norwitz88b78d82006-02-14 08:14:16 +0000[diff] [blame]131if [ $err = 0 -a "$BUILD_DISABLED" != "yes" ]; then
Neal Norwitz461aa5b2006-01-02 20:07:16 +0000[diff] [blame]132 ## FIXME: we should check if this file has changed.
133 ## If it has changed, we should re-run the script to pick up changes.
134 if [ "$ORIG_CHECKSUM" != "$ORIG_CHECKSUM" ]; then
135 exec $FULLPATHNAME $@
136 fi
137
138 F=svn-stat.out
139 start=`current_time`
140 svn stat >& build/$F
141 ## ignore some of the diffs
142 NUM_DIFFS=`egrep -vc '^. (@test|db_home|Lib/test/(regrtest\.py|db_home))$' build/$F`
143 update_status "svn stat ($NUM_DIFFS possibly important diffs)" "$F" $start
144
145 F=configure.out
146 start=`current_time`
147 ./configure --prefix=$INSTALL_DIR --with-pydebug >& build/$F
148 err=$?
149 update_status "Configuring" "$F" $start
150 if [ $err = 0 ]; then
151 F=make.out
152 start=`current_time`
153 make >& build/$F
154 err=$?
155 warnings=`grep warning build/$F | egrep -vc "te?mpnam(_r|)' is dangerous,"`
156 update_status "Building ($warnings warnings)" "$F" $start
157 if [ $err = 0 ]; then
Neal Norwitzd19a4d42006-01-02 22:10:10 +0000[diff] [blame]158 ## make install
Neal Norwitz461aa5b2006-01-02 20:07:16 +0000[diff] [blame]159 F=make-install.out
160 start=`current_time`
161 make install >& build/$F
162 update_status "Installing" "$F" $start
163
Neal Norwitz9815f8b2006-07-26 04:00:18 +0000[diff] [blame]164 if [ ! -x $PYTHON ]; then
165 ln -s ${PYTHON}2.* $PYTHON
166 fi
167
Neal Norwitzd19a4d42006-01-02 22:10:10 +0000[diff] [blame]168 ## make and run basic tests
Neal Norwitz461aa5b2006-01-02 20:07:16 +0000[diff] [blame]169 F=make-test.out
170 start=`current_time`
Neal Norwitz524b59b2006-06-27 04:09:13 +0000[diff] [blame]171 $PYTHON $REGRTEST_ARGS >& build/$F
Neal Norwitz79415522006-02-15 06:07:32 +0000[diff] [blame]172 NUM_FAILURES=`grep -ic " failed:" build/$F`
Neal Norwitz461aa5b2006-01-02 20:07:16 +0000[diff] [blame]173 update_status "Testing basics ($NUM_FAILURES failures)" "$F" $start
Neal Norwitz02c408d2006-01-03 02:18:01 +0000[diff] [blame]174 mail_on_failure "basics" build/$F
Neal Norwitz461aa5b2006-01-02 20:07:16 +0000[diff] [blame]175
Neal Norwitz524b59b2006-06-27 04:09:13 +0000[diff] [blame]176 F=make-test-opt.out
177 start=`current_time`
178 $PYTHON -O $REGRTEST_ARGS >& build/$F
179 NUM_FAILURES=`grep -ic " failed:" build/$F`
180 update_status "Testing opt ($NUM_FAILURES failures)" "$F" $start
181 mail_on_failure "opt" build/$F
182
Neal Norwitzd19a4d42006-01-02 22:10:10 +0000[diff] [blame]183 ## run the tests looking for leaks
Neal Norwitz461aa5b2006-01-02 20:07:16 +0000[diff] [blame]184 F=make-test-refleak.out
185 start=`current_time`
Neal Norwitz0f77da32006-04-17 01:48:41 +0000[diff] [blame]186 ## ensure that the reflog exists so the grep doesn't fail
187 touch $REFLOG
Neal Norwitz524b59b2006-06-27 04:09:13 +0000[diff] [blame]188 $PYTHON $REGRTEST_ARGS -R 4:3:$REFLOG -u network $LEAKY_SKIPS >& build/$F
Neal Norwitzf415d5f2006-02-19 18:48:19 +0000[diff] [blame]189 NUM_FAILURES=`egrep -vc "$LEAKY_TESTS" $REFLOG`
Neal Norwitz461aa5b2006-01-02 20:07:16 +0000[diff] [blame]190 update_status "Testing refleaks ($NUM_FAILURES failures)" "$F" $start
191 mail_on_failure "refleak" $REFLOG
192
Neal Norwitzd19a4d42006-01-02 22:10:10 +0000[diff] [blame]193 ## now try to run all the tests
Neal Norwitz461aa5b2006-01-02 20:07:16 +0000[diff] [blame]194 F=make-testall.out
195 start=`current_time`
Neal Norwitzd19a4d42006-01-02 22:10:10 +0000[diff] [blame]196 ## skip curses when running from cron since there's no terminal
197 ## skip sound since it's not setup on the PSF box (/dev/dsp)
Neal Norwitz524b59b2006-06-27 04:09:13 +0000[diff] [blame]198 $PYTHON $REGRTEST_ARGS -uall -x test_curses test_linuxaudiodev test_ossaudiodev >& build/$F
Neal Norwitz79415522006-02-15 06:07:32 +0000[diff] [blame]199 NUM_FAILURES=`grep -ic " failed:" build/$F`
Neal Norwitz461aa5b2006-01-02 20:07:16 +0000[diff] [blame]200 update_status "Testing all except curses and sound ($NUM_FAILURES failures)" "$F" $start
Neal Norwitzd3a58672006-01-02 23:22:41 +0000[diff] [blame]201 mail_on_failure "all" build/$F
Neal Norwitz461aa5b2006-01-02 20:07:16 +0000[diff] [blame]202 fi
203 fi
204fi
205
206
207## make doc
Neal Norwitz524b59b2006-06-27 04:09:13 +0000[diff] [blame]208cd $DIR/Doc
Neal Norwitz461aa5b2006-01-02 20:07:16 +0000[diff] [blame]209F="make-doc.out"
210start=`current_time`
211make >& ../build/$F
212err=$?
213update_status "Making doc" "$F" $start
214if [ $err != 0 ]; then
215 NUM_FAILURES=1
216 mail_on_failure "doc" ../build/$F
217fi
218
219echo "</ul>" >> $RESULT_FILE
220echo "</body>" >> $RESULT_FILE
221echo "</html>" >> $RESULT_FILE
222
223## copy results
Neal Norwitz88b78d82006-02-14 08:14:16 +0000[diff] [blame]224rsync $RSYNC_OPTS html/* $REMOTE_SYSTEM:$REMOTE_DIR
Neal Norwitz461aa5b2006-01-02 20:07:16 +0000[diff] [blame]225cd ../build
226rsync $RSYNC_OPTS index.html *.out $REMOTE_SYSTEM:$REMOTE_DIR/results/
Neal Norwitz0a903ac2006-06-27 04:26:30 +0000[diff] [blame]227