Blame - Misc/build.sh - platform/external/python/cpython2

blob: de997f5bd880739937b1bf649c516673e1cd82d0 [file] [log] [blame]

Neal Norwitz	461aa5b	2006-01-02 20:07:16 +0000	[diff] [blame]	1	#!/bin/sh
				2
				3	## Script to build and test the latest python from svn. It basically
				4	## does this:
				5	## svn up ; ./configure ; make ; make test ; make install ; cd Doc ; make
				6	##
				7	## Logs are kept and rsync'ed to the host. If there are test failure(s),
				8	## information about the failure(s) is mailed.
				9	##
				10	## This script is run on the PSF's machine as user neal via crontab.
				11	##
				12	## Yes, this script would probably be easier in python, but then
				13	## there's a bootstrap problem. What if Python doesn't build?
				14	##
				15	## This script should be fairly clean Bourne shell, ie not too many
				16	## bash-isms. We should try to keep it portable to other Unixes.
				17	## Even though it will probably only run on Linux. I'm sure there are
				18	## several GNU-isms currently (date +%s and readlink).
				19	##
				20	## Perhaps this script should be broken up into 2 (or more) components.
				21	## Building doc is orthogonal to the rest of the python build/test.
				22	##
				23
				24	## FIXME: we should detect test hangs (eg, if they take more than 45 minutes)
				25
				26	## FIXME: we should run valgrind
				27	## FIXME: we should run code coverage
				28
				29	## Utilities invoked in this script include:
Neal Norwitz	d3a5867	2006-01-02 23:22:41 +0000	[diff] [blame]	30	## basename, date, dirname, expr, grep, readlink, uname
Neal Norwitz	461aa5b	2006-01-02 20:07:16 +0000	[diff] [blame]	31	## cksum, make, mutt, rsync, svn
				32
Neal Norwitz	461aa5b	2006-01-02 20:07:16 +0000	[diff] [blame]	33	## remember where did we started from
				34	DIR=`dirname $0`
				35	if [ "$DIR" = "" ]; then
				36	DIR="."
				37	fi
				38
				39	## make directory absolute
				40	DIR=`readlink -f $DIR`
Neal Norwitz	d3a5867	2006-01-02 23:22:41 +0000	[diff] [blame]	41	FULLPATHNAME="$DIR/`basename $0`"
Neal Norwitz	461aa5b	2006-01-02 20:07:16 +0000	[diff] [blame]	42	## we want Misc/..
				43	DIR=`dirname $DIR`
				44
				45	## Configurable options
				46
				47	FAILURE_SUBJECT="Python Regression Test Failures"
Neal Norwitz	461aa5b	2006-01-02 20:07:16 +0000	[diff] [blame]	48	#FAILURE_MAILTO="YOUR_ACCOUNT@gmail.com"
Neal Norwitz	f415d5f	2006-02-19 18:48:19 +0000	[diff] [blame]	49	FAILURE_MAILTO="python-checkins@python.org"
Neal Norwitz	5d08bd7	2007-04-20 05:20:38 +0000	[diff] [blame]	50	#FAILURE_CC="optional--uncomment and set to desired address"
Neal Norwitz	461aa5b	2006-01-02 20:07:16 +0000	[diff] [blame]	51
				52	REMOTE_SYSTEM="neal@dinsdale.python.org"
				53	REMOTE_DIR="/data/ftp.python.org/pub/docs.python.org/dev/"
				54	RESULT_FILE="$DIR/build/index.html"
				55	INSTALL_DIR="/tmp/python-test/local"
				56	RSYNC_OPTS="-aC -e ssh"
				57
Neal Norwitz	524b59b	2006-06-27 04:09:13 +0000	[diff] [blame]	58	# Always run the installed version of Python.
				59	PYTHON=$INSTALL_DIR/bin/python
				60
				61	# Python options and regression test program that should always be run.
Neal Norwitz	f2fcfa3	2006-08-18 06:14:52 +0000	[diff] [blame]	62	REGRTEST_ARGS="-E -tt $INSTALL_DIR/lib/python2.6/test/regrtest.py"
Neal Norwitz	524b59b	2006-06-27 04:09:13 +0000	[diff] [blame]	63
Neal Norwitz	461aa5b	2006-01-02 20:07:16 +0000	[diff] [blame]	64	REFLOG="build/reflog.txt.out"
Neal Norwitz	ee6d23e	2006-04-12 05:56:00 +0000	[diff] [blame]	65	# These tests are not stable and falsely report leaks sometimes.
Neal Norwitz	f415d5f	2006-02-19 18:48:19 +0000	[diff] [blame]	66	# The entire leak report will be mailed if any test not in this list leaks.
Neal Norwitz	a1f9b7f	2006-04-06 07:58:59 +0000	[diff] [blame]	67	# Note: test_XXX (none currently) really leak, but are disabled
Neal Norwitz	770a800	2006-03-17 04:52:38 +0000	[diff] [blame]	68	# so we don't send spam. Any test which really leaks should only
				69	# be listed here if there are also test cases under Lib/test/leakers.
Neal Norwitz	df46226	2007-06-15 03:11:41 +0000	[diff] [blame]	70	LEAKY_TESTS="test_(cmd_line\|popen2\|socket\|urllib2_localnet)"
Neal Norwitz	ee6d23e	2006-04-12 05:56:00 +0000	[diff] [blame]	71
				72	# Skip these tests altogether when looking for leaks. These tests
				73	# do not need to be stored above in LEAKY_TESTS too.
				74	# test_compiler almost never finishes with the same number of refs
				75	# since it depends on other modules, skip it.
				76	# test_logging causes hangs, skip it.
				77	LEAKY_SKIPS="-x test_compiler test_logging"
Neal Norwitz	f415d5f	2006-02-19 18:48:19 +0000	[diff] [blame]	78
Neal Norwitz	770a800	2006-03-17 04:52:38 +0000	[diff] [blame]	79	# Change this flag to "yes" for old releases to only update/build the docs.
Neal Norwitz	88b78d8	2006-02-14 08:14:16 +0000	[diff] [blame]	80	BUILD_DISABLED="no"
Neal Norwitz	461aa5b	2006-01-02 20:07:16 +0000	[diff] [blame]	81
				82	## utility functions
				83	current_time() {
				84	date +%s
				85	}
				86
				87	update_status() {
				88	now=`current_time`
				89	time=`expr $now - $3`
				90	echo "<li><a href=\"$2\">$1</a> <font size=\"-1\">($time seconds)</font></li>" >> $RESULT_FILE
				91	}
				92
				93	mail_on_failure() {
				94	if [ "$NUM_FAILURES" != "0" ]; then
Neal Norwitz	5d08bd7	2007-04-20 05:20:38 +0000	[diff] [blame]	95	dest=$FAILURE_MAILTO
				96	# FAILURE_CC is optional.
				97	if [ "$FAILURE_CC" != "" ]; then
				98	dest="$dest -c $FAILURE_CC"
				99	fi
				100	mutt -s "$FAILURE_SUBJECT $1 ($NUM_FAILURES)" $dest < $2
Neal Norwitz	461aa5b	2006-01-02 20:07:16 +0000	[diff] [blame]	101	fi
				102	}
				103
				104	## setup
				105	cd $DIR
				106	mkdir -p build
				107	rm -f $RESULT_FILE build/*.out
				108	rm -rf $INSTALL_DIR
				109
				110	## create results file
				111	TITLE="Automated Python Build Results"
Neal Norwitz	b896759	2006-01-16 04:37:22 +0000	[diff] [blame]	112	echo "<html>" >> $RESULT_FILE
				113	echo " <head>" >> $RESULT_FILE
				114	echo " <title>$TITLE</title>" >> $RESULT_FILE
				115	echo " <meta http-equiv=\"refresh\" content=\"43200\">" >> $RESULT_FILE
				116	echo " </head>" >> $RESULT_FILE
Neal Norwitz	461aa5b	2006-01-02 20:07:16 +0000	[diff] [blame]	117	echo "<body>" >> $RESULT_FILE
				118	echo "<h2>Automated Python Build Results</h2>" >> $RESULT_FILE
				119	echo "<table>" >> $RESULT_FILE
				120	echo " <tr>" >> $RESULT_FILE
				121	echo " <td>Built on:</td><td>`date`</td>" >> $RESULT_FILE
				122	echo " </tr><tr>" >> $RESULT_FILE
				123	echo " <td>Hostname:</td><td>`uname -n`</td>" >> $RESULT_FILE
				124	echo " </tr><tr>" >> $RESULT_FILE
				125	echo " <td>Platform:</td><td>`uname -srmpo`</td>" >> $RESULT_FILE
				126	echo " </tr>" >> $RESULT_FILE
				127	echo "</table>" >> $RESULT_FILE
				128	echo "<ul>" >> $RESULT_FILE
				129
				130	## update, build, and test
				131	ORIG_CHECKSUM=`cksum $FULLPATHNAME`
				132	F=svn-update.out
				133	start=`current_time`
				134	svn update >& build/$F
				135	err=$?
				136	update_status "Updating" "$F" $start
Neal Norwitz	88b78d8	2006-02-14 08:14:16 +0000	[diff] [blame]	137	if [ $err = 0 -a "$BUILD_DISABLED" != "yes" ]; then
Neal Norwitz	461aa5b	2006-01-02 20:07:16 +0000	[diff] [blame]	138	## FIXME: we should check if this file has changed.
				139	## If it has changed, we should re-run the script to pick up changes.
				140	if [ "$ORIG_CHECKSUM" != "$ORIG_CHECKSUM" ]; then
				141	exec $FULLPATHNAME $@
				142	fi
				143
				144	F=svn-stat.out
				145	start=`current_time`
				146	svn stat >& build/$F
				147	## ignore some of the diffs
				148	NUM_DIFFS=`egrep -vc '^. (@test\|db_home\|Lib/test/(regrtest\.py\|db_home))$' build/$F`
				149	update_status "svn stat ($NUM_DIFFS possibly important diffs)" "$F" $start
				150
				151	F=configure.out
				152	start=`current_time`
				153	./configure --prefix=$INSTALL_DIR --with-pydebug >& build/$F
				154	err=$?
				155	update_status "Configuring" "$F" $start
				156	if [ $err = 0 ]; then
				157	F=make.out
				158	start=`current_time`
				159	make >& build/$F
				160	err=$?
				161	warnings=`grep warning build/$F \| egrep -vc "te?mpnam(_r\|)' is dangerous,"`
				162	update_status "Building ($warnings warnings)" "$F" $start
				163	if [ $err = 0 ]; then
Neal Norwitz	d19a4d4	2006-01-02 22:10:10 +0000	[diff] [blame]	164	## make install
Neal Norwitz	461aa5b	2006-01-02 20:07:16 +0000	[diff] [blame]	165	F=make-install.out
				166	start=`current_time`
				167	make install >& build/$F
				168	update_status "Installing" "$F" $start
				169
Neal Norwitz	9815f8b	2006-07-26 04:00:18 +0000	[diff] [blame]	170	if [ ! -x $PYTHON ]; then
				171	ln -s ${PYTHON}2.* $PYTHON
				172	fi
				173
Neal Norwitz	d19a4d4	2006-01-02 22:10:10 +0000	[diff] [blame]	174	## make and run basic tests
Neal Norwitz	461aa5b	2006-01-02 20:07:16 +0000	[diff] [blame]	175	F=make-test.out
				176	start=`current_time`
Neal Norwitz	524b59b	2006-06-27 04:09:13 +0000	[diff] [blame]	177	$PYTHON $REGRTEST_ARGS >& build/$F
Neal Norwitz	7941552	2006-02-15 06:07:32 +0000	[diff] [blame]	178	NUM_FAILURES=`grep -ic " failed:" build/$F`
Neal Norwitz	461aa5b	2006-01-02 20:07:16 +0000	[diff] [blame]	179	update_status "Testing basics ($NUM_FAILURES failures)" "$F" $start
Neal Norwitz	02c408d	2006-01-03 02:18:01 +0000	[diff] [blame]	180	mail_on_failure "basics" build/$F
Neal Norwitz	461aa5b	2006-01-02 20:07:16 +0000	[diff] [blame]	181
Neal Norwitz	524b59b	2006-06-27 04:09:13 +0000	[diff] [blame]	182	F=make-test-opt.out
				183	start=`current_time`
				184	$PYTHON -O $REGRTEST_ARGS >& build/$F
				185	NUM_FAILURES=`grep -ic " failed:" build/$F`
				186	update_status "Testing opt ($NUM_FAILURES failures)" "$F" $start
				187	mail_on_failure "opt" build/$F
				188
Neal Norwitz	d19a4d4	2006-01-02 22:10:10 +0000	[diff] [blame]	189	## run the tests looking for leaks
Neal Norwitz	461aa5b	2006-01-02 20:07:16 +0000	[diff] [blame]	190	F=make-test-refleak.out
				191	start=`current_time`
Neal Norwitz	0f77da3	2006-04-17 01:48:41 +0000	[diff] [blame]	192	## ensure that the reflog exists so the grep doesn't fail
				193	touch $REFLOG
Neal Norwitz	524b59b	2006-06-27 04:09:13 +0000	[diff] [blame]	194	$PYTHON $REGRTEST_ARGS -R 4:3:$REFLOG -u network $LEAKY_SKIPS >& build/$F
Neal Norwitz	d984103	2007-05-11 05:55:15 +0000	[diff] [blame]	195	NUM_FAILURES=`egrep -vc "($LEAKY_TESTS\|sum=0)" $REFLOG`
Neal Norwitz	461aa5b	2006-01-02 20:07:16 +0000	[diff] [blame]	196	update_status "Testing refleaks ($NUM_FAILURES failures)" "$F" $start
				197	mail_on_failure "refleak" $REFLOG
				198
Neal Norwitz	d19a4d4	2006-01-02 22:10:10 +0000	[diff] [blame]	199	## now try to run all the tests
Neal Norwitz	461aa5b	2006-01-02 20:07:16 +0000	[diff] [blame]	200	F=make-testall.out
				201	start=`current_time`
Neal Norwitz	d19a4d4	2006-01-02 22:10:10 +0000	[diff] [blame]	202	## skip curses when running from cron since there's no terminal
				203	## skip sound since it's not setup on the PSF box (/dev/dsp)
Neal Norwitz	524b59b	2006-06-27 04:09:13 +0000	[diff] [blame]	204	$PYTHON $REGRTEST_ARGS -uall -x test_curses test_linuxaudiodev test_ossaudiodev >& build/$F
Neal Norwitz	7941552	2006-02-15 06:07:32 +0000	[diff] [blame]	205	NUM_FAILURES=`grep -ic " failed:" build/$F`
Neal Norwitz	461aa5b	2006-01-02 20:07:16 +0000	[diff] [blame]	206	update_status "Testing all except curses and sound ($NUM_FAILURES failures)" "$F" $start
Neal Norwitz	d3a5867	2006-01-02 23:22:41 +0000	[diff] [blame]	207	mail_on_failure "all" build/$F
Neal Norwitz	461aa5b	2006-01-02 20:07:16 +0000	[diff] [blame]	208	fi
				209	fi
				210	fi
				211
				212
				213	## make doc
Neal Norwitz	524b59b	2006-06-27 04:09:13 +0000	[diff] [blame]	214	cd $DIR/Doc
Neal Norwitz	461aa5b	2006-01-02 20:07:16 +0000	[diff] [blame]	215	F="make-doc.out"
				216	start=`current_time`
Neal Norwitz	5c404ae	2007-08-16 05:16:09 +0000	[diff] [blame]	217	# XXX(nnorwitz): For now, keep the code that checks for a conflicted file until
				218	# after the first release of 2.6a1 or 3.0a1. At that point, it will be clear
				219	# if there will be a similar problem with the new doc system.
				220
Neal Norwitz	5d08bd7	2007-04-20 05:20:38 +0000	[diff] [blame]	221	# Doc/commontex/boilerplate.tex is expected to always have an outstanding
				222	# modification for the date. When a release is cut, a conflict occurs.
				223	# This allows us to detect this problem and not try to build the docs
				224	# which will definitely fail with a conflict.
Neal Norwitz	7def3e0	2007-08-16 05:07:03 +0000	[diff] [blame]	225	#CONFLICTED_FILE=commontex/boilerplate.tex
				226	#conflict_count=`grep -c "<<<" $CONFLICTED_FILE`
				227	conflict_count=0
Neal Norwitz	5d08bd7	2007-04-20 05:20:38 +0000	[diff] [blame]	228	if [ $conflict_count != 0 ]; then
				229	echo "Conflict detected in $CONFLICTED_FILE. Doc build skipped." > ../build/$F
				230	err=1
				231	else
Neal Norwitz	82955f6	2007-08-17 04:10:55 +0000	[diff] [blame^]	232	make update html >& ../build/$F
Neal Norwitz	5d08bd7	2007-04-20 05:20:38 +0000	[diff] [blame]	233	err=$?
				234	fi
Neal Norwitz	461aa5b	2006-01-02 20:07:16 +0000	[diff] [blame]	235	update_status "Making doc" "$F" $start
				236	if [ $err != 0 ]; then
				237	NUM_FAILURES=1
				238	mail_on_failure "doc" ../build/$F
				239	fi
				240
				241	echo "</ul>" >> $RESULT_FILE
				242	echo "</body>" >> $RESULT_FILE
				243	echo "</html>" >> $RESULT_FILE
				244
				245	## copy results
Neal Norwitz	7def3e0	2007-08-16 05:07:03 +0000	[diff] [blame]	246	rsync $RSYNC_OPTS build/html/* $REMOTE_SYSTEM:$REMOTE_DIR
Neal Norwitz	461aa5b	2006-01-02 20:07:16 +0000	[diff] [blame]	247	cd ../build
				248	rsync $RSYNC_OPTS index.html *.out $REMOTE_SYSTEM:$REMOTE_DIR/results/