Added some reasonably thorough documentation on how to write skins.  Also added
an example skin which is referred to in the documentation, and is designed to
be a template which can be copied.


git-svn-id: svn://svn.valgrind.org/valgrind/trunk@1122 a5019735-40e9-0310-863c-91ae7b9d1cf9
diff --git a/coregrind/docs/Makefile.am b/coregrind/docs/Makefile.am
index e8a58fa..c11ba7f 100644
--- a/coregrind/docs/Makefile.am
+++ b/coregrind/docs/Makefile.am
@@ -1,5 +1,5 @@
 docdir = $(datadir)/doc/valgrind
 
-doc_DATA = index.html manual.html nav.html techdocs.html
+doc_DATA = index.html manual.html nav.html techdocs.html skins.html
 
 EXTRA_DIST = $(doc_DATA)
diff --git a/coregrind/docs/skins.html b/coregrind/docs/skins.html
new file mode 100644
index 0000000..18d2faa
--- /dev/null
+++ b/coregrind/docs/skins.html
@@ -0,0 +1,650 @@
+<html>
+  <head>
+    <style type="text/css">
+      body      { background-color: #ffffff;
+                  color:            #000000;
+                  font-family:      Times, Helvetica, Arial;
+                  font-size:        14pt}
+      h4        { margin-bottom:    0.3em}
+      code      { color:            #000000;
+                  font-family:      Courier; 
+                  font-size:        13pt }
+      pre       { color:            #000000;
+                  font-family:      Courier; 
+                  font-size:        13pt }
+      a:link    { color:            #0000C0;
+                  text-decoration:  none; }
+      a:visited { color:            #0000C0; 
+                  text-decoration:  none; }
+      a:active  { color:            #0000C0;
+                  text-decoration:  none; }
+    </style>
+    <title>Valgrind</title>
+  </head>
+
+<body bgcolor="#ffffff">
+
+<a name="title">&nbsp;</a>
+<h1 align=center>Valgrind Skins</h1>
+<center>
+  A guide to writing new skins for Valgrind<br>
+  This guide was last updated on 20020926
+</center>
+<p>
+
+<center>
+<a href="mailto:jseward@acm.org">jseward@acm.org</a><br>
+Copyright &copy; 2000-2002 Julian Seward
+<p>
+Valgrind is licensed under the GNU General Public License, 
+version 2<br>
+An open-source tool for supervising execution of Linux-x86 executables.
+</center>
+
+<p>
+
+<hr width="100%">
+<a name="contents"></a>
+<h2>Contents of this manual</h2>
+
+<h4>1&nbsp; <a href="#intro">Introduction</a></h4>
+    1.1&nbsp; <a href="#supexec">Supervised Execution</a><br>
+    1.2&nbsp; <a href="#skins">Skins</a><br>
+    1.3&nbsp; <a href="#execspaces">Execution Spaces</a><br>
+
+<h4>2&nbsp; <a href="#writingaskin">Writing a Skin</a></h4>
+    2.1&nbsp; <a href="#whywriteaskin">Why write a skin?</a><br>
+    2.2&nbsp; <a href="#howskinswork">How skins work</a><br>
+    2.3&nbsp; <a href="#gettingcode">Getting the code</a><br>
+    2.4&nbsp; <a href="#gettingstarted">Getting started</a><br>
+    2.5&nbsp; <a href="#writingcode">Writing the code</a><br>
+    2.6&nbsp; <a href="#init">Initialisation</a><br>
+    2.7&nbsp; <a href="#instr">Instrumentation</a><br>
+    2.8&nbsp; <a href="#fini">Finalisation</a><br>
+    2.9&nbsp; <a href="#otherimportantinfo">Other important information</a><br>
+    2.10&nbsp; <a href="#wordsofadvice">Words of advice</a><br>
+
+<h4>3&nbsp; <a href="#advancedtopics">Advanced Topics</a></h4>
+    3.1&nbsp; <a href="#suppressions">Suppressions</a><br>
+    3.2&nbsp; <a href="#documentation">Documentation</a><br>
+    3.3&nbsp; <a href="#regressiontests">Regression tests</a><br>
+    3.4&nbsp; <a href="#profiling">Profiling</a><br>
+    3.5&nbsp; <a href="#othermakefilehackery">Other makefile hackery</a><br>
+
+<h4>4&nbsp; <a href="#finalwords">Final Words</a></h4>
+
+<hr width="100%">
+
+<a name="intro"></a>
+<h2>1&nbsp; Introduction</h2>
+
+<a name="supexec"></a>
+<h3>1.1&nbsp; Supervised Execution</h3>
+
+Valgrind provides a generic infrastructure for supervising the execution of
+programs.  This is done by providing a way to instrument programs in very
+precise ways, making it relatively easy to support activities such as dynamic
+error detection and profiling.<p>
+
+Although writing a skin is not easy, and requires learning quite a few things
+about Valgrind, it is much easier than instrumenting a program from scratch
+yourself.
+
+<a name="skins"></a>
+<h3>1.2&nbsp; Skins</h3>
+The key idea behind Valgrind's architecture is the division between its
+``core'' and ``skins''.
+<p>
+The core provides the common low-level infrastructure to support program
+instrumentation, including the x86-to-x86 JIT compiler, low-level memory
+manager, signal handling and a scheduler (for pthreads).   It also provides
+certain services that are useful to some but not all skins, such as support
+for error recording and suppression.
+<p>
+But the core leaves certain operations undefined, which must be filled by skins.
+Most notably, skins define how program code should be instrumented.  They can
+also define certain variables to indicate to the core that they would like to
+use certain services, or be notified when certain interesting events occur.
+<p>
+Each skin that is written defines a new program supervision tool.  Writing a
+new tool just requires writing a new skin.  The core takes care of all the hard
+work.
+<p>
+
+<a name="execspaces"></a>
+<h3>1.3&nbsp; Execution Spaces</h3>
+An important concept to understand before writing a skin is that there are
+three spaces in which program code executes:
+
+<ol>
+  <li>User space: this covers most of the program's execution.  The skin is
+      given the code and can instrument it any way it likes, providing (more or
+      less) total control over the code.<p>
+
+      Code executed in user space includes all the program code, almost all of
+      the C library (including things like the dynamic linker), and almost
+      all parts of all other libraries.
+  </li><p>
+
+  <li>Core space: a small proportion of the program's execution takes place
+      entirely within Valgrind's core.  This includes:<p>
+
+      <ul>
+        <li>Dynamic memory management (<code>malloc()</code> etc.)</li>
+
+        <li>Pthread operations and scheduling</li>
+
+        <li>Signal handling</li>
+      </ul><p>
+
+      A skin has no control over these operations;  it never ``sees'' the code
+      doing this work and thus cannot instrument it.  However, the core
+      provides hooks so a skin can be notified when certain interesting events
+      happen, for example when when dynamic memory is allocated or freed, the
+      stack pointer is changed, or a pthread mutex is locked, etc.<p>
+
+      Note that these hooks only notify skins of events relevant to user 
+      space.  For example, when the core allocates some memory for its own use,
+      the skin is not notified of this, because it's not directly part of the
+      supervised program's execution.
+  </li><p>
+      
+  <li>Kernel space: execution in the kernel.  Two kinds:<p>
+   
+      <ol>
+        <li>System calls:  can't be directly observed by either the skin or the
+            core.  But the core does have some idea of what happens to the
+            arguments, and it provides hooks for a skin to wrap system calls.
+        </li><p>
+
+        <li>Other: all other kernel activity (e.g. process scheduling) is
+            totally opaque and irrelevant to the program.
+        </li><p>
+      </ol>
+  </li><p>
+
+  It should be noted that a skin only has direct control over code executed in
+  user space.  This is the vast majority of code executed, but it is not
+  absolutely all of it, so any profiling information recorded by a skin won't
+  be totally accurate.
+</ol>
+
+
+<a name="writingaskin"></a>
+<h2>2&nbsp; Writing a Skin</h2>
+
+<a name="whywriteaskin"</a>
+<h3>2.1&nbsp; Why write a skin?</h3>
+
+Before you write a skin, you should have some idea of what it should do.  What
+is it you want to know about your programs of interest?  Consider some existing
+skins:
+
+<ul>
+  <li>memcheck: among other things, performs fine-grained validity and
+      addressibility checks of every memory reference performed by the program
+      </li><p>
+
+  <li>addrcheck: performs lighterweight addressibility checks of every memory
+      reference performed by the program</li><p>
+
+  <li>cachegrind: tracks every instruction and memory reference to simulate
+      instruction and data caches, tracking cache accesses and misses that
+      occur on every line in the program</li><p>
+
+  <li>helgrind: tracks every memory access and mutex lock/unlock to determine
+      if a program contains any data races</li><p>
+
+  <li>lackey: does simple counting of various things: the number of calls to a
+      particular function (<code>_dl_runtime_resolve()</code>);  the number of
+      basic blocks, x86 instruction, UCode instructions executed;  the number
+      of branches executed and the proportion of those which were taken.</li><p>
+</ul>
+
+These examples give a reasonable idea of what kinds of things Valgrind can be
+used for.  The instrumentation can range from very lightweight (e.g. counting
+the number of times a particular functin is called) to very intrusive (e.g.
+memcheck's memory checking).
+
+<a name="howskinswork"</a>
+<h3>2.2&nbsp; How skins work</h3>
+
+Skins must define various functions for instrumenting programs that are called
+by Valgrind's core, yet they must be implemented in such a way that they can be
+written and compiled without touching Valgrind's core.  This is important,
+because one of our aims is to allow people to write and distribute their own
+skins that can be plugged into Valgrind's core easily.<p>
+
+This is achieved by packaging each skin into a separate shared object which is
+then loaded ahead of the core shared object <code>valgrind.so</code>, using the
+dynamic linker's <code>LD_PRELOAD</code> variable.  Any functions defined in
+the skin that share the name with a function defined in core (such as
+the instrumentation function <code>SK_(instrument)()</code>) override the
+core's definition.  Thus the core can call the necessary skin functions.<p>
+
+This magic is all done for you;  the shared object used is chosen with the
+<code>--skin</code> option to the <code>valgrind</code> startup script.  The
+default skin used is <code>memcheck</code>, Valgrind's original memory checker.
+
+<a name="gettingcode"</a>
+<h3>2.3&nbsp; Getting the code</h3>
+
+To write your own skin, you'll need to check out a copy of Valgrind from the
+CVS repository, rather than using a packaged distribution.  This is because it
+contains several extra files needed for writing skins.<p>
+
+To check out the code from the CVS repository, first login:
+<blockquote><code>
+cvs -d:pserver:anonymous@cvs.valgrind.sourceforge.net:/cvsroot/valgrind login
+</code></blockquote>
+
+Then checkout the code.  To get a copy of the current development version
+(recommended for the brave only):
+<blockquote><code>
+cvs -z3 -d:pserver:anonymous@cvs.valgrind.sourceforge.net:/cvsroot/valgrind co valgrind
+</code></blockquote>
+
+To get a copy of the stable released branch:
+<blockquote><code>
+cvs -z3 -d:pserver:anonymous@cvs.valgrind.sourceforge.net:/cvsroot/valgrind co -r <i>TAG</i> valgrind
+</code></blockquote>
+
+where <code><i>TAG</i></code> has the form <code>VALGRIND_X_Y_Z</code> for
+version X.Y.Z.
+
+<a name="gettingstarted"</a>
+<h3>2.4&nbsp; Getting started</h3>
+
+Valgrind uses GNU <code>automake</code> and <code>autoconf</code> for the
+creation of Makefiles and configuration.  But don't worry, these instructions
+should be enough to get you started even if you know nothing about those
+tools.<p>
+
+In what follows, all filenames are relative to Valgrind's top-level directory
+<code>valgrind/</code>.
+
+<ol>
+  <li>Choose a name for the skin, and an abbreviation that can be used as a
+      short prefix.  We'll use <code>foobar</code> and <code>fb</code> as an
+      example.
+  </li><p>
+
+  <li>Make a new directory <code>foobar/</code> which will hold the skin.
+  </li><p>
+
+  <li>Copy <code>example/Makefile.am</code> into <code>foobar/</code>.
+      Edit it by replacing all occurrences of the string
+      ``<code>example</code>'' with ``<code>foobar</code>'' and the one
+      occurrence of the string ``<code>ex_</code>'' with ``<code>fb_</code>''.
+      It might be worth trying to understand this file, at least a little;  you
+      might have to do more complicated things with it later on.  In
+      particular, the name of the <code>vgskin_foobar_so_SOURCES</code> variable
+      determines the name of the skin's shared object, which determines what
+      name must be passed to the <code>--skin</code> option to use the skin.
+  </li><p>
+
+  <li>Copy <code>example/ex_main.c</code> into
+      <code>foobar/</code>, renaming it as <code>fb_main.c</code>.
+      Edit it by changing the two lines in <code>SK_(pre_clo_init)()</code> to:
+      <pre>
+    needs->name        = "foobar";
+    needs->description = "a foobarring tool";</pre>
+
+      (setting <code>needs->description</code> appropriately).
+  </li><p>
+
+  <li>Edit <code>Makefile.am</code>, adding the new directory
+      <code>foobar</code> to the <code>SUBDIRS</code> variable.
+  </li><p>
+
+  <li>Edit <code>configure.in</code>, adding <code>foobar/Makefile</code> to the
+      <code>AC_OUTPUT</code> list.
+  </li><p>
+
+  <li>Run:
+      <pre>
+    autogen.sh
+    ./configure --prefix=`pwd`/inst
+    make install</pre>
+
+      It should automake, configure and compile without errors, putting copies
+      of the skin's shared object <code>vgskin_foobar.so</code> in
+      <code>foobar/</code> and
+      <code>inst/lib/valgrind/</code>.
+  </li><p>
+
+  <li>You can test it with a command like
+      <pre>
+    inst/bin/valgrind --skin=foobar date</pre>
+
+      (almost any program should work; <code>date</code> is just an example).  
+      The output should be something like this:
+      <pre>
+==738== foobar-1.1.0, a foobarring tool for x86 GNU/Linux.
+==738== Copyright (C) 2000-2002, and GNU GPL'd, by Julian Seward.
+==738== Estimated CPU clock rate is 1400 MHz
+==738== For more details, rerun with: -v
+==738== 
+Wed Sep 25 10:31:54 BST 2002
+==738==</pre>
+
+      The skin does nothing except run the program uninstrumented.
+  </li><p>
+</ol>
+
+These steps don't have to be followed exactly - you can choose different names
+for your source files, and use a different <code>--prefix</code> for
+<code>./configure</code>.<p>
+
+Now that we've setup, built and tested the simplest possible skin, onto the
+interesting stuff...
+
+
+<a name="writingcode"></a>
+<h3>2.5&nbsp; Writing the code</h3>
+
+A skin must define at least these four functions:
+<pre>
+    SK_(pre_clo_init)()
+    SK_(post_clo_init)()
+    SK_(instrument)()
+    SK_(fini)()
+</pre>
+
+In addition, if a skin wants to use some of the optional services provided by
+the core, it may have to define other functions.
+
+<a name="init"></a>
+<h3>2.6&nbsp; Initialisation</h3>
+
+Most of the initialisation should be done in <code>SK_(pre_clo_init)()</code>.
+Only use <code>SK_(post_clo_init)()</code> if a skin provides command line
+options and must do some initialisation after option processing takes place
+(``<code>clo</code>'' stands for ``command line options'').<p>
+
+The first argument to <code>SK_(pre_clo_init)()</code> must be initialised with
+the ``needs'' for a skin.  Of these, <code>name</code> and
+<code>description</code> are compulsory.  The rest are mostly booleans, and can
+be left untouched (they default to <code>False</code>).  They determine whether
+a skin can do various things such as:  record, report and suppress errors;
+process command line options;  wrap system calls;  record extra information
+about malloc'd blocks, etc.<p>
+
+For example, if a skin wants the core's help in recording and reporting errors,
+it must set the <code>skin_errors</code> need to <code>True</code>, and then
+provide definitions of six functions for comparing errors, printing out errors,
+reading suppressions from a suppressions file, etc.  While writing these
+functions requires some work, it's much less than doing error handling from
+scratch because the core is doing most of the work.  See the type
+<code>VgNeeds</code> in <code>include/vg_skin.h</code> for full details of all
+the needs.<p>
+
+The second argument to <code>SK_(pre_clo_init)()</code> must be initialised to
+indicate which events in core the skin wants to be notified about.  These
+include things such as blocks of memory being malloc'd, the stack pointer
+changing, a mutex being locked, etc.  If a skin wants to know about this,
+it should set the relevant pointer in the structure to point to a function,
+which will be called when that event happens.<p>
+
+For example, if the skin want to be notified when a new block of memory is
+malloc'd, it should set the <code>new_mem_heap</code> function pointer, and the
+assigned function will be called each time this happens.  See the type
+<code>VgTrackEvents</code> in <code>include/vg_skin.h</code> for full details
+of all the trackable events.<p>
+
+<a name="instr"></a>
+<h3>2.7&nbsp; Instrumentation</h3>
+
+<code>SK_(instrument)()</code> is the interesting one.  It allows you to
+instrument <i>UCode</i>, which is Valgrind's RISC-like intermediate language.
+UCode is described in the <a href="techdocs.html">technical docs</a>.
+
+The easiest way to instrument UCode is to insert calls to C functions when
+interesting things happen.  See the skin ``lackey''
+(<code>lackey/lk_main.c</code>) for a simple example of this, or
+Cachegrind (<code>cachegrind/cg_main.c</code>) for a more complex
+example.<p>
+
+A much more complicated way to instrument UCode, albeit one that might result
+in faster instrumented programs, is to extend UCode with new UCode
+instructions.  This is recommended for advanced Valgrind hackers only!  See the
+``memcheck'' skin for an example.
+
+<a name="fini"></a>
+<h3>2.8&nbsp; Finalisation</h3>
+
+This is where you can present the final results, such as a summary of the
+information collected.  Any log files should be written out at this point.
+
+<a name="otherimportantinfo"></a>
+<h3>2.9&nbsp; Other important information</h3>
+
+Please note that the core/skin split infrastructure is all very new, and not
+very well documented.  Here are some important points, but there are
+undoubtedly many others that I should note but haven't thought of.<p>
+
+The file <code>include/vg_skin.h</code> contains all the types,
+macros, functions, etc. that a skin should (hopefully) need, and is the only
+<code>.h</code> file a skin should need to <code>#include</code>.<p>
+
+In particular, you probably shouldn't use anything from the C library (there
+are deep reasons for this, trust us).  Valgrind provides an implementation of a
+reasonable subset of the C library, details of which are in
+<code>vg_skin.h</code>.<p>
+
+Similarly, when writing a skin, you shouldn't need to look at any of the code
+in Valgrind's core.  Although it might be useful sometimes to help understand
+something.<p>
+
+<code>vg_skin.h</code> has a reasonable amount of documentation in it that
+should hopefully be enough to get you going.  But ultimately, the skins
+distributed (memcheck, addrcheck, cachegrind, lackey, etc.) are probably the
+best documentation of all, for the moment.<p>
+
+Note that the <code>VG_</code> and <code>SK_</code> macros are used heavily.
+These just prepend longer strings in front of names to avoid potential
+namespace clashes.  We strongly recommend using the <code>SK_</code> macro
+for any global functions and variables in your skin.<p>
+
+<a name="wordsofadvice"</a>
+<h3>2.10&nbsp; Words of Advice</h3>
+
+Writing and debugging skins is not trivial.  Here are some suggestions for
+solving common problems.<p>
+
+If you are getting segmentation faults in C functions used by your skin, the
+usual GDB command:
+<blockquote><code>gdb <i>prog</i> core</code></blockquote>
+usually gives the location of the segmentation fault.<p>
+
+If you want to debug C functions used by your skin, you can attach GDB to
+Valgrind with some effort:
+<ul>
+  <li>Uncomment the following code in <code>coregrind/vg_main.c</code>:
+<pre>
+   /* Hook to delay things long enough so we can get the pid and
+      attach GDB in another shell. */
+#if 0
+   { 
+      Int p, q;
+      for (p = 0; p < 50000; p++)
+         for (q = 0; q < 50000; q++) ;
+   }
+#endif</pre>
+      </li><p>
+      and rebuild Valgrind.
+
+  <li>Then run: 
+      <blockquote><code>valgrind <i>prog</i></code></blockquote>
+
+      Valgrind starts the program, printing its process id, and then delays for
+      a few seconds (you may have to change the loop bounds to get a suitable
+      delay).</li><p>
+      
+  <li>In a second shell run: 
+  
+      <blockquote><code>gdb <i>prog</i> <i>pid</i></code></blockquote></li><p>
+</ul>
+
+GDB may be able to give you useful information.<p>
+
+If you just want to know whether a program point has been reached, using the
+<code>OINK</code> macro (in <code> include/vg_skin.h</code>) can be easier than
+using GDB.<p>
+
+If you are having problems with your UCode instrumentation, it's likely that
+GDB won't be able to help at all.  In this case, Valgrind's
+<code>--trace-codegen</code> option is invaluable for observing the results of
+instrumentation.<p>
+
+The other debugging command line options can be useful too (run <code>valgrind
+-h</code> for the list).<p>
+
+<a name="advancedtopics"></a>
+<h2>3&nbsp; Advanced Topics</h2>
+
+Once a skin becomes more complicated, there are some extra things you may
+want/need to do.
+
+<a name="suppressions"</a>
+<h3>3.1&nbsp; Suppressions</h3>
+
+If your skin reports errors and you want to suppress some common ones, you can
+add suppressions to the suppression files.  The relevant files are 
+<code>valgrind/*.supp</code>;  the final suppression file is aggregated from
+these files by combining the relevant <code>.supp</code> files depending on the
+versions of linux, X and glibc on a system.
+
+<a name="documentation"</a>
+<h3>3.2&nbsp; Documentation</h3>
+
+If you are feeling conscientious and want to write some HTML documentation for
+your skin, follow these steps (using <code>foobar</code> as the example skin
+name again):
+
+<ol>
+  <li>Make a directory <code>foobar/docs/</code>.
+  </li><p>
+
+  <li>Edit <code>foobar/Makefile.am</code>, adding <code>docs</code> to
+      the <code>SUBDIRS</code> variable.
+  </li><p>
+
+  <li>Edit <code>configure.in</code>, adding
+      <code>foobar/docs/Makefile</code> to the <code>AC_OUTPUT</code> list.
+  </li><p>
+
+  <li>Write <code>foobar/docs/Makefile.am</code>.  Use
+      <code>memcheck/docs/Makefile.am</code> as an example.
+  </li>
+
+  <li>Write the documentation;  the top-level file should be called
+      <code>foobar/docs/index.html</code>.
+  </li><p>
+
+  <li>(optional) Add a link in the main documentation index
+      <code>docs/index.html</code> to
+      <code>foobar/docs/index.html</code>
+  </li><p>
+</ol>
+
+<a name="regressiontests"</a>
+<h3>3.3&nbsp; Regression tests</h3>
+
+Valgrind has some support for regression tests.  If you want to write
+regression tests for your skin:
+
+<ol>
+  <li>Make a directory <code>foobar/tests/</code>.
+  </li><p>
+
+  <li>Edit <code>foobar/Makefile.am</code>, adding <code>tests</code> to
+      the <code>SUBDIRS</code> variable.
+  </li><p>
+
+  <li>Edit <code>configure.in</code>, adding
+      <code>foobar/tests/Makefile</code> to the <code>AC_OUTPUT</code> list.
+  </li><p>
+
+  <li>Write <code>foobar/tests/Makefile.am</code>.  Use
+      <code>memcheck/tests/Makefile.am</code> as an example.
+  </li><p>
+
+  <li>Write the tests, <code>.vgtest</code> test description files, 
+      <code>.stdout.exp</code> and <code>.stderr.exp</code> expected output
+      files.  (Note that Valgrind's output goes to stderr.)  Some details
+      on writing and running tests are given in the comments at the top of the
+      testing script <code>tests/vg_regtest</code>.
+  </li><p>
+
+  <li>Write a filter for stderr results <code>foobar/tests/filter_stderr</code>.
+      It can call the existing filters in <code>tests/</code>.  See
+      <code>memcheck/tests/filter_stderr</code> for an example;  in particular
+      note the <code>$dir</code> trick that ensures the filter works correctly
+      from any directory.
+  </li><p>
+</ol>
+
+<a name="profiling"</a>
+<h3>3.4&nbsp; Profiling</h3>
+
+To do simple tick-based profiling of a skin, include the line 
+<blockquote>
+#include "vg_profile.c"
+</blockquote>
+in the skin somewhere, and rebuild (you may have to <code>make clean</code>
+first).  Then run Valgrind with the <code>--profile=yes</code> option.<p>
+
+The profiler is stack-based;  you can register a profiling event with
+<code>VGP_(register_profile_event)()</code> and then use the
+<code>VGP_PUSHCC</code> and <code>VGP_POPCC</code> macros to record time spent
+doing certain things.  New profiling event numbers must not overlap with the
+core profiling event numbers.  See <code>include/vg_skin.h</code> for details
+and the ``memcheck'' skin for an example.
+
+
+<a name="othermakefilehackery"</a>
+<h3>3.5&nbsp; Other makefile hackery</h3>
+
+If you add any directories under <code>valgrind/foobar/</code>, you will
+need to add an appropriate <code>Makefile.am</code> to it, and add a
+corresponding entry to the <code>AC_OUTPUT</code> list in
+<code>valgrind/configure.in</code>.<p>
+
+If you add any scripts to your skin (see Cachegrind for an example) you need to
+add them to the <code>bin_SCRIPTS</code> variable in
+<code>valgrind/foobar/Makefile.am</code>.<p>
+
+
+<a name="finalwords"></a>
+<h2>4&nbsp; Final Words</h2>
+
+This whole core/skin business is very new and experimental, and under active
+development.<p>
+
+The first consequence of this is that the core/skin interface is quite
+immature.  It will almost certainly change in the future;  we have no intention
+of freezing it and then regretting the inevitable stupidities.  Hopefully most
+of the future changes will be to add new features, hooks, functions, etc,
+rather than to change old ones, which should cause a minimum of trouble for
+existing skins, but we can't guarantee it.  Just something to be aware of.<p>
+
+The second consequence of this is that we'd love to hear your feedback about
+it:
+
+<ul>
+  <li>If you love it or hate it</li><p>
+  <li>If you find bugs</li><p>
+  <li>If you write a skin</li><p>
+  <li>If you have suggestions for new features, needs, trackable events,
+      functions</li><p>
+  <li>If you have suggestions for making skins easier to write
+  </li><p>
+  <li>If you have suggestions for improving this documentation  </li><p>
+  <li>If you don't understand something</li><p>
+</ul>
+
+or anything else!<p>
+
+Happy programming.
+
diff --git a/docs/index.html b/docs/index.html
index 9db3347..7e31b94 100644
--- a/docs/index.html
+++ b/docs/index.html
@@ -26,8 +26,9 @@
 
   <body>
     <h2>Documentation Contents</h2>
-    <a href="../coregrind/docs/index.html"><b>Core</b></a><br>
     <a href="../memcheck/docs/index.html"><b>memcheck</b></a><br>
     <a href="../cachegrind/docs/index.html"><b>Cachegrind</b></a><br>
+    <a href="../coregrind/docs/index.html"><b>Core</b></a><br>
+    <a href="../coregrind/docs/skins.html"><b>Skins</b></a><br>
 </body>
 </html>
diff --git a/example/Makefile.am b/example/Makefile.am
new file mode 100644
index 0000000..3ba7f59
--- /dev/null
+++ b/example/Makefile.am
@@ -0,0 +1,18 @@
+
+SUBDIRS = .
+
+CFLAGS = $(WERROR) -DVG_LIBDIR="\"$(libdir)"\" \
+		-Winline -Wall -Wshadow -O -fomit-frame-pointer @PREFERRED_STACK_BOUNDARY@ -g
+
+valdir = $(libdir)/valgrind
+
+INCLUDES = -I$(top_srcdir)/include
+
+val_PROGRAMS = vgskin_example.so
+
+vgskin_example_so_SOURCES = ex_main.c
+vgskin_example_so_LDFLAGS = -shared
+
+##vgskin_example.so$(EXEEXT): $(vgskin_example_so_OBJECTS)
+##	$(CC) $(CFLAGS) $(LDFLAGS) -shared -o vgskin_example.so \
+##		$(vgskin_example_so_OBJECTS)
diff --git a/example/ex_main.c b/example/ex_main.c
new file mode 100644
index 0000000..a9d51fd
--- /dev/null
+++ b/example/ex_main.c
@@ -0,0 +1,29 @@
+
+/*--------------------------------------------------------------------*/
+/*--- An example skin.                                   ex_main.c ---*/
+/*--------------------------------------------------------------------*/
+
+#include "vg_skin.h"
+
+void SK_(pre_clo_init)(VgNeeds* needs, VgTrackEvents* track) 
+{
+   needs->name        = "example";
+   needs->description = "an example Valgrind skin";
+}
+
+void SK_(post_clo_init)(void)
+{
+}
+
+UCodeBlock* SK_(instrument)(UCodeBlock* cb, Addr a)
+{
+    return cb;
+}
+
+void SK_(fini)(void)
+{
+}
+
+/*--------------------------------------------------------------------*/
+/*--- end                                                ex_main.c ---*/
+/*--------------------------------------------------------------------*/