Guido van Rossum | f06ee5f | 1996-11-27 19:52:01 +0000 | [diff] [blame] | 1 | #! /usr/bin/env python |
Guido van Rossum | b25c402 | 1994-05-27 13:32:41 +0000 | [diff] [blame] | 2 | ####################################################################### |
| 3 | # Newslist $Revision$ |
| 4 | # |
| 5 | # Syntax: |
| 6 | # newslist [ -a ] |
| 7 | # |
Tim Peters | e6ddc8b | 2004-07-18 05:56:09 +0000 | [diff] [blame] | 8 | # This is a program to create a directory full of HTML pages |
Guido van Rossum | b25c402 | 1994-05-27 13:32:41 +0000 | [diff] [blame] | 9 | # which between them contain links to all the newsgroups available |
| 10 | # on your server. |
| 11 | # |
Tim Peters | e6ddc8b | 2004-07-18 05:56:09 +0000 | [diff] [blame] | 12 | # The -a option causes a complete list of all groups to be read from |
Guido van Rossum | b25c402 | 1994-05-27 13:32:41 +0000 | [diff] [blame] | 13 | # the server rather than just the ones which have appeared since last |
| 14 | # execution. This recreates the local list from scratch. Use this on |
Guido van Rossum | e6b7979 | 1994-05-27 13:33:17 +0000 | [diff] [blame] | 15 | # the first invocation of the program, and from time to time thereafter. |
Tim Peters | e6ddc8b | 2004-07-18 05:56:09 +0000 | [diff] [blame] | 16 | # When new groups are first created they may appear on your server as |
Guido van Rossum | e6b7979 | 1994-05-27 13:33:17 +0000 | [diff] [blame] | 17 | # empty groups. By default, empty groups are ignored by the -a option. |
| 18 | # However, these new groups will not be created again, and so will not |
| 19 | # appear in the server's list of 'new groups' at a later date. Hence it |
| 20 | # won't appear until you do a '-a' after some articles have appeared. |
Tim Peters | e6ddc8b | 2004-07-18 05:56:09 +0000 | [diff] [blame] | 21 | # |
Guido van Rossum | e6b7979 | 1994-05-27 13:33:17 +0000 | [diff] [blame] | 22 | # I should really keep a list of ignored empty groups and re-check them |
| 23 | # for articles on every run, but I haven't got around to it yet. |
Guido van Rossum | b25c402 | 1994-05-27 13:32:41 +0000 | [diff] [blame] | 24 | # |
| 25 | # This assumes an NNTP news feed. |
| 26 | # |
Tim Peters | e6ddc8b | 2004-07-18 05:56:09 +0000 | [diff] [blame] | 27 | # Feel free to copy, distribute and modify this code for |
| 28 | # non-commercial use. If you make any useful modifications, let me |
Guido van Rossum | b25c402 | 1994-05-27 13:32:41 +0000 | [diff] [blame] | 29 | # know! |
| 30 | # |
| 31 | # (c) Quentin Stafford-Fraser 1994 |
| 32 | # fraser@europarc.xerox.com qs101@cl.cam.ac.uk |
| 33 | # # |
| 34 | ####################################################################### |
Georg Brandl | 22fff43 | 2009-10-27 20:19:02 +0000 | [diff] [blame] | 35 | import sys, nntplib, marshal, time, os |
Guido van Rossum | b25c402 | 1994-05-27 13:32:41 +0000 | [diff] [blame] | 36 | |
| 37 | ####################################################################### |
| 38 | # Check these variables before running! # |
| 39 | |
| 40 | # Top directory. |
| 41 | # Filenames which don't start with / are taken as being relative to this. |
Georg Brandl | 22fff43 | 2009-10-27 20:19:02 +0000 | [diff] [blame] | 42 | topdir = os.path.expanduser('~/newspage') |
Guido van Rossum | b25c402 | 1994-05-27 13:32:41 +0000 | [diff] [blame] | 43 | |
| 44 | # The name of your NNTP host |
Tim Peters | e6ddc8b | 2004-07-18 05:56:09 +0000 | [diff] [blame] | 45 | # eg. |
Guido van Rossum | e6b7979 | 1994-05-27 13:33:17 +0000 | [diff] [blame] | 46 | # newshost = 'nntp-serv.cl.cam.ac.uk' |
Tim Peters | e6ddc8b | 2004-07-18 05:56:09 +0000 | [diff] [blame] | 47 | # or use following to get the name from the NNTPSERVER environment |
Guido van Rossum | b25c402 | 1994-05-27 13:32:41 +0000 | [diff] [blame] | 48 | # variable: |
Georg Brandl | 22fff43 | 2009-10-27 20:19:02 +0000 | [diff] [blame] | 49 | # newshost = os.environ['NNTPSERVER'] |
| 50 | newshost = 'news.example.com' |
Guido van Rossum | b25c402 | 1994-05-27 13:32:41 +0000 | [diff] [blame] | 51 | |
| 52 | # The filename for a local cache of the newsgroup list |
| 53 | treefile = 'grouptree' |
| 54 | |
| 55 | # The filename for descriptions of newsgroups |
| 56 | # I found a suitable one at ftp.uu.net in /uunet-info/newgroups.gz |
| 57 | # You can set this to '' if you don't wish to use one. |
Guido van Rossum | e6b7979 | 1994-05-27 13:33:17 +0000 | [diff] [blame] | 58 | descfile = 'newsgroups' |
Guido van Rossum | b25c402 | 1994-05-27 13:32:41 +0000 | [diff] [blame] | 59 | |
| 60 | # The directory in which HTML pages should be created |
| 61 | # eg. |
| 62 | # pagedir = '/usr/local/lib/html/newspage' |
Tim Peters | e6ddc8b | 2004-07-18 05:56:09 +0000 | [diff] [blame] | 63 | # pagedir = 'pages' |
Guido van Rossum | b25c402 | 1994-05-27 13:32:41 +0000 | [diff] [blame] | 64 | pagedir = topdir |
| 65 | |
| 66 | # The html prefix which will refer to this directory |
Tim Peters | e6ddc8b | 2004-07-18 05:56:09 +0000 | [diff] [blame] | 67 | # eg. |
| 68 | # httppref = '/newspage/', |
Guido van Rossum | e6b7979 | 1994-05-27 13:33:17 +0000 | [diff] [blame] | 69 | # or leave blank for relative links between pages: (Recommended) |
| 70 | # httppref = '' |
Guido van Rossum | b25c402 | 1994-05-27 13:32:41 +0000 | [diff] [blame] | 71 | httppref = '' |
| 72 | |
Tim Peters | e6ddc8b | 2004-07-18 05:56:09 +0000 | [diff] [blame] | 73 | # The name of the 'root' news page in this directory. |
Guido van Rossum | b25c402 | 1994-05-27 13:32:41 +0000 | [diff] [blame] | 74 | # A .html suffix will be added. |
Guido van Rossum | e6b7979 | 1994-05-27 13:33:17 +0000 | [diff] [blame] | 75 | rootpage = 'root' |
Guido van Rossum | b25c402 | 1994-05-27 13:32:41 +0000 | [diff] [blame] | 76 | |
| 77 | # Set skipempty to 0 if you wish to see links to empty groups as well. |
| 78 | # Only affects the -a option. |
Guido van Rossum | e6b7979 | 1994-05-27 13:33:17 +0000 | [diff] [blame] | 79 | skipempty = 1 |
| 80 | |
| 81 | # pagelinkicon can contain html to put an icon after links to |
| 82 | # further pages. This helps to make important links stand out. |
| 83 | # Set to '' if not wanted, or '...' is quite a good one. |
Georg Brandl | 22fff43 | 2009-10-27 20:19:02 +0000 | [diff] [blame] | 84 | pagelinkicon = '... <img src="http://pelican.cl.cam.ac.uk/icons/page.xbm"> ' |
Guido van Rossum | b25c402 | 1994-05-27 13:32:41 +0000 | [diff] [blame] | 85 | |
| 86 | # --------------------------------------------------------------------- |
| 87 | # Less important personal preferences: |
| 88 | |
| 89 | # Sublistsize controls the maximum number of items the will appear as |
| 90 | # an indented sub-list before the whole thing is moved onto a different |
Tim Peters | e6ddc8b | 2004-07-18 05:56:09 +0000 | [diff] [blame] | 91 | # page. The smaller this is, the more pages you will have, but the |
Guido van Rossum | b25c402 | 1994-05-27 13:32:41 +0000 | [diff] [blame] | 92 | # shorter each will be. |
| 93 | sublistsize = 4 |
| 94 | |
| 95 | # That should be all. # |
| 96 | ####################################################################### |
| 97 | |
Guido van Rossum | 9af22a0 | 1994-08-19 15:02:57 +0000 | [diff] [blame] | 98 | for dir in os.curdir, os.environ['HOME']: |
Guido van Rossum | 4117e54 | 1998-09-14 16:44:15 +0000 | [diff] [blame] | 99 | rcfile = os.path.join(dir, '.newslistrc.py') |
| 100 | if os.path.exists(rcfile): |
Collin Winter | 6f2df4d | 2007-07-17 20:59:35 +0000 | [diff] [blame] | 101 | print(rcfile) |
Neal Norwitz | 0168802 | 2007-08-12 00:43:29 +0000 | [diff] [blame] | 102 | exec(open(rcfile).read()) |
Guido van Rossum | 4117e54 | 1998-09-14 16:44:15 +0000 | [diff] [blame] | 103 | break |
Guido van Rossum | 9af22a0 | 1994-08-19 15:02:57 +0000 | [diff] [blame] | 104 | |
Guido van Rossum | b25c402 | 1994-05-27 13:32:41 +0000 | [diff] [blame] | 105 | from nntplib import NNTP |
| 106 | from stat import * |
| 107 | |
Guido van Rossum | ba17905 | 1997-12-09 19:39:12 +0000 | [diff] [blame] | 108 | rcsrev = '$Revision$' |
Georg Brandl | 22fff43 | 2009-10-27 20:19:02 +0000 | [diff] [blame] | 109 | rcsrev = ' '.join([s for s in rcsrev.split() if '$' not in s]) |
Guido van Rossum | b25c402 | 1994-05-27 13:32:41 +0000 | [diff] [blame] | 110 | desc = {} |
| 111 | |
| 112 | # Make (possibly) relative filenames into absolute ones |
| 113 | treefile = os.path.join(topdir,treefile) |
| 114 | descfile = os.path.join(topdir,descfile) |
| 115 | page = os.path.join(topdir,pagedir) |
| 116 | |
| 117 | # First the bits for creating trees --------------------------- |
| 118 | |
| 119 | # Addtotree creates/augments a tree from a list of group names |
| 120 | def addtotree(tree, groups): |
Collin Winter | 6f2df4d | 2007-07-17 20:59:35 +0000 | [diff] [blame] | 121 | print('Updating tree...') |
Tim Peters | e6ddc8b | 2004-07-18 05:56:09 +0000 | [diff] [blame] | 122 | for i in groups: |
Georg Brandl | 22fff43 | 2009-10-27 20:19:02 +0000 | [diff] [blame] | 123 | parts = i.split('.') |
Guido van Rossum | 4117e54 | 1998-09-14 16:44:15 +0000 | [diff] [blame] | 124 | makeleaf(tree, parts) |
Guido van Rossum | b25c402 | 1994-05-27 13:32:41 +0000 | [diff] [blame] | 125 | |
| 126 | # Makeleaf makes a leaf and the branch leading to it if necessary |
| 127 | def makeleaf(tree,path): |
Tim Peters | e6ddc8b | 2004-07-18 05:56:09 +0000 | [diff] [blame] | 128 | j = path[0] |
| 129 | l = len(path) |
Guido van Rossum | b25c402 | 1994-05-27 13:32:41 +0000 | [diff] [blame] | 130 | |
Collin Winter | 6f2df4d | 2007-07-17 20:59:35 +0000 | [diff] [blame] | 131 | if j not in tree: |
Tim Peters | e6ddc8b | 2004-07-18 05:56:09 +0000 | [diff] [blame] | 132 | tree[j] = {} |
| 133 | if l == 1: |
| 134 | tree[j]['.'] = '.' |
| 135 | if l > 1: |
| 136 | makeleaf(tree[j],path[1:]) |
Guido van Rossum | b25c402 | 1994-05-27 13:32:41 +0000 | [diff] [blame] | 137 | |
Tim Peters | e6ddc8b | 2004-07-18 05:56:09 +0000 | [diff] [blame] | 138 | # Then the bits for outputting trees as pages ---------------- |
Guido van Rossum | b25c402 | 1994-05-27 13:32:41 +0000 | [diff] [blame] | 139 | |
| 140 | # Createpage creates an HTML file named <root>.html containing links |
| 141 | # to those groups beginning with <root>. |
| 142 | |
| 143 | def createpage(root, tree, p): |
Georg Brandl | 22fff43 | 2009-10-27 20:19:02 +0000 | [diff] [blame] | 144 | filename = os.path.join(pagedir, root+'.html') |
Tim Peters | e6ddc8b | 2004-07-18 05:56:09 +0000 | [diff] [blame] | 145 | if root == rootpage: |
| 146 | detail = '' |
| 147 | else: |
| 148 | detail = ' under ' + root |
Georg Brandl | 22fff43 | 2009-10-27 20:19:02 +0000 | [diff] [blame] | 149 | with open(filename, 'w') as f: |
| 150 | # f.write('Content-Type: text/html\n') |
| 151 | f.write('<html>\n<head>\n') |
| 152 | f.write('<title>Newsgroups available%s</title>\n' % detail) |
| 153 | f.write('</head>\n<body>\n') |
| 154 | f.write('<h1>Newsgroups available%s</h1>\n' % detail) |
| 155 | f.write('<a href="%s%s.html">Back to top level</a><p>\n' % |
| 156 | (httppref, rootpage)) |
| 157 | printtree(f, tree, 0, p) |
| 158 | f.write('\n<p>') |
| 159 | f.write("<i>This page automatically created by 'newslist' v. %s." % |
| 160 | rcsrev) |
| 161 | f.write(time.ctime(time.time()) + '</i>\n') |
| 162 | f.write('</body>\n</html>\n') |
Guido van Rossum | b25c402 | 1994-05-27 13:32:41 +0000 | [diff] [blame] | 163 | |
| 164 | # Printtree prints the groups as a bulleted list. Groups with |
| 165 | # more than <sublistsize> subgroups will be put on a separate page. |
| 166 | # Other sets of subgroups are just indented. |
| 167 | |
| 168 | def printtree(f, tree, indent, p): |
Tim Peters | e6ddc8b | 2004-07-18 05:56:09 +0000 | [diff] [blame] | 169 | l = len(tree) |
Guido van Rossum | b25c402 | 1994-05-27 13:32:41 +0000 | [diff] [blame] | 170 | |
Georg Brandl | 22fff43 | 2009-10-27 20:19:02 +0000 | [diff] [blame] | 171 | if l > sublistsize and indent > 0: |
Tim Peters | e6ddc8b | 2004-07-18 05:56:09 +0000 | [diff] [blame] | 172 | # Create a new page and a link to it |
Georg Brandl | 22fff43 | 2009-10-27 20:19:02 +0000 | [diff] [blame] | 173 | f.write('<li><b><a href="%s%s.html">' % (httppref, p[1:])) |
| 174 | f.write(p[1:] + '.*') |
| 175 | f.write('</a></b>%s\n' % pagelinkicon) |
Tim Peters | e6ddc8b | 2004-07-18 05:56:09 +0000 | [diff] [blame] | 176 | createpage(p[1:], tree, p) |
| 177 | return |
Guido van Rossum | b25c402 | 1994-05-27 13:32:41 +0000 | [diff] [blame] | 178 | |
Skip Montanaro | 1e8ce58 | 2007-08-06 21:07:53 +0000 | [diff] [blame] | 179 | kl = sorted(tree.keys()) |
Guido van Rossum | b25c402 | 1994-05-27 13:32:41 +0000 | [diff] [blame] | 180 | |
Tim Peters | e6ddc8b | 2004-07-18 05:56:09 +0000 | [diff] [blame] | 181 | if l > 1: |
Tim Peters | e6ddc8b | 2004-07-18 05:56:09 +0000 | [diff] [blame] | 182 | if indent > 0: |
| 183 | # Create a sub-list |
Georg Brandl | 22fff43 | 2009-10-27 20:19:02 +0000 | [diff] [blame] | 184 | f.write('<li>%s\n<ul>' % p[1:]) |
Tim Peters | e6ddc8b | 2004-07-18 05:56:09 +0000 | [diff] [blame] | 185 | else: |
| 186 | # Create a main list |
Georg Brandl | 22fff43 | 2009-10-27 20:19:02 +0000 | [diff] [blame] | 187 | f.write('<ul>') |
Tim Peters | e6ddc8b | 2004-07-18 05:56:09 +0000 | [diff] [blame] | 188 | indent = indent + 1 |
Guido van Rossum | b25c402 | 1994-05-27 13:32:41 +0000 | [diff] [blame] | 189 | |
Tim Peters | e6ddc8b | 2004-07-18 05:56:09 +0000 | [diff] [blame] | 190 | for i in kl: |
| 191 | if i == '.': |
| 192 | # Output a newsgroup |
Georg Brandl | 22fff43 | 2009-10-27 20:19:02 +0000 | [diff] [blame] | 193 | f.write('<li><a href="news:%s">%s</a> ' % (p[1:], p[1:])) |
Collin Winter | 6f2df4d | 2007-07-17 20:59:35 +0000 | [diff] [blame] | 194 | if p[1:] in desc: |
Georg Brandl | 22fff43 | 2009-10-27 20:19:02 +0000 | [diff] [blame] | 195 | f.write(' <i>%s</i>\n' % desc[p[1:]]) |
Tim Peters | e6ddc8b | 2004-07-18 05:56:09 +0000 | [diff] [blame] | 196 | else: |
| 197 | f.write('\n') |
| 198 | else: |
| 199 | # Output a hierarchy |
Georg Brandl | 22fff43 | 2009-10-27 20:19:02 +0000 | [diff] [blame] | 200 | printtree(f, tree[i], indent, p+'.'+i) |
Tim Peters | e6ddc8b | 2004-07-18 05:56:09 +0000 | [diff] [blame] | 201 | |
| 202 | if l > 1: |
Georg Brandl | 22fff43 | 2009-10-27 20:19:02 +0000 | [diff] [blame] | 203 | f.write('\n</ul>') |
Guido van Rossum | b25c402 | 1994-05-27 13:32:41 +0000 | [diff] [blame] | 204 | |
| 205 | # Reading descriptions file --------------------------------------- |
| 206 | |
Georg Brandl | 22fff43 | 2009-10-27 20:19:02 +0000 | [diff] [blame] | 207 | # This returns a dict mapping group name to its description |
Guido van Rossum | b25c402 | 1994-05-27 13:32:41 +0000 | [diff] [blame] | 208 | |
Guido van Rossum | e6b7979 | 1994-05-27 13:33:17 +0000 | [diff] [blame] | 209 | def readdesc(descfile): |
Tim Peters | e6ddc8b | 2004-07-18 05:56:09 +0000 | [diff] [blame] | 210 | global desc |
Tim Peters | e6ddc8b | 2004-07-18 05:56:09 +0000 | [diff] [blame] | 211 | desc = {} |
Guido van Rossum | b25c402 | 1994-05-27 13:32:41 +0000 | [diff] [blame] | 212 | |
Tim Peters | e6ddc8b | 2004-07-18 05:56:09 +0000 | [diff] [blame] | 213 | if descfile == '': |
Guido van Rossum | 4117e54 | 1998-09-14 16:44:15 +0000 | [diff] [blame] | 214 | return |
Guido van Rossum | b25c402 | 1994-05-27 13:32:41 +0000 | [diff] [blame] | 215 | |
Tim Peters | e6ddc8b | 2004-07-18 05:56:09 +0000 | [diff] [blame] | 216 | try: |
Georg Brandl | 22fff43 | 2009-10-27 20:19:02 +0000 | [diff] [blame] | 217 | with open(descfile, 'r') as d: |
| 218 | print('Reading descriptions...') |
| 219 | for l in d: |
| 220 | bits = l.split() |
| 221 | try: |
| 222 | grp = bits[0] |
| 223 | dsc = ' '.join(bits[1:]) |
| 224 | if len(dsc) > 1: |
| 225 | desc[grp] = dsc |
| 226 | except IndexError: |
| 227 | pass |
| 228 | except IOError: |
Collin Winter | 6f2df4d | 2007-07-17 20:59:35 +0000 | [diff] [blame] | 229 | print('Failed to open description file ' + descfile) |
Tim Peters | e6ddc8b | 2004-07-18 05:56:09 +0000 | [diff] [blame] | 230 | return |
Guido van Rossum | b25c402 | 1994-05-27 13:32:41 +0000 | [diff] [blame] | 231 | |
Guido van Rossum | e6b7979 | 1994-05-27 13:33:17 +0000 | [diff] [blame] | 232 | # Check that ouput directory exists, ------------------------------ |
| 233 | # and offer to create it if not |
Guido van Rossum | b25c402 | 1994-05-27 13:32:41 +0000 | [diff] [blame] | 234 | |
Guido van Rossum | e6b7979 | 1994-05-27 13:33:17 +0000 | [diff] [blame] | 235 | def checkopdir(pagedir): |
Tim Peters | e6ddc8b | 2004-07-18 05:56:09 +0000 | [diff] [blame] | 236 | if not os.path.isdir(pagedir): |
Georg Brandl | 22fff43 | 2009-10-27 20:19:02 +0000 | [diff] [blame] | 237 | print('Directory %s does not exist.' % pagedir) |
Collin Winter | 6f2df4d | 2007-07-17 20:59:35 +0000 | [diff] [blame] | 238 | print('Shall I create it for you? (y/n)') |
Tim Peters | e6ddc8b | 2004-07-18 05:56:09 +0000 | [diff] [blame] | 239 | if sys.stdin.readline()[0] == 'y': |
| 240 | try: |
Georg Brandl | 22fff43 | 2009-10-27 20:19:02 +0000 | [diff] [blame] | 241 | os.mkdir(pagedir, 0o777) |
Tim Peters | e6ddc8b | 2004-07-18 05:56:09 +0000 | [diff] [blame] | 242 | except: |
Collin Winter | 6f2df4d | 2007-07-17 20:59:35 +0000 | [diff] [blame] | 243 | print('Sorry - failed!') |
Tim Peters | e6ddc8b | 2004-07-18 05:56:09 +0000 | [diff] [blame] | 244 | sys.exit(1) |
| 245 | else: |
Collin Winter | 6f2df4d | 2007-07-17 20:59:35 +0000 | [diff] [blame] | 246 | print('OK. Exiting.') |
Guido van Rossum | 4117e54 | 1998-09-14 16:44:15 +0000 | [diff] [blame] | 247 | sys.exit(1) |
Guido van Rossum | b25c402 | 1994-05-27 13:32:41 +0000 | [diff] [blame] | 248 | |
Guido van Rossum | e6b7979 | 1994-05-27 13:33:17 +0000 | [diff] [blame] | 249 | # Read and write current local tree ---------------------------------- |
Guido van Rossum | b25c402 | 1994-05-27 13:32:41 +0000 | [diff] [blame] | 250 | |
Guido van Rossum | e6b7979 | 1994-05-27 13:33:17 +0000 | [diff] [blame] | 251 | def readlocallist(treefile): |
Collin Winter | 6f2df4d | 2007-07-17 20:59:35 +0000 | [diff] [blame] | 252 | print('Reading current local group list...') |
Tim Peters | e6ddc8b | 2004-07-18 05:56:09 +0000 | [diff] [blame] | 253 | tree = {} |
| 254 | try: |
| 255 | treetime = time.localtime(os.stat(treefile)[ST_MTIME]) |
| 256 | except: |
Collin Winter | 6f2df4d | 2007-07-17 20:59:35 +0000 | [diff] [blame] | 257 | print('\n*** Failed to open local group cache '+treefile) |
| 258 | print('If this is the first time you have run newslist, then') |
| 259 | print('use the -a option to create it.') |
Tim Peters | e6ddc8b | 2004-07-18 05:56:09 +0000 | [diff] [blame] | 260 | sys.exit(1) |
Georg Brandl | 22fff43 | 2009-10-27 20:19:02 +0000 | [diff] [blame] | 261 | treedate = '%02d%02d%02d' % (treetime[0] % 100, treetime[1], treetime[2]) |
Tim Peters | e6ddc8b | 2004-07-18 05:56:09 +0000 | [diff] [blame] | 262 | try: |
Georg Brandl | 22fff43 | 2009-10-27 20:19:02 +0000 | [diff] [blame] | 263 | with open(treefile, 'rb') as dump: |
| 264 | tree = marshal.load(dump) |
| 265 | except IOError: |
Collin Winter | 6f2df4d | 2007-07-17 20:59:35 +0000 | [diff] [blame] | 266 | print('Cannot open local group list ' + treefile) |
Tim Peters | e6ddc8b | 2004-07-18 05:56:09 +0000 | [diff] [blame] | 267 | return (tree, treedate) |
Guido van Rossum | b25c402 | 1994-05-27 13:32:41 +0000 | [diff] [blame] | 268 | |
Guido van Rossum | e6b7979 | 1994-05-27 13:33:17 +0000 | [diff] [blame] | 269 | def writelocallist(treefile, tree): |
Tim Peters | e6ddc8b | 2004-07-18 05:56:09 +0000 | [diff] [blame] | 270 | try: |
Georg Brandl | 22fff43 | 2009-10-27 20:19:02 +0000 | [diff] [blame] | 271 | with open(treefile, 'wb') as dump: |
| 272 | groups = marshal.dump(tree, dump) |
| 273 | print('Saved list to %s\n' % treefile) |
Tim Peters | e6ddc8b | 2004-07-18 05:56:09 +0000 | [diff] [blame] | 274 | except: |
Georg Brandl | 22fff43 | 2009-10-27 20:19:02 +0000 | [diff] [blame] | 275 | print('Sorry - failed to write to local group cache', treefile) |
Collin Winter | 6f2df4d | 2007-07-17 20:59:35 +0000 | [diff] [blame] | 276 | print('Does it (or its directory) have the correct permissions?') |
Tim Peters | e6ddc8b | 2004-07-18 05:56:09 +0000 | [diff] [blame] | 277 | sys.exit(1) |
Guido van Rossum | e6b7979 | 1994-05-27 13:33:17 +0000 | [diff] [blame] | 278 | |
| 279 | # Return list of all groups on server ----------------------------- |
| 280 | |
| 281 | def getallgroups(server): |
Collin Winter | 6f2df4d | 2007-07-17 20:59:35 +0000 | [diff] [blame] | 282 | print('Getting list of all groups...') |
Georg Brandl | 22fff43 | 2009-10-27 20:19:02 +0000 | [diff] [blame] | 283 | treedate = '010101' |
Tim Peters | e6ddc8b | 2004-07-18 05:56:09 +0000 | [diff] [blame] | 284 | info = server.list()[1] |
| 285 | groups = [] |
Collin Winter | 6f2df4d | 2007-07-17 20:59:35 +0000 | [diff] [blame] | 286 | print('Processing...') |
Tim Peters | e6ddc8b | 2004-07-18 05:56:09 +0000 | [diff] [blame] | 287 | if skipempty: |
Collin Winter | 6f2df4d | 2007-07-17 20:59:35 +0000 | [diff] [blame] | 288 | print('\nIgnoring following empty groups:') |
Tim Peters | e6ddc8b | 2004-07-18 05:56:09 +0000 | [diff] [blame] | 289 | for i in info: |
Georg Brandl | 22fff43 | 2009-10-27 20:19:02 +0000 | [diff] [blame] | 290 | grpname = i[0].split()[0] |
| 291 | if skipempty and int(i[1]) < int(i[2]): |
| 292 | print(grpname.decode() + ' ', end=' ') |
Tim Peters | e6ddc8b | 2004-07-18 05:56:09 +0000 | [diff] [blame] | 293 | else: |
Georg Brandl | 22fff43 | 2009-10-27 20:19:02 +0000 | [diff] [blame] | 294 | groups.append(grpname.decode()) |
Collin Winter | 6f2df4d | 2007-07-17 20:59:35 +0000 | [diff] [blame] | 295 | print('\n') |
Tim Peters | e6ddc8b | 2004-07-18 05:56:09 +0000 | [diff] [blame] | 296 | if skipempty: |
Collin Winter | 6f2df4d | 2007-07-17 20:59:35 +0000 | [diff] [blame] | 297 | print('(End of empty groups)') |
Tim Peters | e6ddc8b | 2004-07-18 05:56:09 +0000 | [diff] [blame] | 298 | return groups |
Guido van Rossum | e6b7979 | 1994-05-27 13:33:17 +0000 | [diff] [blame] | 299 | |
| 300 | # Return list of new groups on server ----------------------------- |
| 301 | |
| 302 | def getnewgroups(server, treedate): |
Georg Brandl | 22fff43 | 2009-10-27 20:19:02 +0000 | [diff] [blame] | 303 | print('Getting list of new groups since start of %s...' % treedate, end=' ') |
| 304 | info = server.newgroups(treedate, '000001')[1] |
Collin Winter | 6f2df4d | 2007-07-17 20:59:35 +0000 | [diff] [blame] | 305 | print('got %d.' % len(info)) |
| 306 | print('Processing...', end=' ') |
Tim Peters | e6ddc8b | 2004-07-18 05:56:09 +0000 | [diff] [blame] | 307 | groups = [] |
| 308 | for i in info: |
Georg Brandl | 22fff43 | 2009-10-27 20:19:02 +0000 | [diff] [blame] | 309 | grpname = i.split()[0] |
| 310 | groups.append(grpname.decode()) |
Collin Winter | 6f2df4d | 2007-07-17 20:59:35 +0000 | [diff] [blame] | 311 | print('Done') |
Tim Peters | e6ddc8b | 2004-07-18 05:56:09 +0000 | [diff] [blame] | 312 | return groups |
Guido van Rossum | e6b7979 | 1994-05-27 13:33:17 +0000 | [diff] [blame] | 313 | |
| 314 | # Now the main program -------------------------------------------- |
| 315 | |
| 316 | def main(): |
Georg Brandl | 22fff43 | 2009-10-27 20:19:02 +0000 | [diff] [blame] | 317 | tree = {} |
Guido van Rossum | e6b7979 | 1994-05-27 13:33:17 +0000 | [diff] [blame] | 318 | |
Tim Peters | e6ddc8b | 2004-07-18 05:56:09 +0000 | [diff] [blame] | 319 | # Check that the output directory exists |
Benjamin Peterson | d7b0328 | 2008-09-13 15:58:53 +0000 | [diff] [blame] | 320 | checkopdir(pagedir) |
Guido van Rossum | e6b7979 | 1994-05-27 13:33:17 +0000 | [diff] [blame] | 321 | |
Tim Peters | e6ddc8b | 2004-07-18 05:56:09 +0000 | [diff] [blame] | 322 | try: |
Georg Brandl | 22fff43 | 2009-10-27 20:19:02 +0000 | [diff] [blame] | 323 | print('Connecting to %s...' % newshost) |
Tim Peters | e6ddc8b | 2004-07-18 05:56:09 +0000 | [diff] [blame] | 324 | if sys.version[0] == '0': |
| 325 | s = NNTP.init(newshost) |
| 326 | else: |
| 327 | s = NNTP(newshost) |
Georg Brandl | 22fff43 | 2009-10-27 20:19:02 +0000 | [diff] [blame] | 328 | connected = True |
Guido van Rossum | b940e11 | 2007-01-10 16:19:56 +0000 | [diff] [blame] | 329 | except (nntplib.error_temp, nntplib.error_perm) as x: |
Collin Winter | 6f2df4d | 2007-07-17 20:59:35 +0000 | [diff] [blame] | 330 | print('Error connecting to host:', x) |
| 331 | print('I\'ll try to use just the local list.') |
Georg Brandl | 22fff43 | 2009-10-27 20:19:02 +0000 | [diff] [blame] | 332 | connected = False |
Guido van Rossum | e6b7979 | 1994-05-27 13:33:17 +0000 | [diff] [blame] | 333 | |
Tim Peters | e6ddc8b | 2004-07-18 05:56:09 +0000 | [diff] [blame] | 334 | # If -a is specified, read the full list of groups from server |
| 335 | if connected and len(sys.argv) > 1 and sys.argv[1] == '-a': |
Tim Peters | e6ddc8b | 2004-07-18 05:56:09 +0000 | [diff] [blame] | 336 | groups = getallgroups(s) |
Guido van Rossum | e6b7979 | 1994-05-27 13:33:17 +0000 | [diff] [blame] | 337 | |
Tim Peters | e6ddc8b | 2004-07-18 05:56:09 +0000 | [diff] [blame] | 338 | # Otherwise just read the local file and then add |
| 339 | # groups created since local file last modified. |
| 340 | else: |
Guido van Rossum | e6b7979 | 1994-05-27 13:33:17 +0000 | [diff] [blame] | 341 | |
Tim Peters | e6ddc8b | 2004-07-18 05:56:09 +0000 | [diff] [blame] | 342 | (tree, treedate) = readlocallist(treefile) |
| 343 | if connected: |
| 344 | groups = getnewgroups(s, treedate) |
Guido van Rossum | b25c402 | 1994-05-27 13:32:41 +0000 | [diff] [blame] | 345 | |
Tim Peters | e6ddc8b | 2004-07-18 05:56:09 +0000 | [diff] [blame] | 346 | if connected: |
| 347 | addtotree(tree, groups) |
| 348 | writelocallist(treefile,tree) |
Guido van Rossum | e6b7979 | 1994-05-27 13:33:17 +0000 | [diff] [blame] | 349 | |
Tim Peters | e6ddc8b | 2004-07-18 05:56:09 +0000 | [diff] [blame] | 350 | # Read group descriptions |
| 351 | readdesc(descfile) |
| 352 | |
Collin Winter | 6f2df4d | 2007-07-17 20:59:35 +0000 | [diff] [blame] | 353 | print('Creating pages...') |
Tim Peters | e6ddc8b | 2004-07-18 05:56:09 +0000 | [diff] [blame] | 354 | createpage(rootpage, tree, '') |
Collin Winter | 6f2df4d | 2007-07-17 20:59:35 +0000 | [diff] [blame] | 355 | print('Done') |
Guido van Rossum | b25c402 | 1994-05-27 13:32:41 +0000 | [diff] [blame] | 356 | |
Johannes Gijsbers | 7a8c43e | 2004-09-11 16:34:35 +0000 | [diff] [blame] | 357 | if __name__ == "__main__": |
| 358 | main() |
Guido van Rossum | b25c402 | 1994-05-27 13:32:41 +0000 | [diff] [blame] | 359 | |
| 360 | # That's all folks |
| 361 | ###################################################################### |