blob: bcf7d1bd3363bd9c06507828e0e298b4ff8ec833 [file] [log] [blame]
Antoine Pitrou69ab9512010-09-29 15:03:40 +00001"""An NNTP client class based on:
2- RFC 977: Network News Transfer Protocol
3- RFC 2980: Common NNTP Extensions
4- RFC 3977: Network News Transfer Protocol (version 2)
Guido van Rossumc629d341992-11-05 10:43:02 +00005
Guido van Rossum54f22ed2000-02-04 15:10:34 +00006Example:
Guido van Rossumc629d341992-11-05 10:43:02 +00007
Guido van Rossum54f22ed2000-02-04 15:10:34 +00008>>> from nntplib import NNTP
9>>> s = NNTP('news')
10>>> resp, count, first, last, name = s.group('comp.lang.python')
Guido van Rossum7131f842007-02-09 20:13:25 +000011>>> print('Group', name, 'has', count, 'articles, range', first, 'to', last)
Guido van Rossum54f22ed2000-02-04 15:10:34 +000012Group comp.lang.python has 51 articles, range 5770 to 5821
Christian Heimes933238a2008-11-05 19:44:21 +000013>>> resp, subs = s.xhdr('subject', '{0}-{1}'.format(first, last))
Guido van Rossum54f22ed2000-02-04 15:10:34 +000014>>> resp = s.quit()
15>>>
Guido van Rossumc629d341992-11-05 10:43:02 +000016
Guido van Rossum54f22ed2000-02-04 15:10:34 +000017Here 'resp' is the server response line.
18Error responses are turned into exceptions.
19
20To post an article from a file:
Christian Heimes933238a2008-11-05 19:44:21 +000021>>> f = open(filename, 'rb') # file containing article, including header
Guido van Rossum54f22ed2000-02-04 15:10:34 +000022>>> resp = s.post(f)
23>>>
24
25For descriptions of all methods, read the comments in the code below.
26Note that all arguments and return values representing article numbers
27are strings, not numbers, since they are rarely used for calculations.
28"""
29
30# RFC 977 by Brian Kantor and Phil Lapsley.
31# xover, xgtitle, xpath, date methods by Kevan Heydon
Guido van Rossum8421c4e1995-09-22 00:52:38 +000032
Antoine Pitrou69ab9512010-09-29 15:03:40 +000033# Incompatible changes from the 2.x nntplib:
34# - all commands are encoded as UTF-8 data (using the "surrogateescape"
35# error handler), except for raw message data (POST, IHAVE)
36# - all responses are decoded as UTF-8 data (using the "surrogateescape"
37# error handler), except for raw message data (ARTICLE, HEAD, BODY)
38# - the `file` argument to various methods is keyword-only
39#
40# - NNTP.date() returns a datetime object
41# - NNTP.newgroups() and NNTP.newnews() take a datetime (or date) object,
42# rather than a pair of (date, time) strings.
43# - NNTP.newgroups() and NNTP.list() return a list of GroupInfo named tuples
44# - NNTP.descriptions() returns a dict mapping group names to descriptions
45# - NNTP.xover() returns a list of dicts mapping field names (header or metadata)
46# to field values; each dict representing a message overview.
47# - NNTP.article(), NNTP.head() and NNTP.body() return a (response, ArticleInfo)
48# tuple.
49# - the "internal" methods have been marked private (they now start with
50# an underscore)
51
52# Other changes from the 2.x/3.1 nntplib:
53# - automatic querying of capabilities at connect
54# - New method NNTP.getcapabilities()
55# - New method NNTP.over()
56# - New helper function decode_header()
57# - NNTP.post() and NNTP.ihave() accept file objects, bytes-like objects and
58# arbitrary iterables yielding lines.
59# - An extensive test suite :-)
60
61# TODO:
62# - return structured data (GroupInfo etc.) everywhere
63# - support HDR
Guido van Rossumc629d341992-11-05 10:43:02 +000064
65# Imports
Guido van Rossum9694fca1997-10-22 21:00:49 +000066import re
Guido van Rossumc629d341992-11-05 10:43:02 +000067import socket
Antoine Pitrou69ab9512010-09-29 15:03:40 +000068import collections
69import datetime
70import warnings
Guido van Rossumc629d341992-11-05 10:43:02 +000071
Antoine Pitrou1cb121e2010-11-09 18:54:37 +000072try:
73 import ssl
Brett Cannoncd171c82013-07-04 17:43:24 -040074except ImportError:
Antoine Pitrou1cb121e2010-11-09 18:54:37 +000075 _have_ssl = False
76else:
77 _have_ssl = True
78
Antoine Pitrou69ab9512010-09-29 15:03:40 +000079from email.header import decode_header as _email_decode_header
80from socket import _GLOBAL_DEFAULT_TIMEOUT
81
82__all__ = ["NNTP",
Berker Peksag96756b62014-09-20 08:53:05 +030083 "NNTPError", "NNTPReplyError", "NNTPTemporaryError",
84 "NNTPPermanentError", "NNTPProtocolError", "NNTPDataError",
Antoine Pitrou69ab9512010-09-29 15:03:40 +000085 "decode_header",
86 ]
Tim Peters2344fae2001-01-15 00:50:52 +000087
Georg Brandl28e78412013-10-27 07:29:47 +010088# maximal line length when calling readline(). This is to prevent
Berker Peksag740c7302014-07-09 20:15:28 +030089# reading arbitrary length lines. RFC 3977 limits NNTP line length to
Georg Brandl28e78412013-10-27 07:29:47 +010090# 512 characters, including CRLF. We have selected 2048 just to be on
91# the safe side.
92_MAXLINE = 2048
93
94
Barry Warsaw9dd78722000-02-10 20:25:53 +000095# Exceptions raised when an error or invalid response is received
96class NNTPError(Exception):
Tim Peters2344fae2001-01-15 00:50:52 +000097 """Base class for all nntplib exceptions"""
98 def __init__(self, *args):
Guido van Rossum68468eb2003-02-27 20:14:51 +000099 Exception.__init__(self, *args)
Tim Peters2344fae2001-01-15 00:50:52 +0000100 try:
101 self.response = args[0]
102 except IndexError:
103 self.response = 'No response given'
Barry Warsaw9dd78722000-02-10 20:25:53 +0000104
105class NNTPReplyError(NNTPError):
Tim Peters2344fae2001-01-15 00:50:52 +0000106 """Unexpected [123]xx reply"""
107 pass
Barry Warsaw9dd78722000-02-10 20:25:53 +0000108
109class NNTPTemporaryError(NNTPError):
Tim Peters2344fae2001-01-15 00:50:52 +0000110 """4xx errors"""
111 pass
Barry Warsaw9dd78722000-02-10 20:25:53 +0000112
113class NNTPPermanentError(NNTPError):
Tim Peters2344fae2001-01-15 00:50:52 +0000114 """5xx errors"""
115 pass
Barry Warsaw9dd78722000-02-10 20:25:53 +0000116
117class NNTPProtocolError(NNTPError):
Tim Peters2344fae2001-01-15 00:50:52 +0000118 """Response does not begin with [1-5]"""
119 pass
Barry Warsaw9dd78722000-02-10 20:25:53 +0000120
121class NNTPDataError(NNTPError):
Tim Peters2344fae2001-01-15 00:50:52 +0000122 """Error in response data"""
123 pass
Barry Warsaw9dd78722000-02-10 20:25:53 +0000124
Tim Peters2344fae2001-01-15 00:50:52 +0000125
Guido van Rossumc629d341992-11-05 10:43:02 +0000126# Standard port used by NNTP servers
127NNTP_PORT = 119
Antoine Pitrou1cb121e2010-11-09 18:54:37 +0000128NNTP_SSL_PORT = 563
Guido van Rossumc629d341992-11-05 10:43:02 +0000129
130# Response numbers that are followed by additional text (e.g. article)
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000131_LONGRESP = {
132 '100', # HELP
133 '101', # CAPABILITIES
134 '211', # LISTGROUP (also not multi-line with GROUP)
135 '215', # LIST
136 '220', # ARTICLE
137 '221', # HEAD, XHDR
138 '222', # BODY
139 '224', # OVER, XOVER
140 '225', # HDR
141 '230', # NEWNEWS
142 '231', # NEWGROUPS
143 '282', # XGTITLE
144}
Guido van Rossumc629d341992-11-05 10:43:02 +0000145
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000146# Default decoded value for LIST OVERVIEW.FMT if not supported
147_DEFAULT_OVERVIEW_FMT = [
148 "subject", "from", "date", "message-id", "references", ":bytes", ":lines"]
149
150# Alternative names allowed in LIST OVERVIEW.FMT response
151_OVERVIEW_FMT_ALTERNATIVES = {
152 'bytes': ':bytes',
153 'lines': ':lines',
154}
Guido van Rossumc629d341992-11-05 10:43:02 +0000155
156# Line terminators (we always output CRLF, but accept any of CRLF, CR, LF)
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000157_CRLF = b'\r\n'
158
159GroupInfo = collections.namedtuple('GroupInfo',
160 ['group', 'last', 'first', 'flag'])
161
162ArticleInfo = collections.namedtuple('ArticleInfo',
163 ['number', 'message_id', 'lines'])
Guido van Rossumc629d341992-11-05 10:43:02 +0000164
165
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000166# Helper function(s)
167def decode_header(header_str):
168 """Takes an unicode string representing a munged header value
169 and decodes it as a (possibly non-ASCII) readable value."""
170 parts = []
171 for v, enc in _email_decode_header(header_str):
172 if isinstance(v, bytes):
173 parts.append(v.decode(enc or 'ascii'))
174 else:
175 parts.append(v)
R David Murray07ea53c2012-06-02 17:56:49 -0400176 return ''.join(parts)
Tim Peters2344fae2001-01-15 00:50:52 +0000177
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000178def _parse_overview_fmt(lines):
179 """Parse a list of string representing the response to LIST OVERVIEW.FMT
180 and return a list of header/metadata names.
181 Raises NNTPDataError if the response is not compliant
182 (cf. RFC 3977, section 8.4)."""
183 fmt = []
184 for line in lines:
185 if line[0] == ':':
186 # Metadata name (e.g. ":bytes")
187 name, _, suffix = line[1:].partition(':')
188 name = ':' + name
189 else:
190 # Header name (e.g. "Subject:" or "Xref:full")
191 name, _, suffix = line.partition(':')
192 name = name.lower()
193 name = _OVERVIEW_FMT_ALTERNATIVES.get(name, name)
194 # Should we do something with the suffix?
195 fmt.append(name)
196 defaults = _DEFAULT_OVERVIEW_FMT
197 if len(fmt) < len(defaults):
198 raise NNTPDataError("LIST OVERVIEW.FMT response too short")
199 if fmt[:len(defaults)] != defaults:
200 raise NNTPDataError("LIST OVERVIEW.FMT redefines default fields")
201 return fmt
202
203def _parse_overview(lines, fmt, data_process_func=None):
204 """Parse the response to a OVER or XOVER command according to the
205 overview format `fmt`."""
206 n_defaults = len(_DEFAULT_OVERVIEW_FMT)
207 overview = []
208 for line in lines:
209 fields = {}
210 article_number, *tokens = line.split('\t')
211 article_number = int(article_number)
212 for i, token in enumerate(tokens):
213 if i >= len(fmt):
214 # XXX should we raise an error? Some servers might not
215 # support LIST OVERVIEW.FMT and still return additional
216 # headers.
217 continue
218 field_name = fmt[i]
219 is_metadata = field_name.startswith(':')
220 if i >= n_defaults and not is_metadata:
221 # Non-default header names are included in full in the response
Antoine Pitrou4103bc02010-11-03 18:18:43 +0000222 # (unless the field is totally empty)
223 h = field_name + ": "
224 if token and token[:len(h)].lower() != h:
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000225 raise NNTPDataError("OVER/XOVER response doesn't include "
226 "names of additional headers")
Antoine Pitrou4103bc02010-11-03 18:18:43 +0000227 token = token[len(h):] if token else None
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000228 fields[fmt[i]] = token
229 overview.append((article_number, fields))
230 return overview
231
232def _parse_datetime(date_str, time_str=None):
233 """Parse a pair of (date, time) strings, and return a datetime object.
234 If only the date is given, it is assumed to be date and time
235 concatenated together (e.g. response to the DATE command).
236 """
237 if time_str is None:
238 time_str = date_str[-6:]
239 date_str = date_str[:-6]
240 hours = int(time_str[:2])
241 minutes = int(time_str[2:4])
242 seconds = int(time_str[4:])
243 year = int(date_str[:-4])
244 month = int(date_str[-4:-2])
245 day = int(date_str[-2:])
246 # RFC 3977 doesn't say how to interpret 2-char years. Assume that
247 # there are no dates before 1970 on Usenet.
248 if year < 70:
249 year += 2000
250 elif year < 100:
251 year += 1900
252 return datetime.datetime(year, month, day, hours, minutes, seconds)
253
254def _unparse_datetime(dt, legacy=False):
255 """Format a date or datetime object as a pair of (date, time) strings
256 in the format required by the NEWNEWS and NEWGROUPS commands. If a
257 date object is passed, the time is assumed to be midnight (00h00).
258
259 The returned representation depends on the legacy flag:
260 * if legacy is False (the default):
261 date has the YYYYMMDD format and time the HHMMSS format
262 * if legacy is True:
263 date has the YYMMDD format and time the HHMMSS format.
264 RFC 3977 compliant servers should understand both formats; therefore,
265 legacy is only needed when talking to old servers.
266 """
267 if not isinstance(dt, datetime.datetime):
268 time_str = "000000"
269 else:
270 time_str = "{0.hour:02d}{0.minute:02d}{0.second:02d}".format(dt)
271 y = dt.year
272 if legacy:
273 y = y % 100
274 date_str = "{0:02d}{1.month:02d}{1.day:02d}".format(y, dt)
275 else:
276 date_str = "{0:04d}{1.month:02d}{1.day:02d}".format(y, dt)
277 return date_str, time_str
278
279
Antoine Pitrou1cb121e2010-11-09 18:54:37 +0000280if _have_ssl:
281
Christian Heimes216d4632013-12-02 20:20:11 +0100282 def _encrypt_on(sock, context, hostname):
Antoine Pitrou1cb121e2010-11-09 18:54:37 +0000283 """Wrap a socket in SSL/TLS. Arguments:
284 - sock: Socket to wrap
285 - context: SSL context to use for the encrypted connection
286 Returns:
287 - sock: New, encrypted socket.
288 """
289 # Generate a default SSL context if none was passed.
290 if context is None:
Christian Heimes67986f92013-11-23 22:43:47 +0100291 context = ssl._create_stdlib_context()
Benjamin Peterson7243b572014-11-23 17:04:34 -0600292 return context.wrap_socket(sock, server_hostname=hostname)
Antoine Pitrou1cb121e2010-11-09 18:54:37 +0000293
294
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000295# The classes themselves
296class _NNTPBase:
297 # UTF-8 is the character set for all NNTP commands and responses: they
298 # are automatically encoded (when sending) and decoded (and receiving)
299 # by this class.
300 # However, some multi-line data blocks can contain arbitrary bytes (for
301 # example, latin-1 or utf-16 data in the body of a message). Commands
302 # taking (POST, IHAVE) or returning (HEAD, BODY, ARTICLE) raw message
303 # data will therefore only accept and produce bytes objects.
304 # Furthermore, since there could be non-compliant servers out there,
305 # we use 'surrogateescape' as the error handler for fault tolerance
306 # and easy round-tripping. This could be useful for some applications
307 # (e.g. NNTP gateways).
308
309 encoding = 'utf-8'
310 errors = 'surrogateescape'
311
Antoine Pitrou1cb121e2010-11-09 18:54:37 +0000312 def __init__(self, file, host,
313 readermode=None, timeout=_GLOBAL_DEFAULT_TIMEOUT):
Tim Peters2344fae2001-01-15 00:50:52 +0000314 """Initialize an instance. Arguments:
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000315 - file: file-like object (open for read/write in binary mode)
Antoine Pitrou859c4ef2010-11-09 18:58:42 +0000316 - host: hostname of the server
Tim Peters2344fae2001-01-15 00:50:52 +0000317 - readermode: if true, send 'mode reader' command after
318 connecting.
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000319 - timeout: timeout (in seconds) used for socket connections
Guido van Rossum54f22ed2000-02-04 15:10:34 +0000320
Tim Peters2344fae2001-01-15 00:50:52 +0000321 readermode is sometimes necessary if you are connecting to an
322 NNTP server on the local machine and intend to call
Ezio Melotti42da6632011-03-15 05:18:48 +0200323 reader-specific commands, such as `group'. If you get
Tim Peters2344fae2001-01-15 00:50:52 +0000324 unexpected NNTPPermanentErrors, you might need to set
325 readermode.
326 """
Antoine Pitrou1cb121e2010-11-09 18:54:37 +0000327 self.host = host
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000328 self.file = file
Tim Peters2344fae2001-01-15 00:50:52 +0000329 self.debugging = 0
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000330 self.welcome = self._getresp()
Tim Petersdfb673b2001-01-16 07:12:46 +0000331
Antoine Pitrou71135622012-02-14 23:29:34 +0100332 # Inquire about capabilities (RFC 3977).
333 self._caps = None
334 self.getcapabilities()
335
Antoine Pitrou1cb121e2010-11-09 18:54:37 +0000336 # 'MODE READER' is sometimes necessary to enable 'reader' mode.
337 # However, the order in which 'MODE READER' and 'AUTHINFO' need to
338 # arrive differs between some NNTP servers. If _setreadermode() fails
339 # with an authorization failed error, it will set this to True;
340 # the login() routine will interpret that as a request to try again
341 # after performing its normal function.
Antoine Pitrou71135622012-02-14 23:29:34 +0100342 # Enable only if we're not already in READER mode anyway.
Antoine Pitrou1cb121e2010-11-09 18:54:37 +0000343 self.readermode_afterauth = False
Antoine Pitrou71135622012-02-14 23:29:34 +0100344 if readermode and 'READER' not in self._caps:
Antoine Pitrou1cb121e2010-11-09 18:54:37 +0000345 self._setreadermode()
Antoine Pitrou71135622012-02-14 23:29:34 +0100346 if not self.readermode_afterauth:
347 # Capabilities might have changed after MODE READER
348 self._caps = None
349 self.getcapabilities()
Tim Petersdfb673b2001-01-16 07:12:46 +0000350
Antoine Pitrou1cb121e2010-11-09 18:54:37 +0000351 # RFC 4642 2.2.2: Both the client and the server MUST know if there is
352 # a TLS session active. A client MUST NOT attempt to start a TLS
353 # session if a TLS session is already active.
354 self.tls_on = False
355
Antoine Pitrou1cb121e2010-11-09 18:54:37 +0000356 # Log in and encryption setup order is left to subclasses.
357 self.authenticated = False
Guido van Rossumc629d341992-11-05 10:43:02 +0000358
Giampaolo Rodolà424298a2011-03-03 18:34:06 +0000359 def __enter__(self):
360 return self
361
362 def __exit__(self, *args):
363 is_connected = lambda: hasattr(self, "file")
364 if is_connected():
365 try:
366 self.quit()
Andrew Svetlov0832af62012-12-18 23:10:48 +0200367 except (OSError, EOFError):
Giampaolo Rodolà424298a2011-03-03 18:34:06 +0000368 pass
369 finally:
370 if is_connected():
371 self._close()
372
Tim Peters2344fae2001-01-15 00:50:52 +0000373 def getwelcome(self):
374 """Get the welcome message from the server
375 (this is read and squirreled away by __init__()).
376 If the response code is 200, posting is allowed;
377 if it 201, posting is not allowed."""
Guido van Rossum54f22ed2000-02-04 15:10:34 +0000378
Guido van Rossumbe19ed72007-02-09 05:37:30 +0000379 if self.debugging: print('*welcome*', repr(self.welcome))
Tim Peters2344fae2001-01-15 00:50:52 +0000380 return self.welcome
Guido van Rossumc629d341992-11-05 10:43:02 +0000381
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000382 def getcapabilities(self):
383 """Get the server capabilities, as read by __init__().
384 If the CAPABILITIES command is not supported, an empty dict is
385 returned."""
Antoine Pitrou1cb121e2010-11-09 18:54:37 +0000386 if self._caps is None:
387 self.nntp_version = 1
388 self.nntp_implementation = None
389 try:
390 resp, caps = self.capabilities()
Antoine Pitrou54411c12012-02-12 19:14:17 +0100391 except (NNTPPermanentError, NNTPTemporaryError):
Antoine Pitrou1cb121e2010-11-09 18:54:37 +0000392 # Server doesn't support capabilities
393 self._caps = {}
394 else:
395 self._caps = caps
396 if 'VERSION' in caps:
397 # The server can advertise several supported versions,
398 # choose the highest.
399 self.nntp_version = max(map(int, caps['VERSION']))
400 if 'IMPLEMENTATION' in caps:
401 self.nntp_implementation = ' '.join(caps['IMPLEMENTATION'])
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000402 return self._caps
403
Tim Peters2344fae2001-01-15 00:50:52 +0000404 def set_debuglevel(self, level):
405 """Set the debugging level. Argument 'level' means:
406 0: no debugging output (default)
407 1: print commands and responses but not body text etc.
408 2: also print raw lines read and sent before stripping CR/LF"""
Guido van Rossum54f22ed2000-02-04 15:10:34 +0000409
Tim Peters2344fae2001-01-15 00:50:52 +0000410 self.debugging = level
411 debug = set_debuglevel
Guido van Rossumc629d341992-11-05 10:43:02 +0000412
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000413 def _putline(self, line):
414 """Internal: send one line to the server, appending CRLF.
415 The `line` must be a bytes-like object."""
416 line = line + _CRLF
Guido van Rossumbe19ed72007-02-09 05:37:30 +0000417 if self.debugging > 1: print('*put*', repr(line))
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000418 self.file.write(line)
419 self.file.flush()
Guido van Rossumc629d341992-11-05 10:43:02 +0000420
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000421 def _putcmd(self, line):
422 """Internal: send one command to the server (through _putline()).
423 The `line` must be an unicode string."""
Guido van Rossumbe19ed72007-02-09 05:37:30 +0000424 if self.debugging: print('*cmd*', repr(line))
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000425 line = line.encode(self.encoding, self.errors)
426 self._putline(line)
Guido van Rossumc629d341992-11-05 10:43:02 +0000427
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000428 def _getline(self, strip_crlf=True):
429 """Internal: return one line from the server, stripping _CRLF.
430 Raise EOFError if the connection is closed.
431 Returns a bytes object."""
Georg Brandl28e78412013-10-27 07:29:47 +0100432 line = self.file.readline(_MAXLINE +1)
433 if len(line) > _MAXLINE:
434 raise NNTPDataError('line too long')
Tim Peters2344fae2001-01-15 00:50:52 +0000435 if self.debugging > 1:
Guido van Rossumbe19ed72007-02-09 05:37:30 +0000436 print('*get*', repr(line))
Tim Peters2344fae2001-01-15 00:50:52 +0000437 if not line: raise EOFError
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000438 if strip_crlf:
439 if line[-2:] == _CRLF:
440 line = line[:-2]
441 elif line[-1:] in _CRLF:
442 line = line[:-1]
Tim Peters2344fae2001-01-15 00:50:52 +0000443 return line
Guido van Rossumc629d341992-11-05 10:43:02 +0000444
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000445 def _getresp(self):
Tim Peters2344fae2001-01-15 00:50:52 +0000446 """Internal: get a response from the server.
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000447 Raise various errors if the response indicates an error.
448 Returns an unicode string."""
449 resp = self._getline()
Guido van Rossumbe19ed72007-02-09 05:37:30 +0000450 if self.debugging: print('*resp*', repr(resp))
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000451 resp = resp.decode(self.encoding, self.errors)
Tim Peters2344fae2001-01-15 00:50:52 +0000452 c = resp[:1]
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000453 if c == '4':
Tim Peters2344fae2001-01-15 00:50:52 +0000454 raise NNTPTemporaryError(resp)
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000455 if c == '5':
Tim Peters2344fae2001-01-15 00:50:52 +0000456 raise NNTPPermanentError(resp)
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000457 if c not in '123':
Tim Peters2344fae2001-01-15 00:50:52 +0000458 raise NNTPProtocolError(resp)
459 return resp
Guido van Rossumc629d341992-11-05 10:43:02 +0000460
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000461 def _getlongresp(self, file=None):
Tim Peters2344fae2001-01-15 00:50:52 +0000462 """Internal: get a response plus following text from the server.
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000463 Raise various errors if the response indicates an error.
464
465 Returns a (response, lines) tuple where `response` is an unicode
466 string and `lines` is a list of bytes objects.
467 If `file` is a file-like object, it must be open in binary mode.
468 """
Guido van Rossumd1d584f2001-10-01 13:46:55 +0000469
470 openedFile = None
471 try:
472 # If a string was passed then open a file with that name
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000473 if isinstance(file, (str, bytes)):
474 openedFile = file = open(file, "wb")
Guido van Rossumd1d584f2001-10-01 13:46:55 +0000475
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000476 resp = self._getresp()
477 if resp[:3] not in _LONGRESP:
Guido van Rossumd1d584f2001-10-01 13:46:55 +0000478 raise NNTPReplyError(resp)
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000479
480 lines = []
481 if file is not None:
482 # XXX lines = None instead?
483 terminators = (b'.' + _CRLF, b'.\n')
484 while 1:
485 line = self._getline(False)
486 if line in terminators:
487 break
488 if line.startswith(b'..'):
489 line = line[1:]
490 file.write(line)
491 else:
492 terminator = b'.'
493 while 1:
494 line = self._getline()
495 if line == terminator:
496 break
497 if line.startswith(b'..'):
498 line = line[1:]
499 lines.append(line)
Guido van Rossumd1d584f2001-10-01 13:46:55 +0000500 finally:
501 # If this method created the file, then it must close it
502 if openedFile:
503 openedFile.close()
504
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000505 return resp, lines
Guido van Rossumc629d341992-11-05 10:43:02 +0000506
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000507 def _shortcmd(self, line):
508 """Internal: send a command and get the response.
509 Same return value as _getresp()."""
510 self._putcmd(line)
511 return self._getresp()
Guido van Rossumc629d341992-11-05 10:43:02 +0000512
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000513 def _longcmd(self, line, file=None):
514 """Internal: send a command and get the response plus following text.
515 Same return value as _getlongresp()."""
516 self._putcmd(line)
517 return self._getlongresp(file)
Guido van Rossumc629d341992-11-05 10:43:02 +0000518
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000519 def _longcmdstring(self, line, file=None):
520 """Internal: send a command and get the response plus following text.
521 Same as _longcmd() and _getlongresp(), except that the returned `lines`
522 are unicode strings rather than bytes objects.
523 """
524 self._putcmd(line)
525 resp, list = self._getlongresp(file)
526 return resp, [line.decode(self.encoding, self.errors)
527 for line in list]
528
529 def _getoverviewfmt(self):
530 """Internal: get the overview format. Queries the server if not
531 already done, else returns the cached value."""
532 try:
533 return self._cachedoverviewfmt
534 except AttributeError:
535 pass
536 try:
537 resp, lines = self._longcmdstring("LIST OVERVIEW.FMT")
538 except NNTPPermanentError:
539 # Not supported by server?
540 fmt = _DEFAULT_OVERVIEW_FMT[:]
541 else:
542 fmt = _parse_overview_fmt(lines)
543 self._cachedoverviewfmt = fmt
544 return fmt
545
546 def _grouplist(self, lines):
547 # Parse lines into "group last first flag"
548 return [GroupInfo(*line.split()) for line in lines]
549
550 def capabilities(self):
551 """Process a CAPABILITIES command. Not supported by all servers.
Tim Peters2344fae2001-01-15 00:50:52 +0000552 Return:
553 - resp: server response if successful
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000554 - caps: a dictionary mapping capability names to lists of tokens
555 (for example {'VERSION': ['2'], 'OVER': [], LIST: ['ACTIVE', 'HEADERS'] })
556 """
557 caps = {}
558 resp, lines = self._longcmdstring("CAPABILITIES")
559 for line in lines:
560 name, *tokens = line.split()
561 caps[name] = tokens
562 return resp, caps
Guido van Rossum54f22ed2000-02-04 15:10:34 +0000563
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000564 def newgroups(self, date, *, file=None):
565 """Process a NEWGROUPS command. Arguments:
566 - date: a date or datetime object
567 Return:
568 - resp: server response if successful
569 - list: list of newsgroup names
570 """
571 if not isinstance(date, (datetime.date, datetime.date)):
572 raise TypeError(
573 "the date parameter must be a date or datetime object, "
574 "not '{:40}'".format(date.__class__.__name__))
575 date_str, time_str = _unparse_datetime(date, self.nntp_version < 2)
576 cmd = 'NEWGROUPS {0} {1}'.format(date_str, time_str)
577 resp, lines = self._longcmdstring(cmd, file)
578 return resp, self._grouplist(lines)
Guido van Rossumc629d341992-11-05 10:43:02 +0000579
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000580 def newnews(self, group, date, *, file=None):
Tim Peters2344fae2001-01-15 00:50:52 +0000581 """Process a NEWNEWS command. Arguments:
582 - group: group name or '*'
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000583 - date: a date or datetime object
Tim Peters2344fae2001-01-15 00:50:52 +0000584 Return:
585 - resp: server response if successful
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000586 - list: list of message ids
587 """
588 if not isinstance(date, (datetime.date, datetime.date)):
589 raise TypeError(
590 "the date parameter must be a date or datetime object, "
591 "not '{:40}'".format(date.__class__.__name__))
592 date_str, time_str = _unparse_datetime(date, self.nntp_version < 2)
593 cmd = 'NEWNEWS {0} {1} {2}'.format(group, date_str, time_str)
594 return self._longcmdstring(cmd, file)
Guido van Rossum54f22ed2000-02-04 15:10:34 +0000595
Antoine Pitrou08eeada2010-11-04 21:36:15 +0000596 def list(self, group_pattern=None, *, file=None):
597 """Process a LIST or LIST ACTIVE command. Arguments:
598 - group_pattern: a pattern indicating which groups to query
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000599 - file: Filename string or file object to store the result in
600 Returns:
Tim Peters2344fae2001-01-15 00:50:52 +0000601 - resp: server response if successful
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000602 - list: list of (group, last, first, flag) (strings)
603 """
Antoine Pitrou08eeada2010-11-04 21:36:15 +0000604 if group_pattern is not None:
605 command = 'LIST ACTIVE ' + group_pattern
606 else:
607 command = 'LIST'
608 resp, lines = self._longcmdstring(command, file)
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000609 return resp, self._grouplist(lines)
Guido van Rossum54f22ed2000-02-04 15:10:34 +0000610
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000611 def _getdescriptions(self, group_pattern, return_all):
612 line_pat = re.compile('^(?P<group>[^ \t]+)[ \t]+(.*)$')
613 # Try the more std (acc. to RFC2980) LIST NEWSGROUPS first
614 resp, lines = self._longcmdstring('LIST NEWSGROUPS ' + group_pattern)
615 if not resp.startswith('215'):
616 # Now the deprecated XGTITLE. This either raises an error
617 # or succeeds with the same output structure as LIST
618 # NEWSGROUPS.
619 resp, lines = self._longcmdstring('XGTITLE ' + group_pattern)
620 groups = {}
621 for raw_line in lines:
622 match = line_pat.search(raw_line.strip())
623 if match:
624 name, desc = match.group(1, 2)
625 if not return_all:
626 return desc
627 groups[name] = desc
628 if return_all:
629 return resp, groups
630 else:
631 # Nothing found
632 return ''
Guido van Rossumc629d341992-11-05 10:43:02 +0000633
Martin v. Löwiscc0f9322004-07-26 12:40:50 +0000634 def description(self, group):
Martin v. Löwiscc0f9322004-07-26 12:40:50 +0000635 """Get a description for a single group. If more than one
636 group matches ('group' is a pattern), return the first. If no
637 group matches, return an empty string.
638
639 This elides the response code from the server, since it can
640 only be '215' or '285' (for xgtitle) anyway. If the response
641 code is needed, use the 'descriptions' method.
642
643 NOTE: This neither checks for a wildcard in 'group' nor does
644 it check whether the group actually exists."""
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000645 return self._getdescriptions(group, False)
Martin v. Löwiscc0f9322004-07-26 12:40:50 +0000646
647 def descriptions(self, group_pattern):
648 """Get descriptions for a range of groups."""
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000649 return self._getdescriptions(group_pattern, True)
Martin v. Löwiscc0f9322004-07-26 12:40:50 +0000650
Tim Peters2344fae2001-01-15 00:50:52 +0000651 def group(self, name):
652 """Process a GROUP command. Argument:
653 - group: the group name
654 Returns:
655 - resp: server response if successful
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000656 - count: number of articles
657 - first: first article number
658 - last: last article number
659 - name: the group name
660 """
661 resp = self._shortcmd('GROUP ' + name)
662 if not resp.startswith('211'):
Tim Peters2344fae2001-01-15 00:50:52 +0000663 raise NNTPReplyError(resp)
Eric S. Raymondb9c24fb2001-02-09 07:02:17 +0000664 words = resp.split()
Tim Peters2344fae2001-01-15 00:50:52 +0000665 count = first = last = 0
666 n = len(words)
667 if n > 1:
668 count = words[1]
669 if n > 2:
670 first = words[2]
671 if n > 3:
672 last = words[3]
673 if n > 4:
Eric S. Raymondb9c24fb2001-02-09 07:02:17 +0000674 name = words[4].lower()
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000675 return resp, int(count), int(first), int(last), name
Guido van Rossumc629d341992-11-05 10:43:02 +0000676
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000677 def help(self, *, file=None):
678 """Process a HELP command. Argument:
679 - file: Filename string or file object to store the result in
Tim Peters2344fae2001-01-15 00:50:52 +0000680 Returns:
681 - resp: server response if successful
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000682 - list: list of strings returned by the server in response to the
683 HELP command
684 """
685 return self._longcmdstring('HELP', file)
Guido van Rossum54f22ed2000-02-04 15:10:34 +0000686
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000687 def _statparse(self, resp):
688 """Internal: parse the response line of a STAT, NEXT, LAST,
689 ARTICLE, HEAD or BODY command."""
690 if not resp.startswith('22'):
691 raise NNTPReplyError(resp)
692 words = resp.split()
693 art_num = int(words[1])
694 message_id = words[2]
695 return resp, art_num, message_id
696
697 def _statcmd(self, line):
698 """Internal: process a STAT, NEXT or LAST command."""
699 resp = self._shortcmd(line)
700 return self._statparse(resp)
701
702 def stat(self, message_spec=None):
703 """Process a STAT command. Argument:
704 - message_spec: article number or message id (if not specified,
705 the current article is selected)
706 Returns:
707 - resp: server response if successful
708 - art_num: the article number
709 - message_id: the message id
710 """
711 if message_spec:
712 return self._statcmd('STAT {0}'.format(message_spec))
713 else:
714 return self._statcmd('STAT')
Guido van Rossumc629d341992-11-05 10:43:02 +0000715
Tim Peters2344fae2001-01-15 00:50:52 +0000716 def next(self):
717 """Process a NEXT command. No arguments. Return as for STAT."""
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000718 return self._statcmd('NEXT')
Guido van Rossumc629d341992-11-05 10:43:02 +0000719
Tim Peters2344fae2001-01-15 00:50:52 +0000720 def last(self):
721 """Process a LAST command. No arguments. Return as for STAT."""
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000722 return self._statcmd('LAST')
Guido van Rossumc629d341992-11-05 10:43:02 +0000723
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000724 def _artcmd(self, line, file=None):
Tim Peters2344fae2001-01-15 00:50:52 +0000725 """Internal: process a HEAD, BODY or ARTICLE command."""
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000726 resp, lines = self._longcmd(line, file)
727 resp, art_num, message_id = self._statparse(resp)
728 return resp, ArticleInfo(art_num, message_id, lines)
Guido van Rossumc629d341992-11-05 10:43:02 +0000729
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000730 def head(self, message_spec=None, *, file=None):
Tim Peters2344fae2001-01-15 00:50:52 +0000731 """Process a HEAD command. Argument:
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000732 - message_spec: article number or message id
733 - file: filename string or file object to store the headers in
Tim Peters2344fae2001-01-15 00:50:52 +0000734 Returns:
735 - resp: server response if successful
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000736 - ArticleInfo: (article number, message id, list of header lines)
737 """
738 if message_spec is not None:
739 cmd = 'HEAD {0}'.format(message_spec)
740 else:
741 cmd = 'HEAD'
742 return self._artcmd(cmd, file)
Guido van Rossum54f22ed2000-02-04 15:10:34 +0000743
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000744 def body(self, message_spec=None, *, file=None):
Tim Peters2344fae2001-01-15 00:50:52 +0000745 """Process a BODY command. Argument:
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000746 - message_spec: article number or message id
747 - file: filename string or file object to store the body in
Tim Peters2344fae2001-01-15 00:50:52 +0000748 Returns:
749 - resp: server response if successful
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000750 - ArticleInfo: (article number, message id, list of body lines)
751 """
752 if message_spec is not None:
753 cmd = 'BODY {0}'.format(message_spec)
754 else:
755 cmd = 'BODY'
756 return self._artcmd(cmd, file)
Guido van Rossum54f22ed2000-02-04 15:10:34 +0000757
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000758 def article(self, message_spec=None, *, file=None):
Tim Peters2344fae2001-01-15 00:50:52 +0000759 """Process an ARTICLE command. Argument:
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000760 - message_spec: article number or message id
761 - file: filename string or file object to store the article in
Tim Peters2344fae2001-01-15 00:50:52 +0000762 Returns:
763 - resp: server response if successful
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000764 - ArticleInfo: (article number, message id, list of article lines)
765 """
766 if message_spec is not None:
767 cmd = 'ARTICLE {0}'.format(message_spec)
768 else:
769 cmd = 'ARTICLE'
770 return self._artcmd(cmd, file)
Guido van Rossumc629d341992-11-05 10:43:02 +0000771
Tim Peters2344fae2001-01-15 00:50:52 +0000772 def slave(self):
773 """Process a SLAVE command. Returns:
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000774 - resp: server response if successful
775 """
776 return self._shortcmd('SLAVE')
Guido van Rossum54f22ed2000-02-04 15:10:34 +0000777
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000778 def xhdr(self, hdr, str, *, file=None):
Tim Peters2344fae2001-01-15 00:50:52 +0000779 """Process an XHDR command (optional server extension). Arguments:
780 - hdr: the header type (e.g. 'subject')
781 - str: an article nr, a message id, or a range nr1-nr2
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000782 - file: Filename string or file object to store the result in
Tim Peters2344fae2001-01-15 00:50:52 +0000783 Returns:
784 - resp: server response if successful
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000785 - list: list of (nr, value) strings
786 """
787 pat = re.compile('^([0-9]+) ?(.*)\n?')
788 resp, lines = self._longcmdstring('XHDR {0} {1}'.format(hdr, str), file)
789 def remove_number(line):
Tim Peters2344fae2001-01-15 00:50:52 +0000790 m = pat.match(line)
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000791 return m.group(1, 2) if m else line
792 return resp, [remove_number(line) for line in lines]
Guido van Rossumc629d341992-11-05 10:43:02 +0000793
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000794 def xover(self, start, end, *, file=None):
Tim Peters2344fae2001-01-15 00:50:52 +0000795 """Process an XOVER command (optional server extension) Arguments:
796 - start: start of range
797 - end: end of range
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000798 - file: Filename string or file object to store the result in
Tim Peters2344fae2001-01-15 00:50:52 +0000799 Returns:
800 - resp: server response if successful
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000801 - list: list of dicts containing the response fields
802 """
803 resp, lines = self._longcmdstring('XOVER {0}-{1}'.format(start, end),
804 file)
805 fmt = self._getoverviewfmt()
806 return resp, _parse_overview(lines, fmt)
Guido van Rossum54f22ed2000-02-04 15:10:34 +0000807
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000808 def over(self, message_spec, *, file=None):
809 """Process an OVER command. If the command isn't supported, fall
810 back to XOVER. Arguments:
811 - message_spec:
812 - either a message id, indicating the article to fetch
813 information about
814 - or a (start, end) tuple, indicating a range of article numbers;
815 if end is None, information up to the newest message will be
816 retrieved
817 - or None, indicating the current article number must be used
818 - file: Filename string or file object to store the result in
819 Returns:
820 - resp: server response if successful
821 - list: list of dicts containing the response fields
Guido van Rossum8421c4e1995-09-22 00:52:38 +0000822
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000823 NOTE: the "message id" form isn't supported by XOVER
824 """
825 cmd = 'OVER' if 'OVER' in self._caps else 'XOVER'
826 if isinstance(message_spec, (tuple, list)):
827 start, end = message_spec
828 cmd += ' {0}-{1}'.format(start, end or '')
829 elif message_spec is not None:
830 cmd = cmd + ' ' + message_spec
831 resp, lines = self._longcmdstring(cmd, file)
832 fmt = self._getoverviewfmt()
833 return resp, _parse_overview(lines, fmt)
834
835 def xgtitle(self, group, *, file=None):
Tim Peters2344fae2001-01-15 00:50:52 +0000836 """Process an XGTITLE command (optional server extension) Arguments:
837 - group: group name wildcard (i.e. news.*)
838 Returns:
839 - resp: server response if successful
840 - list: list of (name,title) strings"""
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000841 warnings.warn("The XGTITLE extension is not actively used, "
842 "use descriptions() instead",
Florent Xicluna67317752011-12-10 11:07:42 +0100843 DeprecationWarning, 2)
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000844 line_pat = re.compile('^([^ \t]+)[ \t]+(.*)$')
845 resp, raw_lines = self._longcmdstring('XGTITLE ' + group, file)
Tim Peters2344fae2001-01-15 00:50:52 +0000846 lines = []
847 for raw_line in raw_lines:
Eric S. Raymondb9c24fb2001-02-09 07:02:17 +0000848 match = line_pat.search(raw_line.strip())
Tim Peters2344fae2001-01-15 00:50:52 +0000849 if match:
850 lines.append(match.group(1, 2))
851 return resp, lines
Guido van Rossum8421c4e1995-09-22 00:52:38 +0000852
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000853 def xpath(self, id):
Tim Peters2344fae2001-01-15 00:50:52 +0000854 """Process an XPATH command (optional server extension) Arguments:
855 - id: Message id of article
856 Returns:
857 resp: server response if successful
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000858 path: directory path to article
859 """
860 warnings.warn("The XPATH extension is not actively used",
Florent Xicluna67317752011-12-10 11:07:42 +0100861 DeprecationWarning, 2)
Guido van Rossum54f22ed2000-02-04 15:10:34 +0000862
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000863 resp = self._shortcmd('XPATH {0}'.format(id))
864 if not resp.startswith('223'):
Tim Peters2344fae2001-01-15 00:50:52 +0000865 raise NNTPReplyError(resp)
866 try:
Eric S. Raymondb9c24fb2001-02-09 07:02:17 +0000867 [resp_num, path] = resp.split()
Tim Peters2344fae2001-01-15 00:50:52 +0000868 except ValueError:
869 raise NNTPReplyError(resp)
870 else:
871 return resp, path
Guido van Rossum8421c4e1995-09-22 00:52:38 +0000872
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000873 def date(self):
874 """Process the DATE command.
Tim Peters2344fae2001-01-15 00:50:52 +0000875 Returns:
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000876 - resp: server response if successful
877 - date: datetime object
878 """
879 resp = self._shortcmd("DATE")
880 if not resp.startswith('111'):
Tim Peters2344fae2001-01-15 00:50:52 +0000881 raise NNTPReplyError(resp)
Eric S. Raymondb9c24fb2001-02-09 07:02:17 +0000882 elem = resp.split()
Tim Peters2344fae2001-01-15 00:50:52 +0000883 if len(elem) != 2:
884 raise NNTPDataError(resp)
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000885 date = elem[1]
886 if len(date) != 14:
Tim Peters2344fae2001-01-15 00:50:52 +0000887 raise NNTPDataError(resp)
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000888 return resp, _parse_datetime(date, None)
Guido van Rossum8421c4e1995-09-22 00:52:38 +0000889
Christian Heimes933238a2008-11-05 19:44:21 +0000890 def _post(self, command, f):
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000891 resp = self._shortcmd(command)
892 # Raises a specific exception if posting is not allowed
893 if not resp.startswith('3'):
Christian Heimes933238a2008-11-05 19:44:21 +0000894 raise NNTPReplyError(resp)
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000895 if isinstance(f, (bytes, bytearray)):
896 f = f.splitlines()
897 # We don't use _putline() because:
898 # - we don't want additional CRLF if the file or iterable is already
899 # in the right format
900 # - we don't want a spurious flush() after each line is written
901 for line in f:
902 if not line.endswith(_CRLF):
903 line = line.rstrip(b"\r\n") + _CRLF
Christian Heimes933238a2008-11-05 19:44:21 +0000904 if line.startswith(b'.'):
905 line = b'.' + line
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000906 self.file.write(line)
907 self.file.write(b".\r\n")
908 self.file.flush()
909 return self._getresp()
Guido van Rossum8421c4e1995-09-22 00:52:38 +0000910
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000911 def post(self, data):
Tim Peters2344fae2001-01-15 00:50:52 +0000912 """Process a POST command. Arguments:
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000913 - data: bytes object, iterable or file containing the article
Tim Peters2344fae2001-01-15 00:50:52 +0000914 Returns:
915 - resp: server response if successful"""
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000916 return self._post('POST', data)
Guido van Rossumc629d341992-11-05 10:43:02 +0000917
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000918 def ihave(self, message_id, data):
Tim Peters2344fae2001-01-15 00:50:52 +0000919 """Process an IHAVE command. Arguments:
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000920 - message_id: message-id of the article
921 - data: file containing the article
Tim Peters2344fae2001-01-15 00:50:52 +0000922 Returns:
923 - resp: server response if successful
924 Note that if the server refuses the article an exception is raised."""
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000925 return self._post('IHAVE {0}'.format(message_id), data)
926
927 def _close(self):
928 self.file.close()
929 del self.file
Guido van Rossumc629d341992-11-05 10:43:02 +0000930
Tim Peters2344fae2001-01-15 00:50:52 +0000931 def quit(self):
932 """Process a QUIT command and close the socket. Returns:
933 - resp: server response if successful"""
Antoine Pitrou69ab9512010-09-29 15:03:40 +0000934 try:
935 resp = self._shortcmd('QUIT')
936 finally:
937 self._close()
Tim Peters2344fae2001-01-15 00:50:52 +0000938 return resp
Guido van Rossume2ed9df1997-08-26 23:26:18 +0000939
Antoine Pitrou1cb121e2010-11-09 18:54:37 +0000940 def login(self, user=None, password=None, usenetrc=True):
941 if self.authenticated:
942 raise ValueError("Already logged in.")
943 if not user and not usenetrc:
944 raise ValueError(
945 "At least one of `user` and `usenetrc` must be specified")
946 # If no login/password was specified but netrc was requested,
947 # try to get them from ~/.netrc
948 # Presume that if .netrc has an entry, NNRP authentication is required.
949 try:
950 if usenetrc and not user:
951 import netrc
952 credentials = netrc.netrc()
953 auth = credentials.authenticators(self.host)
954 if auth:
955 user = auth[0]
956 password = auth[2]
Andrew Svetlovf7a17b42012-12-25 16:47:37 +0200957 except OSError:
Antoine Pitrou1cb121e2010-11-09 18:54:37 +0000958 pass
959 # Perform NNTP authentication if needed.
960 if not user:
961 return
962 resp = self._shortcmd('authinfo user ' + user)
963 if resp.startswith('381'):
964 if not password:
965 raise NNTPReplyError(resp)
966 else:
967 resp = self._shortcmd('authinfo pass ' + password)
968 if not resp.startswith('281'):
969 raise NNTPPermanentError(resp)
Antoine Pitrou54411c12012-02-12 19:14:17 +0100970 # Capabilities might have changed after login
971 self._caps = None
972 self.getcapabilities()
Antoine Pitrou1cb121e2010-11-09 18:54:37 +0000973 # Attempt to send mode reader if it was requested after login.
Antoine Pitrou71135622012-02-14 23:29:34 +0100974 # Only do so if we're not in reader mode already.
975 if self.readermode_afterauth and 'READER' not in self._caps:
Antoine Pitrou1cb121e2010-11-09 18:54:37 +0000976 self._setreadermode()
Antoine Pitrou71135622012-02-14 23:29:34 +0100977 # Capabilities might have changed after MODE READER
978 self._caps = None
979 self.getcapabilities()
Antoine Pitrou1cb121e2010-11-09 18:54:37 +0000980
981 def _setreadermode(self):
982 try:
983 self.welcome = self._shortcmd('mode reader')
984 except NNTPPermanentError:
985 # Error 5xx, probably 'not implemented'
986 pass
987 except NNTPTemporaryError as e:
988 if e.response.startswith('480'):
989 # Need authorization before 'mode reader'
990 self.readermode_afterauth = True
991 else:
992 raise
993
994 if _have_ssl:
995 def starttls(self, context=None):
996 """Process a STARTTLS command. Arguments:
997 - context: SSL context to use for the encrypted connection
998 """
999 # Per RFC 4642, STARTTLS MUST NOT be sent after authentication or if
1000 # a TLS session already exists.
1001 if self.tls_on:
1002 raise ValueError("TLS is already enabled.")
1003 if self.authenticated:
1004 raise ValueError("TLS cannot be started after authentication.")
1005 resp = self._shortcmd('STARTTLS')
1006 if resp.startswith('382'):
1007 self.file.close()
Christian Heimes216d4632013-12-02 20:20:11 +01001008 self.sock = _encrypt_on(self.sock, context, self.host)
Antoine Pitrou1cb121e2010-11-09 18:54:37 +00001009 self.file = self.sock.makefile("rwb")
1010 self.tls_on = True
1011 # Capabilities may change after TLS starts up, so ask for them
1012 # again.
1013 self._caps = None
1014 self.getcapabilities()
1015 else:
1016 raise NNTPError("TLS failed to start.")
1017
Guido van Rossume2ed9df1997-08-26 23:26:18 +00001018
Antoine Pitrou69ab9512010-09-29 15:03:40 +00001019class NNTP(_NNTPBase):
1020
1021 def __init__(self, host, port=NNTP_PORT, user=None, password=None,
Antoine Pitrou859c4ef2010-11-09 18:58:42 +00001022 readermode=None, usenetrc=False,
Antoine Pitrou69ab9512010-09-29 15:03:40 +00001023 timeout=_GLOBAL_DEFAULT_TIMEOUT):
1024 """Initialize an instance. Arguments:
1025 - host: hostname to connect to
1026 - port: port to connect to (default the standard NNTP port)
1027 - user: username to authenticate with
1028 - password: password to use with username
1029 - readermode: if true, send 'mode reader' command after
1030 connecting.
1031 - usenetrc: allow loading username and password from ~/.netrc file
1032 if not specified explicitly
1033 - timeout: timeout (in seconds) used for socket connections
1034
1035 readermode is sometimes necessary if you are connecting to an
1036 NNTP server on the local machine and intend to call
Ezio Melotti4969f702011-03-15 05:59:46 +02001037 reader-specific commands, such as `group'. If you get
Antoine Pitrou69ab9512010-09-29 15:03:40 +00001038 unexpected NNTPPermanentErrors, you might need to set
1039 readermode.
1040 """
1041 self.host = host
1042 self.port = port
1043 self.sock = socket.create_connection((host, port), timeout)
1044 file = self.sock.makefile("rwb")
Antoine Pitrou1cb121e2010-11-09 18:54:37 +00001045 _NNTPBase.__init__(self, file, host,
1046 readermode, timeout)
1047 if user or usenetrc:
1048 self.login(user, password, usenetrc)
Antoine Pitrou69ab9512010-09-29 15:03:40 +00001049
1050 def _close(self):
1051 try:
1052 _NNTPBase._close(self)
1053 finally:
1054 self.sock.close()
1055
1056
Antoine Pitrou1cb121e2010-11-09 18:54:37 +00001057if _have_ssl:
1058 class NNTP_SSL(_NNTPBase):
1059
1060 def __init__(self, host, port=NNTP_SSL_PORT,
1061 user=None, password=None, ssl_context=None,
Antoine Pitrou859c4ef2010-11-09 18:58:42 +00001062 readermode=None, usenetrc=False,
Antoine Pitrou1cb121e2010-11-09 18:54:37 +00001063 timeout=_GLOBAL_DEFAULT_TIMEOUT):
1064 """This works identically to NNTP.__init__, except for the change
1065 in default port and the `ssl_context` argument for SSL connections.
1066 """
1067 self.sock = socket.create_connection((host, port), timeout)
Christian Heimes216d4632013-12-02 20:20:11 +01001068 self.sock = _encrypt_on(self.sock, ssl_context, host)
Antoine Pitrou1cb121e2010-11-09 18:54:37 +00001069 file = self.sock.makefile("rwb")
1070 _NNTPBase.__init__(self, file, host,
1071 readermode=readermode, timeout=timeout)
1072 if user or usenetrc:
1073 self.login(user, password, usenetrc)
1074
1075 def _close(self):
1076 try:
1077 _NNTPBase._close(self)
1078 finally:
1079 self.sock.close()
1080
1081 __all__.append("NNTP_SSL")
1082
1083
Neal Norwitzef679562002-11-14 02:19:44 +00001084# Test retrieval when run as a script.
Eric S. Raymondb2db5872002-11-13 23:05:35 +00001085if __name__ == '__main__':
Antoine Pitrou69ab9512010-09-29 15:03:40 +00001086 import argparse
Antoine Pitrou69ab9512010-09-29 15:03:40 +00001087
1088 parser = argparse.ArgumentParser(description="""\
1089 nntplib built-in demo - display the latest articles in a newsgroup""")
1090 parser.add_argument('-g', '--group', default='gmane.comp.python.general',
1091 help='group to fetch messages from (default: %(default)s)')
1092 parser.add_argument('-s', '--server', default='news.gmane.org',
1093 help='NNTP server hostname (default: %(default)s)')
Antoine Pitrou1cb121e2010-11-09 18:54:37 +00001094 parser.add_argument('-p', '--port', default=-1, type=int,
1095 help='NNTP port number (default: %s / %s)' % (NNTP_PORT, NNTP_SSL_PORT))
Antoine Pitrou69ab9512010-09-29 15:03:40 +00001096 parser.add_argument('-n', '--nb-articles', default=10, type=int,
1097 help='number of articles to fetch (default: %(default)s)')
Antoine Pitrou1cb121e2010-11-09 18:54:37 +00001098 parser.add_argument('-S', '--ssl', action='store_true', default=False,
1099 help='use NNTP over SSL')
Antoine Pitrou69ab9512010-09-29 15:03:40 +00001100 args = parser.parse_args()
1101
Antoine Pitrou1cb121e2010-11-09 18:54:37 +00001102 port = args.port
1103 if not args.ssl:
1104 if port == -1:
1105 port = NNTP_PORT
1106 s = NNTP(host=args.server, port=port)
1107 else:
1108 if port == -1:
1109 port = NNTP_SSL_PORT
1110 s = NNTP_SSL(host=args.server, port=port)
1111
1112 caps = s.getcapabilities()
1113 if 'STARTTLS' in caps:
1114 s.starttls()
Antoine Pitrou69ab9512010-09-29 15:03:40 +00001115 resp, count, first, last, name = s.group(args.group)
Guido van Rossumbe19ed72007-02-09 05:37:30 +00001116 print('Group', name, 'has', count, 'articles, range', first, 'to', last)
Antoine Pitrou69ab9512010-09-29 15:03:40 +00001117
1118 def cut(s, lim):
1119 if len(s) > lim:
1120 s = s[:lim - 4] + "..."
1121 return s
1122
1123 first = str(int(last) - args.nb_articles + 1)
1124 resp, overviews = s.xover(first, last)
1125 for artnum, over in overviews:
1126 author = decode_header(over['from']).split('<', 1)[0]
1127 subject = decode_header(over['subject'])
1128 lines = int(over[':lines'])
1129 print("{:7} {:20} {:42} ({})".format(
1130 artnum, cut(author, 20), cut(subject, 42), lines)
1131 )
1132
1133 s.quit()