1
2
3 """Multi-credential file store with lock support.
4
5 This module implements a JSON credential store where multiple
6 credentials can be stored in one file. That file supports locking
7 both in a single process and across processes.
8
9 The credential themselves are keyed off of:
10 * client_id
11 * user_agent
12 * scope
13
14 The format of the stored data is like so:
15 {
16 'file_version': 1,
17 'data': [
18 {
19 'key': {
20 'clientId': '<client id>',
21 'userAgent': '<user agent>',
22 'scope': '<scope>'
23 },
24 'credential': {
25 # JSON serialized Credentials.
26 }
27 }
28 ]
29 }
30 """
31
32 __author__ = 'jbeda@google.com (Joe Beda)'
33
34 import base64
35 import errno
36 import logging
37 import os
38 import threading
39
40 from anyjson import simplejson
41 from oauth2client.client import Storage as BaseStorage
42 from oauth2client.client import Credentials
43 from oauth2client import util
44 from locked_file import LockedFile
45
46 logger = logging.getLogger(__name__)
47
48
49 _multistores = {}
50 _multistores_lock = threading.Lock()
51
52
53 -class Error(Exception):
54 """Base error for this module."""
55 pass
56
59 """The credential store is a newer version that supported."""
60 pass
61
66 """Get a Storage instance for a credential.
67
68 Args:
69 filename: The JSON file storing a set of credentials
70 client_id: The client_id for the credential
71 user_agent: The user agent for the credential
72 scope: string or iterable of strings, Scope(s) being requested
73 warn_on_readonly: if True, log a warning if the store is readonly
74
75 Returns:
76 An object derived from client.Storage for getting/setting the
77 credential.
78 """
79
80 key = {'clientId': client_id, 'userAgent': user_agent,
81 'scope': util.scopes_to_string(scope)}
82 return get_credential_storage_custom_key(
83 filename, key, warn_on_readonly=warn_on_readonly)
84
89 """Get a Storage instance for a credential using a single string as a key.
90
91 Allows you to provide a string as a custom key that will be used for
92 credential storage and retrieval.
93
94 Args:
95 filename: The JSON file storing a set of credentials
96 key_string: A string to use as the key for storing this credential.
97 warn_on_readonly: if True, log a warning if the store is readonly
98
99 Returns:
100 An object derived from client.Storage for getting/setting the
101 credential.
102 """
103
104 key_dict = {'key': key_string}
105 return get_credential_storage_custom_key(
106 filename, key_dict, warn_on_readonly=warn_on_readonly)
107
112 """Get a Storage instance for a credential using a dictionary as a key.
113
114 Allows you to provide a dictionary as a custom key that will be used for
115 credential storage and retrieval.
116
117 Args:
118 filename: The JSON file storing a set of credentials
119 key_dict: A dictionary to use as the key for storing this credential. There
120 is no ordering of the keys in the dictionary. Logically equivalent
121 dictionaries will produce equivalent storage keys.
122 warn_on_readonly: if True, log a warning if the store is readonly
123
124 Returns:
125 An object derived from client.Storage for getting/setting the
126 credential.
127 """
128 filename = os.path.expanduser(filename)
129 _multistores_lock.acquire()
130 try:
131 multistore = _multistores.setdefault(
132 filename, _MultiStore(filename, warn_on_readonly=warn_on_readonly))
133 finally:
134 _multistores_lock.release()
135 key = util.dict_to_tuple_key(key_dict)
136 return multistore._get_storage(key)
137
140 """A file backed store for multiple credentials."""
141
142 @util.positional(2)
143 - def __init__(self, filename, warn_on_readonly=True):
144 """Initialize the class.
145
146 This will create the file if necessary.
147 """
148 self._file = LockedFile(filename, 'r+b', 'rb')
149 self._thread_lock = threading.Lock()
150 self._read_only = False
151 self._warn_on_readonly = warn_on_readonly
152
153 self._create_file_if_needed()
154
155
156
157
158
159
160
161
162 self._data = None
163
165 """A Storage object that knows how to read/write a single credential."""
166
168 self._multistore = multistore
169 self._key = key
170
172 """Acquires any lock necessary to access this Storage.
173
174 This lock is not reentrant.
175 """
176 self._multistore._lock()
177
179 """Release the Storage lock.
180
181 Trying to release a lock that isn't held will result in a
182 RuntimeError.
183 """
184 self._multistore._unlock()
185
187 """Retrieve credential.
188
189 The Storage lock must be held when this is called.
190
191 Returns:
192 oauth2client.client.Credentials
193 """
194 credential = self._multistore._get_credential(self._key)
195 if credential:
196 credential.set_store(self)
197 return credential
198
200 """Write a credential.
201
202 The Storage lock must be held when this is called.
203
204 Args:
205 credentials: Credentials, the credentials to store.
206 """
207 self._multistore._update_credential(self._key, credentials)
208
210 """Delete a credential.
211
212 The Storage lock must be held when this is called.
213
214 Args:
215 credentials: Credentials, the credentials to store.
216 """
217 self._multistore._delete_credential(self._key)
218
220 """Create an empty file if necessary.
221
222 This method will not initialize the file. Instead it implements a
223 simple version of "touch" to ensure the file has been created.
224 """
225 if not os.path.exists(self._file.filename()):
226 old_umask = os.umask(0177)
227 try:
228 open(self._file.filename(), 'a+b').close()
229 finally:
230 os.umask(old_umask)
231
233 """Lock the entire multistore."""
234 self._thread_lock.acquire()
235 self._file.open_and_lock()
236 if not self._file.is_locked():
237 self._read_only = True
238 if self._warn_on_readonly:
239 logger.warn('The credentials file (%s) is not writable. Opening in '
240 'read-only mode. Any refreshed credentials will only be '
241 'valid for this run.' % self._file.filename())
242 if os.path.getsize(self._file.filename()) == 0:
243 logger.debug('Initializing empty multistore file')
244
245 self._data = {}
246 self._write()
247 elif not self._read_only or self._data is None:
248
249
250
251
252
253 self._refresh_data_cache()
254
256 """Release the lock on the multistore."""
257 self._file.unlock_and_close()
258 self._thread_lock.release()
259
261 """Get the raw content of the multistore file.
262
263 The multistore must be locked when this is called.
264
265 Returns:
266 The contents of the multistore decoded as JSON.
267 """
268 assert self._thread_lock.locked()
269 self._file.file_handle().seek(0)
270 return simplejson.load(self._file.file_handle())
271
273 """Write a JSON serializable data structure to the multistore.
274
275 The multistore must be locked when this is called.
276
277 Args:
278 data: The data to be serialized and written.
279 """
280 assert self._thread_lock.locked()
281 if self._read_only:
282 return
283 self._file.file_handle().seek(0)
284 simplejson.dump(data, self._file.file_handle(), sort_keys=True, indent=2)
285 self._file.file_handle().truncate()
286
288 """Refresh the contents of the multistore.
289
290 The multistore must be locked when this is called.
291
292 Raises:
293 NewerCredentialStoreError: Raised when a newer client has written the
294 store.
295 """
296 self._data = {}
297 try:
298 raw_data = self._locked_json_read()
299 except Exception:
300 logger.warn('Credential data store could not be loaded. '
301 'Will ignore and overwrite.')
302 return
303
304 version = 0
305 try:
306 version = raw_data['file_version']
307 except Exception:
308 logger.warn('Missing version for credential data store. It may be '
309 'corrupt or an old version. Overwriting.')
310 if version > 1:
311 raise NewerCredentialStoreError(
312 'Credential file has file_version of %d. '
313 'Only file_version of 1 is supported.' % version)
314
315 credentials = []
316 try:
317 credentials = raw_data['data']
318 except (TypeError, KeyError):
319 pass
320
321 for cred_entry in credentials:
322 try:
323 (key, credential) = self._decode_credential_from_json(cred_entry)
324 self._data[key] = credential
325 except:
326
327 logger.info('Error decoding credential, skipping', exc_info=True)
328
330 """Load a credential from our JSON serialization.
331
332 Args:
333 cred_entry: A dict entry from the data member of our format
334
335 Returns:
336 (key, cred) where the key is the key tuple and the cred is the
337 OAuth2Credential object.
338 """
339 raw_key = cred_entry['key']
340 key = util.dict_to_tuple_key(raw_key)
341 credential = None
342 credential = Credentials.new_from_json(simplejson.dumps(cred_entry['credential']))
343 return (key, credential)
344
346 """Write the cached data back out.
347
348 The multistore must be locked.
349 """
350 raw_data = {'file_version': 1}
351 raw_creds = []
352 raw_data['data'] = raw_creds
353 for (cred_key, cred) in self._data.items():
354 raw_key = dict(cred_key)
355 raw_cred = simplejson.loads(cred.to_json())
356 raw_creds.append({'key': raw_key, 'credential': raw_cred})
357 self._locked_json_write(raw_data)
358
360 """Get a credential from the multistore.
361
362 The multistore must be locked.
363
364 Args:
365 key: The key used to retrieve the credential
366
367 Returns:
368 The credential specified or None if not present
369 """
370 return self._data.get(key, None)
371
373 """Update a credential and write the multistore.
374
375 This must be called when the multistore is locked.
376
377 Args:
378 key: The key used to retrieve the credential
379 cred: The OAuth2Credential to update/set
380 """
381 self._data[key] = cred
382 self._write()
383
385 """Delete a credential and write the multistore.
386
387 This must be called when the multistore is locked.
388
389 Args:
390 key: The key used to retrieve the credential
391 """
392 try:
393 del self._data[key]
394 except KeyError:
395 pass
396 self._write()
397
399 """Get a Storage object to get/set a credential.
400
401 This Storage is a 'view' into the multistore.
402
403 Args:
404 key: The key used to retrieve the credential
405
406 Returns:
407 A Storage object that can be used to get/set this cred
408 """
409 return self._Storage(self, key)
410