Package oauth2client :: Module multistore_file
[hide private]
[frames] | no frames]

Source Code for Module oauth2client.multistore_file

  1  # Copyright 2011 Google Inc. All Rights Reserved. 
  2   
  3  """Multi-credential file store with lock support. 
  4   
  5  This module implements a JSON credential store where multiple 
  6  credentials can be stored in one file.  That file supports locking 
  7  both in a single process and across processes. 
  8   
  9  The credential themselves are keyed off of: 
 10  * client_id 
 11  * user_agent 
 12  * scope 
 13   
 14  The format of the stored data is like so: 
 15  { 
 16    'file_version': 1, 
 17    'data': [ 
 18      { 
 19        'key': { 
 20          'clientId': '<client id>', 
 21          'userAgent': '<user agent>', 
 22          'scope': '<scope>' 
 23        }, 
 24        'credential': { 
 25          # JSON serialized Credentials. 
 26        } 
 27      } 
 28    ] 
 29  } 
 30  """ 
 31   
 32  __author__ = 'jbeda@google.com (Joe Beda)' 
 33   
 34  import base64 
 35  import errno 
 36  import logging 
 37  import os 
 38  import threading 
 39   
 40  from anyjson import simplejson 
 41  from oauth2client.client import Storage as BaseStorage 
 42  from oauth2client.client import Credentials 
 43  from oauth2client import util 
 44  from locked_file import LockedFile 
 45   
 46  logger = logging.getLogger(__name__) 
 47   
 48  # A dict from 'filename'->_MultiStore instances 
 49  _multistores = {} 
 50  _multistores_lock = threading.Lock() 
51 52 53 -class Error(Exception):
54 """Base error for this module.""" 55 pass
56
57 58 -class NewerCredentialStoreError(Error):
59 """The credential store is a newer version that supported.""" 60 pass
61
62 63 @util.positional(4) 64 -def get_credential_storage(filename, client_id, user_agent, scope, 65 warn_on_readonly=True):
66 """Get a Storage instance for a credential. 67 68 Args: 69 filename: The JSON file storing a set of credentials 70 client_id: The client_id for the credential 71 user_agent: The user agent for the credential 72 scope: string or list of strings, Scope(s) being requested 73 warn_on_readonly: if True, log a warning if the store is readonly 74 75 Returns: 76 An object derived from client.Storage for getting/setting the 77 credential. 78 """ 79 filename = os.path.expanduser(filename) 80 _multistores_lock.acquire() 81 try: 82 multistore = _multistores.setdefault( 83 filename, _MultiStore(filename, warn_on_readonly=warn_on_readonly)) 84 finally: 85 _multistores_lock.release() 86 if type(scope) is list: 87 scope = ' '.join(scope) 88 return multistore._get_storage(client_id, user_agent, scope)
89
90 91 -class _MultiStore(object):
92 """A file backed store for multiple credentials.""" 93 94 @util.positional(2)
95 - def __init__(self, filename, warn_on_readonly=True):
96 """Initialize the class. 97 98 This will create the file if necessary. 99 """ 100 self._file = LockedFile(filename, 'r+b', 'rb') 101 self._thread_lock = threading.Lock() 102 self._read_only = False 103 self._warn_on_readonly = warn_on_readonly 104 105 self._create_file_if_needed() 106 107 # Cache of deserialized store. This is only valid after the 108 # _MultiStore is locked or _refresh_data_cache is called. This is 109 # of the form of: 110 # 111 # (client_id, user_agent, scope) -> OAuth2Credential 112 # 113 # If this is None, then the store hasn't been read yet. 114 self._data = None
115
116 - class _Storage(BaseStorage):
117 """A Storage object that knows how to read/write a single credential.""" 118
119 - def __init__(self, multistore, client_id, user_agent, scope):
120 self._multistore = multistore 121 self._client_id = client_id 122 self._user_agent = user_agent 123 self._scope = scope
124
125 - def acquire_lock(self):
126 """Acquires any lock necessary to access this Storage. 127 128 This lock is not reentrant. 129 """ 130 self._multistore._lock()
131
132 - def release_lock(self):
133 """Release the Storage lock. 134 135 Trying to release a lock that isn't held will result in a 136 RuntimeError. 137 """ 138 self._multistore._unlock()
139
140 - def locked_get(self):
141 """Retrieve credential. 142 143 The Storage lock must be held when this is called. 144 145 Returns: 146 oauth2client.client.Credentials 147 """ 148 credential = self._multistore._get_credential( 149 self._client_id, self._user_agent, self._scope) 150 if credential: 151 credential.set_store(self) 152 return credential
153
154 - def locked_put(self, credentials):
155 """Write a credential. 156 157 The Storage lock must be held when this is called. 158 159 Args: 160 credentials: Credentials, the credentials to store. 161 """ 162 self._multistore._update_credential(credentials, self._scope)
163
164 - def locked_delete(self):
165 """Delete a credential. 166 167 The Storage lock must be held when this is called. 168 169 Args: 170 credentials: Credentials, the credentials to store. 171 """ 172 self._multistore._delete_credential(self._client_id, self._user_agent, 173 self._scope)
174
175 - def _create_file_if_needed(self):
176 """Create an empty file if necessary. 177 178 This method will not initialize the file. Instead it implements a 179 simple version of "touch" to ensure the file has been created. 180 """ 181 if not os.path.exists(self._file.filename()): 182 old_umask = os.umask(0177) 183 try: 184 open(self._file.filename(), 'a+b').close() 185 finally: 186 os.umask(old_umask)
187
188 - def _lock(self):
189 """Lock the entire multistore.""" 190 self._thread_lock.acquire() 191 self._file.open_and_lock() 192 if not self._file.is_locked(): 193 self._read_only = True 194 if self._warn_on_readonly: 195 logger.warn('The credentials file (%s) is not writable. Opening in ' 196 'read-only mode. Any refreshed credentials will only be ' 197 'valid for this run.' % self._file.filename()) 198 if os.path.getsize(self._file.filename()) == 0: 199 logger.debug('Initializing empty multistore file') 200 # The multistore is empty so write out an empty file. 201 self._data = {} 202 self._write() 203 elif not self._read_only or self._data is None: 204 # Only refresh the data if we are read/write or we haven't 205 # cached the data yet. If we are readonly, we assume is isn't 206 # changing out from under us and that we only have to read it 207 # once. This prevents us from whacking any new access keys that 208 # we have cached in memory but were unable to write out. 209 self._refresh_data_cache()
210
211 - def _unlock(self):
212 """Release the lock on the multistore.""" 213 self._file.unlock_and_close() 214 self._thread_lock.release()
215
216 - def _locked_json_read(self):
217 """Get the raw content of the multistore file. 218 219 The multistore must be locked when this is called. 220 221 Returns: 222 The contents of the multistore decoded as JSON. 223 """ 224 assert self._thread_lock.locked() 225 self._file.file_handle().seek(0) 226 return simplejson.load(self._file.file_handle())
227
228 - def _locked_json_write(self, data):
229 """Write a JSON serializable data structure to the multistore. 230 231 The multistore must be locked when this is called. 232 233 Args: 234 data: The data to be serialized and written. 235 """ 236 assert self._thread_lock.locked() 237 if self._read_only: 238 return 239 self._file.file_handle().seek(0) 240 simplejson.dump(data, self._file.file_handle(), sort_keys=True, indent=2) 241 self._file.file_handle().truncate()
242
243 - def _refresh_data_cache(self):
244 """Refresh the contents of the multistore. 245 246 The multistore must be locked when this is called. 247 248 Raises: 249 NewerCredentialStoreError: Raised when a newer client has written the 250 store. 251 """ 252 self._data = {} 253 try: 254 raw_data = self._locked_json_read() 255 except Exception: 256 logger.warn('Credential data store could not be loaded. ' 257 'Will ignore and overwrite.') 258 return 259 260 version = 0 261 try: 262 version = raw_data['file_version'] 263 except Exception: 264 logger.warn('Missing version for credential data store. It may be ' 265 'corrupt or an old version. Overwriting.') 266 if version > 1: 267 raise NewerCredentialStoreError( 268 'Credential file has file_version of %d. ' 269 'Only file_version of 1 is supported.' % version) 270 271 credentials = [] 272 try: 273 credentials = raw_data['data'] 274 except (TypeError, KeyError): 275 pass 276 277 for cred_entry in credentials: 278 try: 279 (key, credential) = self._decode_credential_from_json(cred_entry) 280 self._data[key] = credential 281 except: 282 # If something goes wrong loading a credential, just ignore it 283 logger.info('Error decoding credential, skipping', exc_info=True)
284
285 - def _decode_credential_from_json(self, cred_entry):
286 """Load a credential from our JSON serialization. 287 288 Args: 289 cred_entry: A dict entry from the data member of our format 290 291 Returns: 292 (key, cred) where the key is the key tuple and the cred is the 293 OAuth2Credential object. 294 """ 295 raw_key = cred_entry['key'] 296 client_id = raw_key['clientId'] 297 user_agent = raw_key['userAgent'] 298 scope = raw_key['scope'] 299 key = (client_id, user_agent, scope) 300 credential = None 301 credential = Credentials.new_from_json(simplejson.dumps(cred_entry['credential'])) 302 return (key, credential)
303
304 - def _write(self):
305 """Write the cached data back out. 306 307 The multistore must be locked. 308 """ 309 raw_data = {'file_version': 1} 310 raw_creds = [] 311 raw_data['data'] = raw_creds 312 for (cred_key, cred) in self._data.items(): 313 raw_key = { 314 'clientId': cred_key[0], 315 'userAgent': cred_key[1], 316 'scope': cred_key[2] 317 } 318 raw_cred = simplejson.loads(cred.to_json()) 319 raw_creds.append({'key': raw_key, 'credential': raw_cred}) 320 self._locked_json_write(raw_data)
321
322 - def _get_credential(self, client_id, user_agent, scope):
323 """Get a credential from the multistore. 324 325 The multistore must be locked. 326 327 Args: 328 client_id: The client_id for the credential 329 user_agent: The user agent for the credential 330 scope: A string for the scope(s) being requested 331 332 Returns: 333 The credential specified or None if not present 334 """ 335 key = (client_id, user_agent, scope) 336 337 return self._data.get(key, None)
338
339 - def _update_credential(self, cred, scope):
340 """Update a credential and write the multistore. 341 342 This must be called when the multistore is locked. 343 344 Args: 345 cred: The OAuth2Credential to update/set 346 scope: The scope(s) that this credential covers 347 """ 348 key = (cred.client_id, cred.user_agent, scope) 349 self._data[key] = cred 350 self._write()
351
352 - def _delete_credential(self, client_id, user_agent, scope):
353 """Delete a credential and write the multistore. 354 355 This must be called when the multistore is locked. 356 357 Args: 358 client_id: The client_id for the credential 359 user_agent: The user agent for the credential 360 scope: The scope(s) that this credential covers 361 """ 362 key = (client_id, user_agent, scope) 363 try: 364 del self._data[key] 365 except KeyError: 366 pass 367 self._write()
368
369 - def _get_storage(self, client_id, user_agent, scope):
370 """Get a Storage object to get/set a credential. 371 372 This Storage is a 'view' into the multistore. 373 374 Args: 375 client_id: The client_id for the credential 376 user_agent: The user agent for the credential 377 scope: A string for the scope(s) being requested 378 379 Returns: 380 A Storage object that can be used to get/set this cred 381 """ 382 return self._Storage(self, client_id, user_agent, scope)
383