Package oauth2client :: Module multistore_file
[hide private]
[frames] | no frames]

Source Code for Module oauth2client.multistore_file

  1  # Copyright 2011 Google Inc. All Rights Reserved. 
  2   
  3  """Multi-credential file store with lock support. 
  4   
  5  This module implements a JSON credential store where multiple 
  6  credentials can be stored in one file.  That file supports locking 
  7  both in a single process and across processes. 
  8   
  9  The credential themselves are keyed off of: 
 10  * client_id 
 11  * user_agent 
 12  * scope 
 13   
 14  The format of the stored data is like so: 
 15  { 
 16    'file_version': 1, 
 17    'data': [ 
 18      { 
 19        'key': { 
 20          'clientId': '<client id>', 
 21          'userAgent': '<user agent>', 
 22          'scope': '<scope>' 
 23        }, 
 24        'credential': { 
 25          # JSON serialized Credentials. 
 26        } 
 27      } 
 28    ] 
 29  } 
 30  """ 
 31   
 32  __author__ = 'jbeda@google.com (Joe Beda)' 
 33   
 34  import base64 
 35  import errno 
 36  import fcntl 
 37  import logging 
 38  import os 
 39  import threading 
 40   
 41  from anyjson import simplejson 
 42  from client import Storage as BaseStorage 
 43  from client import Credentials 
 44   
 45  logger = logging.getLogger(__name__) 
 46   
 47  # A dict from 'filename'->_MultiStore instances 
 48  _multistores = {} 
 49  _multistores_lock = threading.Lock() 
 50   
 51   
52 -class Error(Exception):
53 """Base error for this module.""" 54 pass
55 56
57 -class NewerCredentialStoreError(Error):
58 """The credential store is a newer version that supported.""" 59 pass
60 61
62 -def get_credential_storage(filename, client_id, user_agent, scope, 63 warn_on_readonly=True):
64 """Get a Storage instance for a credential. 65 66 Args: 67 filename: The JSON file storing a set of credentials 68 client_id: The client_id for the credential 69 user_agent: The user agent for the credential 70 scope: string or list of strings, Scope(s) being requested 71 warn_on_readonly: if True, log a warning if the store is readonly 72 73 Returns: 74 An object derived from client.Storage for getting/setting the 75 credential. 76 """ 77 filename = os.path.realpath(os.path.expanduser(filename)) 78 _multistores_lock.acquire() 79 try: 80 multistore = _multistores.setdefault( 81 filename, _MultiStore(filename, warn_on_readonly)) 82 finally: 83 _multistores_lock.release() 84 if type(scope) is list: 85 scope = ' '.join(scope) 86 return multistore._get_storage(client_id, user_agent, scope)
87 88
89 -class _MultiStore(object):
90 """A file backed store for multiple credentials.""" 91
92 - def __init__(self, filename, warn_on_readonly=True):
93 """Initialize the class. 94 95 This will create the file if necessary. 96 """ 97 self._filename = filename 98 self._thread_lock = threading.Lock() 99 self._file_handle = None 100 self._read_only = False 101 self._warn_on_readonly = warn_on_readonly 102 103 self._create_file_if_needed() 104 105 # Cache of deserialized store. This is only valid after the 106 # _MultiStore is locked or _refresh_data_cache is called. This is 107 # of the form of: 108 # 109 # (client_id, user_agent, scope) -> OAuth2Credential 110 # 111 # If this is None, then the store hasn't been read yet. 112 self._data = None
113
114 - class _Storage(BaseStorage):
115 """A Storage object that knows how to read/write a single credential.""" 116
117 - def __init__(self, multistore, client_id, user_agent, scope):
118 self._multistore = multistore 119 self._client_id = client_id 120 self._user_agent = user_agent 121 self._scope = scope
122
123 - def acquire_lock(self):
124 """Acquires any lock necessary to access this Storage. 125 126 This lock is not reentrant. 127 """ 128 self._multistore._lock()
129
130 - def release_lock(self):
131 """Release the Storage lock. 132 133 Trying to release a lock that isn't held will result in a 134 RuntimeError. 135 """ 136 self._multistore._unlock()
137
138 - def locked_get(self):
139 """Retrieve credential. 140 141 The Storage lock must be held when this is called. 142 143 Returns: 144 oauth2client.client.Credentials 145 """ 146 credential = self._multistore._get_credential( 147 self._client_id, self._user_agent, self._scope) 148 if credential: 149 credential.set_store(self) 150 return credential
151
152 - def locked_put(self, credentials):
153 """Write a credential. 154 155 The Storage lock must be held when this is called. 156 157 Args: 158 credentials: Credentials, the credentials to store. 159 """ 160 self._multistore._update_credential(credentials, self._scope)
161
162 - def locked_delete(self):
163 """Delete a credential. 164 165 The Storage lock must be held when this is called. 166 167 Args: 168 credentials: Credentials, the credentials to store. 169 """ 170 self._multistore._delete_credential(self._client_id, self._user_agent, 171 self._scope)
172
173 - def _create_file_if_needed(self):
174 """Create an empty file if necessary. 175 176 This method will not initialize the file. Instead it implements a 177 simple version of "touch" to ensure the file has been created. 178 """ 179 if not os.path.exists(self._filename): 180 old_umask = os.umask(0177) 181 try: 182 open(self._filename, 'a+b').close() 183 finally: 184 os.umask(old_umask)
185
186 - def _lock(self):
187 """Lock the entire multistore.""" 188 self._thread_lock.acquire() 189 # Check to see if the file is writeable. 190 try: 191 self._file_handle = open(self._filename, 'r+b') 192 fcntl.lockf(self._file_handle.fileno(), fcntl.LOCK_EX) 193 except IOError, e: 194 if e.errno != errno.EACCES: 195 raise e 196 self._file_handle = open(self._filename, 'rb') 197 self._read_only = True 198 if self._warn_on_readonly: 199 logger.warn('The credentials file (%s) is not writable. Opening in ' 200 'read-only mode. Any refreshed credentials will only be ' 201 'valid for this run.' % self._filename) 202 if os.path.getsize(self._filename) == 0: 203 logger.debug('Initializing empty multistore file') 204 # The multistore is empty so write out an empty file. 205 self._data = {} 206 self._write() 207 elif not self._read_only or self._data is None: 208 # Only refresh the data if we are read/write or we haven't 209 # cached the data yet. If we are readonly, we assume is isn't 210 # changing out from under us and that we only have to read it 211 # once. This prevents us from whacking any new access keys that 212 # we have cached in memory but were unable to write out. 213 self._refresh_data_cache()
214
215 - def _unlock(self):
216 """Release the lock on the multistore.""" 217 if not self._read_only: 218 fcntl.lockf(self._file_handle.fileno(), fcntl.LOCK_UN) 219 self._file_handle.close() 220 self._thread_lock.release()
221
222 - def _locked_json_read(self):
223 """Get the raw content of the multistore file. 224 225 The multistore must be locked when this is called. 226 227 Returns: 228 The contents of the multistore decoded as JSON. 229 """ 230 assert self._thread_lock.locked() 231 self._file_handle.seek(0) 232 return simplejson.load(self._file_handle)
233
234 - def _locked_json_write(self, data):
235 """Write a JSON serializable data structure to the multistore. 236 237 The multistore must be locked when this is called. 238 239 Args: 240 data: The data to be serialized and written. 241 """ 242 assert self._thread_lock.locked() 243 if self._read_only: 244 return 245 self._file_handle.seek(0) 246 simplejson.dump(data, self._file_handle, sort_keys=True, indent=2) 247 self._file_handle.truncate()
248
249 - def _refresh_data_cache(self):
250 """Refresh the contents of the multistore. 251 252 The multistore must be locked when this is called. 253 254 Raises: 255 NewerCredentialStoreError: Raised when a newer client has written the 256 store. 257 """ 258 self._data = {} 259 try: 260 raw_data = self._locked_json_read() 261 except Exception: 262 logger.warn('Credential data store could not be loaded. ' 263 'Will ignore and overwrite.') 264 return 265 266 version = 0 267 try: 268 version = raw_data['file_version'] 269 except Exception: 270 logger.warn('Missing version for credential data store. It may be ' 271 'corrupt or an old version. Overwriting.') 272 if version > 1: 273 raise NewerCredentialStoreError( 274 'Credential file has file_version of %d. ' 275 'Only file_version of 1 is supported.' % version) 276 277 credentials = [] 278 try: 279 credentials = raw_data['data'] 280 except (TypeError, KeyError): 281 pass 282 283 for cred_entry in credentials: 284 try: 285 (key, credential) = self._decode_credential_from_json(cred_entry) 286 self._data[key] = credential 287 except: 288 # If something goes wrong loading a credential, just ignore it 289 logger.info('Error decoding credential, skipping', exc_info=True)
290
291 - def _decode_credential_from_json(self, cred_entry):
292 """Load a credential from our JSON serialization. 293 294 Args: 295 cred_entry: A dict entry from the data member of our format 296 297 Returns: 298 (key, cred) where the key is the key tuple and the cred is the 299 OAuth2Credential object. 300 """ 301 raw_key = cred_entry['key'] 302 client_id = raw_key['clientId'] 303 user_agent = raw_key['userAgent'] 304 scope = raw_key['scope'] 305 key = (client_id, user_agent, scope) 306 credential = None 307 credential = Credentials.new_from_json(simplejson.dumps(cred_entry['credential'])) 308 return (key, credential)
309
310 - def _write(self):
311 """Write the cached data back out. 312 313 The multistore must be locked. 314 """ 315 raw_data = {'file_version': 1} 316 raw_creds = [] 317 raw_data['data'] = raw_creds 318 for (cred_key, cred) in self._data.items(): 319 raw_key = { 320 'clientId': cred_key[0], 321 'userAgent': cred_key[1], 322 'scope': cred_key[2] 323 } 324 raw_cred = simplejson.loads(cred.to_json()) 325 raw_creds.append({'key': raw_key, 'credential': raw_cred}) 326 self._locked_json_write(raw_data)
327
328 - def _get_credential(self, client_id, user_agent, scope):
329 """Get a credential from the multistore. 330 331 The multistore must be locked. 332 333 Args: 334 client_id: The client_id for the credential 335 user_agent: The user agent for the credential 336 scope: A string for the scope(s) being requested 337 338 Returns: 339 The credential specified or None if not present 340 """ 341 key = (client_id, user_agent, scope) 342 343 return self._data.get(key, None)
344
345 - def _update_credential(self, cred, scope):
346 """Update a credential and write the multistore. 347 348 This must be called when the multistore is locked. 349 350 Args: 351 cred: The OAuth2Credential to update/set 352 scope: The scope(s) that this credential covers 353 """ 354 key = (cred.client_id, cred.user_agent, scope) 355 self._data[key] = cred 356 self._write()
357
358 - def _delete_credential(self, client_id, user_agent, scope):
359 """Delete a credential and write the multistore. 360 361 This must be called when the multistore is locked. 362 363 Args: 364 client_id: The client_id for the credential 365 user_agent: The user agent for the credential 366 scope: The scope(s) that this credential covers 367 """ 368 key = (client_id, user_agent, scope) 369 try: 370 del self._data[key] 371 except KeyError: 372 pass 373 self._write()
374
375 - def _get_storage(self, client_id, user_agent, scope):
376 """Get a Storage object to get/set a credential. 377 378 This Storage is a 'view' into the multistore. 379 380 Args: 381 client_id: The client_id for the credential 382 user_agent: The user agent for the credential 383 scope: A string for the scope(s) being requested 384 385 Returns: 386 A Storage object that can be used to get/set this cred 387 """ 388 return self._Storage(self, client_id, user_agent, scope)
389