Package oauth2client :: Module multistore_file
[hide private]
[frames] | no frames]

Source Code for Module oauth2client.multistore_file

  1  # Copyright 2011 Google Inc. All Rights Reserved. 
  2   
  3  """Multi-credential file store with lock support. 
  4   
  5  This module implements a JSON credential store where multiple 
  6  credentials can be stored in one file.  That file supports locking 
  7  both in a single process and across processes. 
  8   
  9  The credential themselves are keyed off of: 
 10  * client_id 
 11  * user_agent 
 12  * scope 
 13   
 14  The format of the stored data is like so: 
 15  { 
 16    'file_version': 1, 
 17    'data': [ 
 18      { 
 19        'key': { 
 20          'clientId': '<client id>', 
 21          'userAgent': '<user agent>', 
 22          'scope': '<scope>' 
 23        }, 
 24        'credential': { 
 25          # JSON serialized Credentials. 
 26        } 
 27      } 
 28    ] 
 29  } 
 30  """ 
 31   
 32  __author__ = 'jbeda@google.com (Joe Beda)' 
 33   
 34  import base64 
 35  import errno 
 36  import logging 
 37  import os 
 38  import threading 
 39   
 40  from anyjson import simplejson 
 41  from client import Storage as BaseStorage 
 42  from client import Credentials 
 43  from locked_file import LockedFile 
 44   
 45  logger = logging.getLogger(__name__) 
 46   
 47  # A dict from 'filename'->_MultiStore instances 
 48  _multistores = {} 
 49  _multistores_lock = threading.Lock() 
 50   
 51   
52 -class Error(Exception):
53 """Base error for this module.""" 54 pass
55 56
57 -class NewerCredentialStoreError(Error):
58 """The credential store is a newer version that supported.""" 59 pass
60 61
62 -def get_credential_storage(filename, client_id, user_agent, scope, 63 warn_on_readonly=True):
64 """Get a Storage instance for a credential. 65 66 Args: 67 filename: The JSON file storing a set of credentials 68 client_id: The client_id for the credential 69 user_agent: The user agent for the credential 70 scope: string or list of strings, Scope(s) being requested 71 warn_on_readonly: if True, log a warning if the store is readonly 72 73 Returns: 74 An object derived from client.Storage for getting/setting the 75 credential. 76 """ 77 filename = os.path.realpath(os.path.expanduser(filename)) 78 _multistores_lock.acquire() 79 try: 80 multistore = _multistores.setdefault( 81 filename, _MultiStore(filename, warn_on_readonly)) 82 finally: 83 _multistores_lock.release() 84 if type(scope) is list: 85 scope = ' '.join(scope) 86 return multistore._get_storage(client_id, user_agent, scope)
87 88
89 -class _MultiStore(object):
90 """A file backed store for multiple credentials.""" 91
92 - def __init__(self, filename, warn_on_readonly=True):
93 """Initialize the class. 94 95 This will create the file if necessary. 96 """ 97 self._file = LockedFile(filename, 'r+b', 'rb') 98 self._thread_lock = threading.Lock() 99 self._read_only = False 100 self._warn_on_readonly = warn_on_readonly 101 102 self._create_file_if_needed() 103 104 # Cache of deserialized store. This is only valid after the 105 # _MultiStore is locked or _refresh_data_cache is called. This is 106 # of the form of: 107 # 108 # (client_id, user_agent, scope) -> OAuth2Credential 109 # 110 # If this is None, then the store hasn't been read yet. 111 self._data = None
112
113 - class _Storage(BaseStorage):
114 """A Storage object that knows how to read/write a single credential.""" 115
116 - def __init__(self, multistore, client_id, user_agent, scope):
117 self._multistore = multistore 118 self._client_id = client_id 119 self._user_agent = user_agent 120 self._scope = scope
121
122 - def acquire_lock(self):
123 """Acquires any lock necessary to access this Storage. 124 125 This lock is not reentrant. 126 """ 127 self._multistore._lock()
128
129 - def release_lock(self):
130 """Release the Storage lock. 131 132 Trying to release a lock that isn't held will result in a 133 RuntimeError. 134 """ 135 self._multistore._unlock()
136
137 - def locked_get(self):
138 """Retrieve credential. 139 140 The Storage lock must be held when this is called. 141 142 Returns: 143 oauth2client.client.Credentials 144 """ 145 credential = self._multistore._get_credential( 146 self._client_id, self._user_agent, self._scope) 147 if credential: 148 credential.set_store(self) 149 return credential
150
151 - def locked_put(self, credentials):
152 """Write a credential. 153 154 The Storage lock must be held when this is called. 155 156 Args: 157 credentials: Credentials, the credentials to store. 158 """ 159 self._multistore._update_credential(credentials, self._scope)
160
161 - def locked_delete(self):
162 """Delete a credential. 163 164 The Storage lock must be held when this is called. 165 166 Args: 167 credentials: Credentials, the credentials to store. 168 """ 169 self._multistore._delete_credential(self._client_id, self._user_agent, 170 self._scope)
171
172 - def _create_file_if_needed(self):
173 """Create an empty file if necessary. 174 175 This method will not initialize the file. Instead it implements a 176 simple version of "touch" to ensure the file has been created. 177 """ 178 if not os.path.exists(self._file.filename()): 179 old_umask = os.umask(0177) 180 try: 181 open(self._file.filename(), 'a+b').close() 182 finally: 183 os.umask(old_umask)
184
185 - def _lock(self):
186 """Lock the entire multistore.""" 187 self._thread_lock.acquire() 188 self._file.open_and_lock() 189 if not self._file.is_locked(): 190 self._read_only = True 191 if self._warn_on_readonly: 192 logger.warn('The credentials file (%s) is not writable. Opening in ' 193 'read-only mode. Any refreshed credentials will only be ' 194 'valid for this run.' % self._file.filename()) 195 if os.path.getsize(self._file.filename()) == 0: 196 logger.debug('Initializing empty multistore file') 197 # The multistore is empty so write out an empty file. 198 self._data = {} 199 self._write() 200 elif not self._read_only or self._data is None: 201 # Only refresh the data if we are read/write or we haven't 202 # cached the data yet. If we are readonly, we assume is isn't 203 # changing out from under us and that we only have to read it 204 # once. This prevents us from whacking any new access keys that 205 # we have cached in memory but were unable to write out. 206 self._refresh_data_cache()
207
208 - def _unlock(self):
209 """Release the lock on the multistore.""" 210 self._file.unlock_and_close() 211 self._thread_lock.release()
212
213 - def _locked_json_read(self):
214 """Get the raw content of the multistore file. 215 216 The multistore must be locked when this is called. 217 218 Returns: 219 The contents of the multistore decoded as JSON. 220 """ 221 assert self._thread_lock.locked() 222 self._file.file_handle().seek(0) 223 return simplejson.load(self._file.file_handle())
224
225 - def _locked_json_write(self, data):
226 """Write a JSON serializable data structure to the multistore. 227 228 The multistore must be locked when this is called. 229 230 Args: 231 data: The data to be serialized and written. 232 """ 233 assert self._thread_lock.locked() 234 if self._read_only: 235 return 236 self._file.file_handle().seek(0) 237 simplejson.dump(data, self._file.file_handle(), sort_keys=True, indent=2) 238 self._file.file_handle().truncate()
239
240 - def _refresh_data_cache(self):
241 """Refresh the contents of the multistore. 242 243 The multistore must be locked when this is called. 244 245 Raises: 246 NewerCredentialStoreError: Raised when a newer client has written the 247 store. 248 """ 249 self._data = {} 250 try: 251 raw_data = self._locked_json_read() 252 except Exception: 253 logger.warn('Credential data store could not be loaded. ' 254 'Will ignore and overwrite.') 255 return 256 257 version = 0 258 try: 259 version = raw_data['file_version'] 260 except Exception: 261 logger.warn('Missing version for credential data store. It may be ' 262 'corrupt or an old version. Overwriting.') 263 if version > 1: 264 raise NewerCredentialStoreError( 265 'Credential file has file_version of %d. ' 266 'Only file_version of 1 is supported.' % version) 267 268 credentials = [] 269 try: 270 credentials = raw_data['data'] 271 except (TypeError, KeyError): 272 pass 273 274 for cred_entry in credentials: 275 try: 276 (key, credential) = self._decode_credential_from_json(cred_entry) 277 self._data[key] = credential 278 except: 279 # If something goes wrong loading a credential, just ignore it 280 logger.info('Error decoding credential, skipping', exc_info=True)
281
282 - def _decode_credential_from_json(self, cred_entry):
283 """Load a credential from our JSON serialization. 284 285 Args: 286 cred_entry: A dict entry from the data member of our format 287 288 Returns: 289 (key, cred) where the key is the key tuple and the cred is the 290 OAuth2Credential object. 291 """ 292 raw_key = cred_entry['key'] 293 client_id = raw_key['clientId'] 294 user_agent = raw_key['userAgent'] 295 scope = raw_key['scope'] 296 key = (client_id, user_agent, scope) 297 credential = None 298 credential = Credentials.new_from_json(simplejson.dumps(cred_entry['credential'])) 299 return (key, credential)
300
301 - def _write(self):
302 """Write the cached data back out. 303 304 The multistore must be locked. 305 """ 306 raw_data = {'file_version': 1} 307 raw_creds = [] 308 raw_data['data'] = raw_creds 309 for (cred_key, cred) in self._data.items(): 310 raw_key = { 311 'clientId': cred_key[0], 312 'userAgent': cred_key[1], 313 'scope': cred_key[2] 314 } 315 raw_cred = simplejson.loads(cred.to_json()) 316 raw_creds.append({'key': raw_key, 'credential': raw_cred}) 317 self._locked_json_write(raw_data)
318
319 - def _get_credential(self, client_id, user_agent, scope):
320 """Get a credential from the multistore. 321 322 The multistore must be locked. 323 324 Args: 325 client_id: The client_id for the credential 326 user_agent: The user agent for the credential 327 scope: A string for the scope(s) being requested 328 329 Returns: 330 The credential specified or None if not present 331 """ 332 key = (client_id, user_agent, scope) 333 334 return self._data.get(key, None)
335
336 - def _update_credential(self, cred, scope):
337 """Update a credential and write the multistore. 338 339 This must be called when the multistore is locked. 340 341 Args: 342 cred: The OAuth2Credential to update/set 343 scope: The scope(s) that this credential covers 344 """ 345 key = (cred.client_id, cred.user_agent, scope) 346 self._data[key] = cred 347 self._write()
348
349 - def _delete_credential(self, client_id, user_agent, scope):
350 """Delete a credential and write the multistore. 351 352 This must be called when the multistore is locked. 353 354 Args: 355 client_id: The client_id for the credential 356 user_agent: The user agent for the credential 357 scope: The scope(s) that this credential covers 358 """ 359 key = (client_id, user_agent, scope) 360 try: 361 del self._data[key] 362 except KeyError: 363 pass 364 self._write()
365
366 - def _get_storage(self, client_id, user_agent, scope):
367 """Get a Storage object to get/set a credential. 368 369 This Storage is a 'view' into the multistore. 370 371 Args: 372 client_id: The client_id for the credential 373 user_agent: The user agent for the credential 374 scope: A string for the scope(s) being requested 375 376 Returns: 377 A Storage object that can be used to get/set this cred 378 """ 379 return self._Storage(self, client_id, user_agent, scope)
380