Trees | Index | Help |
---|
|
1 2 from c3errors import * 3 from configParser import C3Object 4 import os, md5, sha, time 5 6 try: 7 # name when installed by hand 8 import bsddb3 as bdb 9 except: 10 # name that comes in python 2.3 11 import bsddb as bdb 12 1315 """ Base Store implementation. Provides non-storage-specific functions """ 16 17 # Instantiate some type of simple record store 18 currentId = -1 19 idNormalizer = None 2056 5722 C3Object.__init__(self, session, node, parent) 23 24 # databasePath is some sort of on disk filename 25 # Might be directory, but might not, so can't use defaultPath 26 # Not a path to an xml config 27 databasePath = self.get_path(session, 'databasePath') 28 if (not databasePath): 29 databasePath = "store_" + self.id 30 31 if (not os.path.isabs(databasePath)): 32 # Prepend defaultPath from parents 33 dfp = self.get_path(session, 'defaultPath') 34 if (not dfp): 35 raise(ConfigFileException("Store has relative path, and no visible defaultPath.")) 36 databasePath = os.path.join(dfp, databasePath) 37 38 # Record fully resolved path 39 self.paths['databasePath'] = databasePath 40 41 digestPath = self.get_path(session, 'digestPath') 42 if (digestPath): 43 if (not os.path.isabs(digestPath)): 44 # Prepend defaultPath from parents 45 dfp = self.get_path(session, 'defaultPath') 46 if (not dfp): 47 raise(ConfigFileException("Store has relative digestPath, and no visible defaultPath.")) 48 digestPath = os.path.join(dfp, digestPath) 49 self.paths['digestPath'] = digestPath 50 51 self.idNormalizer = self.get_path(session, 'idNormalizer') 52 self._verifyDatabases(session)5355 pass59 store = None 60 cursor = None 61 cxn = None 62 nextData = None 6391 9265 self.cxn = bdb.db.DB() 66 self.store = store 67 dbp = self.store.get_path(None, 'databasePath') 68 self.cxn.open(dbp) 69 self.cursor = self.cxn.cursor() 70 self.nextData = self.cursor.first()7173 return self7476 try: 77 while self.nextData and (self.nextData[0][:2] == "__"): 78 self.nextData = self.cursor.next() 79 d = self.nextData 80 if not d: 81 raise StopIteration() 82 self.nextData = self.cursor.next() 83 return d 84 except: 85 raise StopIteration()8694 """ Berkeley DB based storage """ 95 cxn = None 96 digestCxn = None 97 digestForId = 0 98 102218104 self.digestForId = self.get_setting(session, 'digestForId') 105 dbp = self.get_path(session, 'databasePath') 106 self._verifyDatabase(dbp) 107 dbp = self.get_path(session, 'digestPath') 108 if (dbp): 109 self._verifyDatabase(dbp)110112 if (not os.path.exists(dbp)): 113 # We don't exist, try and instantiate new database 114 self._initialise(dbp) 115 else: 116 cxn = bdb.db.DB() 117 try: 118 cxn.open(dbp) 119 cxn.close() 120 except: 121 # Still don't exist 122 self._initialise(dbp)123125 cxn = bdb.db.DB() 126 cxn.set_flags(bdb.db.DB_RECNUM) 127 cxn.open(dbp, dbtype=bdb.db.DB_BTREE, flags = bdb.db.DB_CREATE, mode=0660) 128 cxn.close()129131 if self.cxn == None: 132 cxn = bdb.db.DB() 133 cxn.set_flags(bdb.db.DB_RECNUM) 134 if session.environment == "apache": 135 cxn.open(self.get_path(None, 'databasePath'), flags=bdb.db.DB_NOMMAP) 136 else: 137 dbp = self.get_path(session, 'databasePath') 138 cxn.open(dbp) 139 self.cxn = cxn 140 dbp = self.get_path(None, 'digestPath') 141 if (dbp): 142 cxn = bdb.db.DB() 143 if session.environment == "apache": 144 cxn.open(dbp, flags=bdb.db.DB_NOMMAP) 145 else: 146 cxn.open(dbp) 147 self.digestCxn = cxn 148 else: 149 self.digestCxn = self.cxn150152 try: 153 self.cxn.close() 154 self.cxn = None 155 except: 156 pass 157 if self.digestCxn != None: 158 self.digestCxn.close() 159 self.digestCxn = None 160 else: 161 pass162 166 170172 self._openContainer(session) 173 # NB: First record is 0 174 if (self.currentId == -1 or session.environment == "apache"): 175 if (self.digestCxn != self.cxn or self.digestForId): 176 c = self.digestCxn.cursor() 177 item = c.last() 178 if item: 179 id = int(item[0][6:]) + 1 180 else: 181 id = 0 182 else: 183 c = self.cxn.cursor() 184 id = '999999999999' 185 if (self.idNormalizer): 186 id = self.idNormalizer.process_string(session, id) 187 try: 188 try: 189 item = c.set_range(id) 190 if (item[0][0] == "_"): 191 item = c.prev() 192 except: 193 item = c.last() 194 id = int(item[0]) + 1 195 except: 196 id = 0 197 else: 198 id = self.currentId +1 199 self.currentId = id 200 return id201 203 self._openContainer(session) 204 205 id = str(id) 206 if (self.idNormalizer <> None): 207 id = self.idNormalizer.process_string(session, id) 208 209 try: 210 self.cxn.put(id, data) 211 except: 212 data = data.encode('utf-8') 213 self.cxn.put(id, data) 214 215 if (size): 216 self.cxn.put("__size_%s" % id, str(size)) 217 return None220 # Check record doesn't already exist 221 digest = self.get_setting(session, "digest") 222 if (digest): 223 if (digest == 'md5'): 224 dmod = md5 225 elif (digest == 'sha'): 226 dmod = sha 227 else: 228 raise ConfigFileException("Unknown digest type: %s" % digest) 229 m = dmod.new() 230 231 data = data.encode('utf-8') 232 m.update(data) 233 234 self._openContainer(session) 235 digest = m.hexdigest() 236 cxn = self.digestCxn 237 exist = cxn.get("__d2i_" + digest) 238 if exist: 239 raise ObjectAlreadyExistsException(exist) 240 elif store: 241 self.store_checkSum(session, id, digest) 242 return digest243245 id = str(id) 246 if self.idNormalizer != None: 247 tid = self.idNormalizer.process_string(session, id) 248 if tid: 249 id = tid 250 cxn = self.digestCxn 251 cxn.put("__d2i_" + digest, id) 252 cxn.put("__i2d_" + id, digest)253255 self._openContainer(session) 256 sid = str(id) 257 if (self.idNormalizer <> None): 258 sid = self.idNormalizer.process_string(session, sid) 259 data = self.cxn.get(sid) 260 return data261263 self._openContainer(session) 264 digest = self.fetch_checksum(session, id) 265 sid = str(id) 266 if (self.idNormalizer <> None): 267 sid = self.idNormalizer.process_string(session, str(id)) 268 try: 269 self.cxn.delete(sid) 270 except: 271 # Already deleted? 272 pass 273 try: 274 self.cxn.delete("__size_%s" % sid) 275 except: 276 # Size not stored 277 pass 278 if digest: 279 cxn = self.digestCxn 280 cxn.delete("__d2i_" + digest) 281 cxn.delete("__i2d_" + sid) 282 self._closeContainer(session)283285 self._openContainer(session) 286 sid = str(id) 287 if (self.idNormalizer <> None): 288 sid = self.idNormalizer.process_string(session, sid) 289 rsz = self.cxn.get("__size_%s" % (sid)) 290 if (rsz): 291 return long(rsz) 292 else: 293 return -1294296 self._openContainer(session) 297 sid = str(id) 298 if self.idNormalizer != None: 299 sid = self.idNormalizer.process_string(session, sid) 300 cxn = self.digestCxn 301 data = cxn.get("__i2d_" + sid) 302 return data303 304306 # return numReq ids from start 307 ids = [] 308 self._openContainer(session) 309 310 if numReq == -1 and not start: 311 keys = self.cxn.keys() 312 f = lambda k: k[0:2] != "__" 313 return filter(f, keys) 314 315 sid = str(start) 316 if self.idNormalizer != None: 317 sid = self.idNormalizer.process_string(session, sid) 318 319 if (self.digestForId): 320 # use digest database for ids 321 c = self.digestCxn.cursor() 322 spt = "__i2d_%s" % sid 323 try: 324 (key, data) = c.set_range(spt) 325 except: 326 return [] 327 ids.append(key[6:]) 328 if numReq == -1: 329 tup = c.next() 330 while tup: 331 if tup[0][:5] == "__i2d": 332 ids.append(tup[0][6:]) 333 tup = c.next() 334 else: 335 tup = () 336 else: 337 for x in range(numReq-1): 338 tup = c.next() 339 if tup and tup[0][:5] == "__i2d": 340 ids.append(tup[0][6:]) 341 else: 342 break 343 return ids 344 345 # Use normal ids 346 c = self.cxn.cursor() 347 if (sid == ""): 348 try: 349 (key, data) = c.first() 350 except: 351 # No data in store 352 return [] 353 else: 354 try: 355 (key, data) = c.set_range(sid) 356 except: 357 # No data after point in store 358 return [] 359 ids.append(key) 360 if numReq == -1: 361 tup = c.next() 362 while tup: 363 if tup[0][0] != "_": 364 ids.append(tup[0]) 365 tup = c.next() 366 else: 367 for x in range(numReq-1): 368 tup = c.next() 369 if tup: 370 ids.append(tup[0]) 371 else: 372 break 373 return ids374376 # delete all entries 377 self._openContainer(session) 378 self.cxn.truncate() 379 self.digestCxn.truncate()380382 self._openContainer(session) 383 # requires db created with DB_RECNUM 384 nk = self.cxn.stat(bdb.db.DB_FAST_STAT)['nkeys'] 385 if self.digestCxn != self.cxn and self.digestCxn != None: 386 # rec, size, digest elsewhere 387 return nk / 2 388 elif not self.get_setting(session, 'digest'): 389 # rec, size, no digest 390 return nk / 2 391 else: 392 # rec, size, digest -> rec, rec -> digest 393 return nk / 4
Trees | Index | Help |
---|
Generated by Epydoc 3.0alpha2 on Wed Aug 9 18:11:15 2006 | http://epydoc.sf.net |