source file: /home/buildslave/tahoe/edgy/build/src/allmydata/uri.py
file stats: 365 lines, 365 executed: 100.0% covered
1.
2. import re, urllib
3. from zope.interface import implements
4. from twisted.python.components import registerAdapter
5. from allmydata import storage
6. from allmydata.util import base32, hashutil
7. from allmydata.interfaces import IURI, IDirnodeURI, IFileURI, IVerifierURI, \
8. IMutableFileURI, INewDirectoryURI, IReadonlyNewDirectoryURI
9.
10. # the URI shall be an ascii representation of the file. It shall contain
11. # enough information to retrieve and validate the contents. It shall be
12. # expressed in a limited character set (namely [TODO]).
13.
14. BASE32STR_128bits = '(%s{25}%s)' % (base32.BASE32CHAR, base32.BASE32CHAR_3bits)
15. BASE32STR_256bits = '(%s{51}%s)' % (base32.BASE32CHAR, base32.BASE32CHAR_1bits)
16.
17. SEP='(?::|%3A)'
18. NUMBER='([0-9]+)'
19.
20. # URIs (soon to be renamed "caps") are always allowed to come with a leading
21. # 'http://127.0.0.1:8123/uri/' that will be ignored.
22. OPTIONALHTTPLEAD=r'(?:https?://(?:127.0.0.1|localhost):8123/uri/)?'
23.
24.
25. class _BaseURI:
26. def __hash__(self):
27. return hash((self.__class__, self.to_string()))
28. def __cmp__(self, them):
29. if cmp(type(self), type(them)):
30. return cmp(type(self), type(them))
31. if cmp(self.__class__, them.__class__):
32. return cmp(self.__class__, them.__class__)
33. return cmp(self.to_string(), them.to_string())
34. def to_human_encoding(self):
35. return 'http://127.0.0.1:8123/uri/'+self.to_string()
36.
37. class CHKFileURI(_BaseURI):
38. implements(IURI, IFileURI)
39.
40. STRING_RE=re.compile('^URI:CHK:'+BASE32STR_128bits+':'+
41. BASE32STR_256bits+':'+NUMBER+':'+NUMBER+':'+NUMBER+
42. '$')
43. HUMAN_RE=re.compile('^'+OPTIONALHTTPLEAD+'URI'+SEP+'CHK'+SEP+
44. BASE32STR_128bits+SEP+BASE32STR_256bits+SEP+NUMBER+
45. SEP+NUMBER+SEP+NUMBER+'$')
46.
47. def __init__(self, key, uri_extension_hash, needed_shares, total_shares,
48. size):
49. self.key = key
50. self.uri_extension_hash = uri_extension_hash
51. self.needed_shares = needed_shares
52. self.total_shares = total_shares
53. self.size = size
54. self.storage_index = hashutil.storage_index_hash(self.key)
55. assert len(self.storage_index) == 16
56. self.storage_index = hashutil.storage_index_hash(key)
57. assert len(self.storage_index) == 16 # sha256 hash truncated to 128
58.
59. @classmethod
60. def init_from_human_encoding(cls, uri):
61. mo = cls.HUMAN_RE.search(uri)
62. assert mo, uri
63. return cls(base32.a2b(mo.group(1)), base32.a2b(mo.group(2)),
64. int(mo.group(3)), int(mo.group(4)), int(mo.group(5)))
65.
66. @classmethod
67. def init_from_string(cls, uri):
68. mo = cls.STRING_RE.search(uri)
69. assert mo, uri
70. return cls(base32.a2b(mo.group(1)), base32.a2b(mo.group(2)),
71. int(mo.group(3)), int(mo.group(4)), int(mo.group(5)))
72.
73. def to_string(self):
74. assert isinstance(self.needed_shares, int)
75. assert isinstance(self.total_shares, int)
76. assert isinstance(self.size, (int,long))
77.
78. return ('URI:CHK:%s:%s:%d:%d:%d' %
79. (base32.b2a(self.key),
80. base32.b2a(self.uri_extension_hash),
81. self.needed_shares,
82. self.total_shares,
83. self.size))
84.
85. def is_readonly(self):
86. return True
87. def is_mutable(self):
88. return False
89. def get_readonly(self):
90. return self
91.
92. def get_size(self):
93. return self.size
94.
95. def get_verifier(self):
96. return CHKFileVerifierURI(storage_index=self.storage_index,
97. uri_extension_hash=self.uri_extension_hash,
98. needed_shares=self.needed_shares,
99. total_shares=self.total_shares,
100. size=self.size)
101.
102. class CHKFileVerifierURI(_BaseURI):
103. implements(IVerifierURI)
104.
105. STRING_RE=re.compile('^URI:CHK-Verifier:'+BASE32STR_128bits+':'+
106. BASE32STR_256bits+':'+NUMBER+':'+NUMBER+':'+NUMBER)
107. HUMAN_RE=re.compile('^'+OPTIONALHTTPLEAD+'URI'+SEP+'CHK-Verifier'+SEP+
108. BASE32STR_128bits+SEP+BASE32STR_256bits+SEP+NUMBER+
109. SEP+NUMBER+SEP+NUMBER)
110.
111. def __init__(self, storage_index, uri_extension_hash,
112. needed_shares, total_shares, size):
113. assert len(storage_index) == 16
114. self.storage_index = storage_index
115. self.uri_extension_hash = uri_extension_hash
116. self.needed_shares = needed_shares
117. self.total_shares = total_shares
118. self.size = size
119.
120. @classmethod
121. def init_from_human_encoding(cls, uri):
122. mo = cls.HUMAN_RE.search(uri)
123. assert mo, uri
124. return cls(base32.a2b(mo.group(1)), base32.a2b(mo.group(2)),
125. int(mo.group(3)), int(mo.group(4)), int(mo.group(5)))
126.
127. @classmethod
128. def init_from_string(cls, uri):
129. mo = cls.STRING_RE.search(uri)
130. assert mo, (uri, cls, cls.STRING_RE)
131. return cls(storage.si_a2b(mo.group(1)), base32.a2b(mo.group(2)),
132. int(mo.group(3)), int(mo.group(4)), int(mo.group(5)))
133.
134. def to_string(self):
135. assert isinstance(self.needed_shares, int)
136. assert isinstance(self.total_shares, int)
137. assert isinstance(self.size, (int,long))
138.
139. return ('URI:CHK-Verifier:%s:%s:%d:%d:%d' %
140. (storage.si_b2a(self.storage_index),
141. base32.b2a(self.uri_extension_hash),
142. self.needed_shares,
143. self.total_shares,
144. self.size))
145.
146.
147. class LiteralFileURI(_BaseURI):
148. implements(IURI, IFileURI)
149.
150. STRING_RE=re.compile('^URI:LIT:'+base32.BASE32STR_anybytes+'$')
151. HUMAN_RE=re.compile('^'+OPTIONALHTTPLEAD+'URI'+SEP+'LIT'+SEP+base32.BASE32STR_anybytes+'$')
152.
153. def __init__(self, data=None):
154. if data is not None:
155. self.data = data
156.
157. @classmethod
158. def init_from_human_encoding(cls, uri):
159. mo = cls.HUMAN_RE.search(uri)
160. assert mo, uri
161. return cls(base32.a2b(mo.group(1)))
162.
163. @classmethod
164. def init_from_string(cls, uri):
165. mo = cls.STRING_RE.search(uri)
166. assert mo, uri
167. return cls(base32.a2b(mo.group(1)))
168.
169. def to_string(self):
170. return 'URI:LIT:%s' % base32.b2a(self.data)
171.
172. def is_readonly(self):
173. return True
174. def is_mutable(self):
175. return False
176. def get_readonly(self):
177. return self
178.
179. def get_verifier(self):
180. # LIT files need no verification, all the data is present in the URI
181. return None
182.
183. def get_size(self):
184. return len(self.data)
185.
186. class WriteableSSKFileURI(_BaseURI):
187. implements(IURI, IMutableFileURI)
188.
189. BASE_STRING='URI:SSK:'
190. STRING_RE=re.compile('^'+BASE_STRING+BASE32STR_128bits+':'+
191. BASE32STR_256bits+'$')
192. HUMAN_RE=re.compile('^'+OPTIONALHTTPLEAD+'URI'+SEP+'SSK'+SEP+
193. BASE32STR_128bits+SEP+BASE32STR_256bits+'$')
194.
195. def __init__(self, writekey, fingerprint):
196. self.writekey = writekey
197. self.readkey = hashutil.ssk_readkey_hash(writekey)
198. self.storage_index = hashutil.ssk_storage_index_hash(self.readkey)
199. assert len(self.storage_index) == 16
200. self.fingerprint = fingerprint
201.
202. @classmethod
203. def init_from_human_encoding(cls, uri):
204. mo = cls.HUMAN_RE.search(uri)
205. assert mo, uri
206. return cls(base32.a2b(mo.group(1)), base32.a2b(mo.group(2)))
207.
208. @classmethod
209. def init_from_string(cls, uri):
210. mo = cls.STRING_RE.search(uri)
211. assert mo, (uri, cls)
212. return cls(base32.a2b(mo.group(1)), base32.a2b(mo.group(2)))
213.
214. def to_string(self):
215. assert isinstance(self.writekey, str)
216. assert isinstance(self.fingerprint, str)
217. return 'URI:SSK:%s:%s' % (base32.b2a(self.writekey),
218. base32.b2a(self.fingerprint))
219.
220. def __repr__(self):
221. return "<%s %s>" % (self.__class__.__name__, self.abbrev())
222.
223. def abbrev(self):
224. return base32.b2a(self.writekey[:5])
225.
226. def is_readonly(self):
227. return False
228. def is_mutable(self):
229. return True
230. def get_readonly(self):
231. return ReadonlySSKFileURI(self.readkey, self.fingerprint)
232. def get_verifier(self):
233. return SSKVerifierURI(self.storage_index, self.fingerprint)
234.
235. class ReadonlySSKFileURI(_BaseURI):
236. implements(IURI, IMutableFileURI)
237.
238. BASE_STRING='URI:SSK-RO:'
239. STRING_RE=re.compile('^URI:SSK-RO:'+BASE32STR_128bits+':'+BASE32STR_256bits+'$')
240. HUMAN_RE=re.compile('^'+OPTIONALHTTPLEAD+'URI'+SEP+'SSK-RO'+SEP+BASE32STR_128bits+SEP+BASE32STR_256bits+'$')
241.
242. def __init__(self, readkey, fingerprint):
243. self.readkey = readkey
244. self.storage_index = hashutil.ssk_storage_index_hash(self.readkey)
245. assert len(self.storage_index) == 16
246. self.fingerprint = fingerprint
247.
248. @classmethod
249. def init_from_human_encoding(cls, uri):
250. mo = cls.HUMAN_RE.search(uri)
251. assert mo, uri
252. return cls(base32.a2b(mo.group(1)), base32.a2b(mo.group(2)))
253.
254. @classmethod
255. def init_from_string(cls, uri):
256. mo = cls.STRING_RE.search(uri)
257. assert mo, uri
258. return cls(base32.a2b(mo.group(1)), base32.a2b(mo.group(2)))
259.
260. def to_string(self):
261. assert isinstance(self.readkey, str)
262. assert isinstance(self.fingerprint, str)
263. return 'URI:SSK-RO:%s:%s' % (base32.b2a(self.readkey),
264. base32.b2a(self.fingerprint))
265.
266. def __repr__(self):
267. return "<%s %s>" % (self.__class__.__name__, self.abbrev())
268.
269. def abbrev(self):
270. return base32.b2a(self.readkey[:5])
271.
272. def is_readonly(self):
273. return True
274. def is_mutable(self):
275. return True
276. def get_readonly(self):
277. return self
278. def get_verifier(self):
279. return SSKVerifierURI(self.storage_index, self.fingerprint)
280.
281. class SSKVerifierURI(_BaseURI):
282. implements(IVerifierURI)
283.
284. BASE_STRING='URI:SSK-Verifier:'
285. STRING_RE=re.compile('^'+BASE_STRING+BASE32STR_128bits+':'+BASE32STR_256bits+'$')
286. HUMAN_RE=re.compile('^'+OPTIONALHTTPLEAD+'URI'+SEP+'SSK-Verifier'+SEP+BASE32STR_128bits+SEP+BASE32STR_256bits+'$')
287.
288. def __init__(self, storage_index, fingerprint):
289. assert len(storage_index) == 16
290. self.storage_index = storage_index
291. self.fingerprint = fingerprint
292.
293. @classmethod
294. def init_from_human_encoding(cls, uri):
295. mo = cls.HUMAN_RE.search(uri)
296. assert mo, uri
297. return cls(storage.si_a2b(mo.group(1)), base32.a2b(mo.group(2)))
298.
299. @classmethod
300. def init_from_string(cls, uri):
301. mo = cls.STRING_RE.search(uri)
302. assert mo, (uri, cls)
303. return cls(storage.si_a2b(mo.group(1)), base32.a2b(mo.group(2)))
304.
305. def to_string(self):
306. assert isinstance(self.storage_index, str)
307. assert isinstance(self.fingerprint, str)
308. return 'URI:SSK-Verifier:%s:%s' % (storage.si_b2a(self.storage_index),
309. base32.b2a(self.fingerprint))
310.
311. class _NewDirectoryBaseURI(_BaseURI):
312. implements(IURI, IDirnodeURI)
313. def __init__(self, filenode_uri=None):
314. self._filenode_uri = filenode_uri
315.
316. def __repr__(self):
317. return "<%s %s>" % (self.__class__.__name__, self.abbrev())
318.
319. @classmethod
320. def init_from_string(cls, uri):
321. mo = cls.BASE_STRING_RE.search(uri)
322. assert mo, (uri, cls)
323. bits = uri[mo.end():]
324. fn = cls.INNER_URI_CLASS.init_from_string(
325. cls.INNER_URI_CLASS.BASE_STRING+bits)
326. return cls(fn)
327.
328. @classmethod
329. def init_from_human_encoding(cls, uri):
330. mo = cls.BASE_HUMAN_RE.search(uri)
331. assert mo, (uri, cls)
332. bits = uri[mo.end():]
333. while bits and bits[-1] == '/':
334. bits = bits[:-1]
335. fn = cls.INNER_URI_CLASS.init_from_string(
336. cls.INNER_URI_CLASS.BASE_STRING+urllib.unquote(bits))
337. return cls(fn)
338.
339. def to_string(self):
340. fnuri = self._filenode_uri.to_string()
341. mo = re.match(self.INNER_URI_CLASS.BASE_STRING, fnuri)
342. assert mo, fnuri
343. bits = fnuri[mo.end():]
344. return self.BASE_STRING+bits
345.
346. def abbrev(self):
347. return self._filenode_uri.to_string().split(':')[2][:5]
348.
349. def get_filenode_uri(self):
350. return self._filenode_uri
351.
352. def is_mutable(self):
353. return True
354.
355. def get_verifier(self):
356. return NewDirectoryURIVerifier(self._filenode_uri.get_verifier())
357.
358. class NewDirectoryURI(_NewDirectoryBaseURI):
359. implements(INewDirectoryURI)
360.
361. BASE_STRING='URI:DIR2:'
362. BASE_STRING_RE=re.compile('^'+BASE_STRING)
363. BASE_HUMAN_RE=re.compile('^'+OPTIONALHTTPLEAD+'URI'+SEP+'DIR2'+SEP)
364. INNER_URI_CLASS=WriteableSSKFileURI
365.
366. def __init__(self, filenode_uri=None):
367. if filenode_uri:
368. assert not filenode_uri.is_readonly()
369. _NewDirectoryBaseURI.__init__(self, filenode_uri)
370.
371. def is_readonly(self):
372. return False
373.
374. def get_readonly(self):
375. return ReadonlyNewDirectoryURI(self._filenode_uri.get_readonly())
376.
377. class ReadonlyNewDirectoryURI(_NewDirectoryBaseURI):
378. implements(IReadonlyNewDirectoryURI)
379.
380. BASE_STRING='URI:DIR2-RO:'
381. BASE_STRING_RE=re.compile('^'+BASE_STRING)
382. BASE_HUMAN_RE=re.compile('^'+OPTIONALHTTPLEAD+'URI'+SEP+'DIR2-RO'+SEP)
383. INNER_URI_CLASS=ReadonlySSKFileURI
384.
385. def __init__(self, filenode_uri=None):
386. if filenode_uri:
387. assert filenode_uri.is_readonly()
388. _NewDirectoryBaseURI.__init__(self, filenode_uri)
389.
390. def is_readonly(self):
391. return True
392.
393. def get_readonly(self):
394. return self
395.
396. class NewDirectoryURIVerifier(_NewDirectoryBaseURI):
397. implements(IVerifierURI)
398.
399. BASE_STRING='URI:DIR2-Verifier:'
400. BASE_STRING_RE=re.compile('^'+BASE_STRING)
401. BASE_HUMAN_RE=re.compile('^'+OPTIONALHTTPLEAD+'URI'+SEP+'DIR2-Verifier'+SEP)
402. INNER_URI_CLASS=SSKVerifierURI
403.
404. def __init__(self, filenode_uri=None):
405. if filenode_uri:
406. filenode_uri = IVerifierURI(filenode_uri)
407. self._filenode_uri = filenode_uri
408.
409. def get_filenode_uri(self):
410. return self._filenode_uri
411.
412.
413.
414. def from_string(s):
415. if s.startswith('URI:CHK:'):
416. return CHKFileURI.init_from_string(s)
417. elif s.startswith('URI:CHK-Verifier:'):
418. return CHKFileVerifierURI.init_from_string(s)
419. elif s.startswith('URI:LIT:'):
420. return LiteralFileURI.init_from_string(s)
421. elif s.startswith('URI:SSK:'):
422. return WriteableSSKFileURI.init_from_string(s)
423. elif s.startswith('URI:SSK-RO:'):
424. return ReadonlySSKFileURI.init_from_string(s)
425. elif s.startswith('URI:SSK-Verifier:'):
426. return SSKVerifierURI.init_from_string(s)
427. elif s.startswith('URI:DIR2:'):
428. return NewDirectoryURI.init_from_string(s)
429. elif s.startswith('URI:DIR2-RO:'):
430. return ReadonlyNewDirectoryURI.init_from_string(s)
431. elif s.startswith('URI:DIR2-Verifier:'):
432. return NewDirectoryURIVerifier.init_from_string(s)
433. else:
434. raise TypeError("unknown URI type: %s.." % s[:12])
435.
436. registerAdapter(from_string, str, IURI)
437.
438. def is_uri(s):
439. try:
440. uri = from_string(s)
441. return True
442. except (TypeError, AssertionError):
443. return False
444.
445. def from_string_dirnode(s):
446. u = from_string(s)
447. assert IDirnodeURI.providedBy(u)
448. return u
449.
450. registerAdapter(from_string_dirnode, str, IDirnodeURI)
451.
452. def from_string_filenode(s):
453. u = from_string(s)
454. assert IFileURI.providedBy(u)
455. return u
456.
457. registerAdapter(from_string_filenode, str, IFileURI)
458.
459. def from_string_mutable_filenode(s):
460. u = from_string(s)
461. assert IMutableFileURI.providedBy(u)
462. return u
463. registerAdapter(from_string_mutable_filenode, str, IMutableFileURI)
464.
465. def from_string_verifier(s):
466. u = from_string(s)
467. assert IVerifierURI.providedBy(u)
468. return u
469. registerAdapter(from_string_verifier, str, IVerifierURI)
470.
471.
472. def pack_extension(data):
473. pieces = []
474. for k in sorted(data.keys()):
475. value = data[k]
476. if isinstance(value, (int, long)):
477. value = "%d" % value
478. assert isinstance(value, str), k
479. assert re.match(r'^[a-zA-Z_\-]+$', k)
480. pieces.append(k + ':' + hashutil.netstring(value))
481. uri_extension = ''.join(pieces)
482. return uri_extension
483.
484. def unpack_extension(data):
485. d = {}
486. while data:
487. colon = data.index(':')
488. key = data[:colon]
489. data = data[colon+1:]
490.
491. colon = data.index(':')
492. number = data[:colon]
493. length = int(number)
494. data = data[colon+1:]
495.
496. value = data[:length]
497. assert data[length] == ','
498. data = data[length+1:]
499.
500. d[key] = value
501.
502. # convert certain things to numbers
503. for intkey in ('size', 'segment_size', 'num_segments',
504. 'needed_shares', 'total_shares'):
505. if intkey in d:
506. d[intkey] = int(d[intkey])
507. return d
508.
509.
510. def unpack_extension_readable(data):
511. unpacked = unpack_extension(data)
512. unpacked["UEB_hash"] = hashutil.uri_extension_hash(data)
513. for k in sorted(unpacked.keys()):
514. if 'hash' in k:
515. unpacked[k] = base32.b2a(unpacked[k])
516. return unpacked
517.