source file: /home/buildslave/tahoe/edgy/build/src/allmydata/interfaces.py
file stats: 316 lines, 316 executed: 100.0% covered
1.
2. from zope.interface import Interface
3. from foolscap.schema import StringConstraint, ListOf, TupleOf, SetOf, DictOf, \
4. ChoiceOf, IntegerConstraint
5. from foolscap import RemoteInterface, Referenceable
6.
7. HASH_SIZE=32
8.
9. Hash = StringConstraint(maxLength=HASH_SIZE,
10. minLength=HASH_SIZE)# binary format 32-byte SHA256 hash
11. Nodeid = StringConstraint(maxLength=20,
12. minLength=20) # binary format 20-byte SHA1 hash
13. FURL = StringConstraint(1000)
14. StorageIndex = StringConstraint(16)
15. URI = StringConstraint(300) # kind of arbitrary
16.
17. MAX_BUCKETS = 200 # per peer
18.
19. ShareData = StringConstraint(None)
20. URIExtensionData = StringConstraint(1000)
21. Number = IntegerConstraint(8) # 2**(8*8) == 16EiB ~= 18e18 ~= 18 exabytes
22. Offset = Number
23. ReadSize = int # the 'int' constraint is 2**31 == 2Gib -- large files are processed in not-so-large increments
24. WriteEnablerSecret = Hash # used to protect mutable bucket modifications
25. LeaseRenewSecret = Hash # used to protect bucket lease renewal requests
26. LeaseCancelSecret = Hash # used to protect bucket lease cancellation requests
27.
28. class RIStubClient(RemoteInterface):
29. """Each client publishes a service announcement for a dummy object called
30. the StubClient. This object doesn't actually offer any services, but the
31. announcement helps the Introducer keep track of which clients are
32. subscribed (so the grid admin can keep track of things like the size of
33. the grid and the client versions in use. This is the (empty)
34. RemoteInterface for the StubClient."""
35.
36. class RIBucketWriter(RemoteInterface):
37. def write(offset=Offset, data=ShareData):
38. return None
39.
40. def close():
41. """
42. If the data that has been written is incomplete or inconsistent then
43. the server will throw the data away, else it will store it for future
44. retrieval.
45. """
46. return None
47.
48. def abort():
49. """Abandon all the data that has been written.
50. """
51. return None
52.
53. class RIBucketReader(RemoteInterface):
54. def read(offset=Offset, length=ReadSize):
55. return ShareData
56.
57. TestVector = ListOf(TupleOf(Offset, ReadSize, str, str))
58. # elements are (offset, length, operator, specimen)
59. # operator is one of "lt, le, eq, ne, ge, gt"
60. # nop always passes and is used to fetch data while writing.
61. # you should use length==len(specimen) for everything except nop
62. DataVector = ListOf(TupleOf(Offset, ShareData))
63. # (offset, data). This limits us to 30 writes of 1MiB each per call
64. TestAndWriteVectorsForShares = DictOf(int,
65. TupleOf(TestVector,
66. DataVector,
67. ChoiceOf(None, Offset), # new_length
68. ))
69. ReadVector = ListOf(TupleOf(Offset, ReadSize))
70. ReadData = ListOf(ShareData)
71. # returns data[offset:offset+length] for each element of TestVector
72.
73. class RIStorageServer(RemoteInterface):
74. __remote_name__ = "RIStorageServer.tahoe.allmydata.com"
75.
76. def get_versions():
77. """
78. Return a tuple of (my_version, oldest_supported) strings. Each string can be parsed by
79. a pyutil.version_class.Version instance or a distutils.version.LooseVersion instance,
80. and then compared. The first goal is to make sure that nodes are not confused by
81. speaking to an incompatible peer. The second goal is to enable the development of
82. backwards-compatibility code.
83.
84. The meaning of the oldest_supported element is that if you treat this storage server as
85. though it were of that version, then you will not be disappointed.
86.
87. The precise meaning of this method might change in incompatible ways until we get the
88. whole compatibility scheme nailed down.
89. """
90. return TupleOf(str, str)
91.
92. def allocate_buckets(storage_index=StorageIndex,
93. renew_secret=LeaseRenewSecret,
94. cancel_secret=LeaseCancelSecret,
95. sharenums=SetOf(int, maxLength=MAX_BUCKETS),
96. allocated_size=Offset, canary=Referenceable):
97. """
98. @param storage_index: the index of the bucket to be created or
99. increfed.
100. @param sharenums: these are the share numbers (probably between 0 and
101. 99) that the sender is proposing to store on this
102. server.
103. @param renew_secret: This is the secret used to protect bucket refresh
104. This secret is generated by the client and
105. stored for later comparison by the server. Each
106. server is given a different secret.
107. @param cancel_secret: Like renew_secret, but protects bucket decref.
108. @param canary: If the canary is lost before close(), the bucket is
109. deleted.
110. @return: tuple of (alreadygot, allocated), where alreadygot is what we
111. already have and is what we hereby agree to accept. New
112. leases are added for shares in both lists.
113. """
114. return TupleOf(SetOf(int, maxLength=MAX_BUCKETS),
115. DictOf(int, RIBucketWriter, maxKeys=MAX_BUCKETS))
116.
117. def add_lease(storage_index=StorageIndex,
118. renew_secret=LeaseRenewSecret,
119. cancel_secret=LeaseCancelSecret):
120. """
121. Add a new lease on the given bucket. If the renew_secret matches an
122. existing lease, that lease will be renewed instead.
123. """
124. return None
125.
126. def renew_lease(storage_index=StorageIndex, renew_secret=LeaseRenewSecret):
127. """
128. Renew the lease on a given bucket. Some networks will use this, some
129. will not.
130. """
131. return None
132.
133. def cancel_lease(storage_index=StorageIndex,
134. cancel_secret=LeaseCancelSecret):
135. """
136. Cancel the lease on a given bucket. If this was the last lease on the
137. bucket, the bucket will be deleted.
138. """
139. return None
140.
141. def get_buckets(storage_index=StorageIndex):
142. return DictOf(int, RIBucketReader, maxKeys=MAX_BUCKETS)
143.
144.
145.
146. def slot_readv(storage_index=StorageIndex,
147. shares=ListOf(int), readv=ReadVector):
148. """Read a vector from the numbered shares associated with the given
149. storage index. An empty shares list means to return data from all
150. known shares. Returns a dictionary with one key per share."""
151. return DictOf(int, ReadData) # shnum -> results
152.
153. def slot_testv_and_readv_and_writev(storage_index=StorageIndex,
154. secrets=TupleOf(WriteEnablerSecret,
155. LeaseRenewSecret,
156. LeaseCancelSecret),
157. tw_vectors=TestAndWriteVectorsForShares,
158. r_vector=ReadVector,
159. ):
160. """General-purpose test-and-set operation for mutable slots. Perform
161. a bunch of comparisons against the existing shares. If they all pass,
162. then apply a bunch of write vectors to those shares. Then use the
163. read vectors to extract data from all the shares and return the data.
164.
165. This method is, um, large. The goal is to allow clients to update all
166. the shares associated with a mutable file in a single round trip.
167.
168. @param storage_index: the index of the bucket to be created or
169. increfed.
170. @param write_enabler: a secret that is stored along with the slot.
171. Writes are accepted from any caller who can
172. present the matching secret. A different secret
173. should be used for each slot*server pair.
174. @param renew_secret: This is the secret used to protect bucket refresh
175. This secret is generated by the client and
176. stored for later comparison by the server. Each
177. server is given a different secret.
178. @param cancel_secret: Like renew_secret, but protects bucket decref.
179.
180. The 'secrets' argument is a tuple of (write_enabler, renew_secret,
181. cancel_secret). The first is required to perform any write. The
182. latter two are used when allocating new shares. To simply acquire a
183. new lease on existing shares, use an empty testv and an empty writev.
184.
185. Each share can have a separate test vector (i.e. a list of
186. comparisons to perform). If all vectors for all shares pass, then all
187. writes for all shares are recorded. Each comparison is a 4-tuple of
188. (offset, length, operator, specimen), which effectively does a bool(
189. (read(offset, length)) OPERATOR specimen ) and only performs the
190. write if all these evaluate to True. Basic test-and-set uses 'eq'.
191. Write-if-newer uses a seqnum and (offset, length, 'lt', specimen).
192. Write-if-same-or-newer uses 'le'.
193.
194. Reads from the end of the container are truncated, and missing shares
195. behave like empty ones, so to assert that a share doesn't exist (for
196. use when creating a new share), use (0, 1, 'eq', '').
197.
198. The write vector will be applied to the given share, expanding it if
199. necessary. A write vector applied to a share number that did not
200. exist previously will cause that share to be created.
201.
202. Each write vector is accompanied by a 'new_length' argument. If
203. new_length is not None, use it to set the size of the container. This
204. can be used to pre-allocate space for a series of upcoming writes, or
205. truncate existing data. If the container is growing, new_length will
206. be applied before datav. If the container is shrinking, it will be
207. applied afterwards.
208.
209. The read vector is used to extract data from all known shares,
210. *before* any writes have been applied. The same vector is used for
211. all shares. This captures the state that was tested by the test
212. vector.
213.
214. This method returns two values: a boolean and a dict. The boolean is
215. True if the write vectors were applied, False if not. The dict is
216. keyed by share number, and each value contains a list of strings, one
217. for each element of the read vector.
218.
219. If the write_enabler is wrong, this will raise BadWriteEnablerError.
220. To enable share migration (using update_write_enabler), the exception
221. will have the nodeid used for the old write enabler embedded in it,
222. in the following string::
223.
224. The write enabler was recorded by nodeid '%s'.
225.
226. Note that the nodeid here is encoded using the same base32 encoding
227. used by Foolscap and allmydata.util.idlib.nodeid_b2a().
228.
229. """
230. return TupleOf(bool, DictOf(int, ReadData))
231.
232. class IStorageBucketWriter(Interface):
233. def put_block(segmentnum=int, data=ShareData):
234. """@param data: For most segments, this data will be 'blocksize'
235. bytes in length. The last segment might be shorter.
236. @return: a Deferred that fires (with None) when the operation completes
237. """
238.
239. def put_plaintext_hashes(hashes=ListOf(Hash, maxLength=2**20)):
240. """
241. @return: a Deferred that fires (with None) when the operation completes
242. """
243.
244. def put_crypttext_hashes(hashes=ListOf(Hash, maxLength=2**20)):
245. """
246. @return: a Deferred that fires (with None) when the operation completes
247. """
248.
249. def put_block_hashes(blockhashes=ListOf(Hash, maxLength=2**20)):
250. """
251. @return: a Deferred that fires (with None) when the operation completes
252. """
253.
254. def put_share_hashes(sharehashes=ListOf(TupleOf(int, Hash),
255. maxLength=2**20)):
256. """
257. @return: a Deferred that fires (with None) when the operation completes
258. """
259.
260. def put_uri_extension(data=URIExtensionData):
261. """This block of data contains integrity-checking information (hashes
262. of plaintext, crypttext, and shares), as well as encoding parameters
263. that are necessary to recover the data. This is a serialized dict
264. mapping strings to other strings. The hash of this data is kept in
265. the URI and verified before any of the data is used. All buckets for
266. a given file contain identical copies of this data.
267.
268. The serialization format is specified with the following pseudocode:
269. for k in sorted(dict.keys()):
270. assert re.match(r'^[a-zA-Z_\-]+$', k)
271. write(k + ':' + netstring(dict[k]))
272.
273. @return: a Deferred that fires (with None) when the operation completes
274. """
275.
276. def close():
277. """Finish writing and close the bucket. The share is not finalized
278. until this method is called: if the uploading client disconnects
279. before calling close(), the partially-written share will be
280. discarded.
281.
282. @return: a Deferred that fires (with None) when the operation completes
283. """
284.
285. class IStorageBucketReader(Interface):
286.
287. def get_block(blocknum=int):
288. """Most blocks will be the same size. The last block might be shorter
289. than the others.
290.
291. @return: ShareData
292. """
293.
294. def get_plaintext_hashes():
295. """
296. @return: ListOf(Hash, maxLength=2**20)
297. """
298.
299. def get_crypttext_hashes():
300. """
301. @return: ListOf(Hash, maxLength=2**20)
302. """
303.
304. def get_block_hashes():
305. """
306. @return: ListOf(Hash, maxLength=2**20)
307. """
308.
309. def get_share_hashes():
310. """
311. @return: ListOf(TupleOf(int, Hash), maxLength=2**20)
312. """
313.
314. def get_uri_extension():
315. """
316. @return: URIExtensionData
317. """
318.
319.
320.
321. # hm, we need a solution for forward references in schemas
322. from foolscap.schema import Any
323.
324. FileNode_ = Any() # TODO: foolscap needs constraints on copyables
325. DirectoryNode_ = Any() # TODO: same
326. AnyNode_ = ChoiceOf(FileNode_, DirectoryNode_)
327. EncryptedThing = str
328.
329. class IURI(Interface):
330. def init_from_string(uri):
331. """Accept a string (as created by my to_string() method) and populate
332. this instance with its data. I am not normally called directly,
333. please use the module-level uri.from_string() function to convert
334. arbitrary URI strings into IURI-providing instances."""
335.
336. def is_readonly():
337. """Return False if this URI be used to modify the data. Return True
338. if this URI cannot be used to modify the data."""
339.
340. def is_mutable():
341. """Return True if the data can be modified by *somebody* (perhaps
342. someone who has a more powerful URI than this one)."""
343.
344. def get_readonly():
345. """Return another IURI instance, which represents a read-only form of
346. this one. If is_readonly() is True, this returns self."""
347.
348. def get_verifier():
349. """Return an instance that provides IVerifierURI, which can be used
350. to check on the availability of the file or directory, without
351. providing enough capabilities to actually read or modify the
352. contents. This may return None if the file does not need checking or
353. verification (e.g. LIT URIs).
354. """
355.
356. def to_string():
357. """Return a string of printable ASCII characters, suitable for
358. passing into init_from_string."""
359.
360. class IVerifierURI(Interface):
361. def init_from_string(uri):
362. """Accept a string (as created by my to_string() method) and populate
363. this instance with its data. I am not normally called directly,
364. please use the module-level uri.from_string() function to convert
365. arbitrary URI strings into IURI-providing instances."""
366.
367. def to_string():
368. """Return a string of printable ASCII characters, suitable for
369. passing into init_from_string."""
370.
371. class IDirnodeURI(Interface):
372. """I am a URI which represents a dirnode."""
373.
374.
375. class IFileURI(Interface):
376. """I am a URI which represents a filenode."""
377. def get_size():
378. """Return the length (in bytes) of the file that I represent."""
379.
380. class IMutableFileURI(Interface):
381. """I am a URI which represents a mutable filenode."""
382. class INewDirectoryURI(Interface):
383. pass
384. class IReadonlyNewDirectoryURI(Interface):
385. pass
386.
387.
388. class IFilesystemNode(Interface):
389. def get_uri():
390. """
391. Return the URI that can be used by others to get access to this
392. node. If this node is read-only, the URI will only offer read-only
393. access. If this node is read-write, the URI will offer read-write
394. access.
395.
396. If you have read-write access to a node and wish to share merely
397. read-only access with others, use get_readonly_uri().
398. """
399.
400. def get_readonly_uri():
401. """Return the directory URI that can be used by others to get
402. read-only access to this directory node. The result is a read-only
403. URI, regardless of whether this dirnode is read-only or read-write.
404.
405. If you have merely read-only access to this dirnode,
406. get_readonly_uri() will return the same thing as get_uri().
407. """
408.
409. def get_verifier():
410. """Return an IVerifierURI instance that represents the
411. 'verifiy/refresh capability' for this node. The holder of this
412. capability will be able to renew the lease for this node, protecting
413. it from garbage-collection. They will also be able to ask a server if
414. it holds a share for the file or directory.
415. """
416.
417. def get_storage_index():
418. """Return a string with the (binary) storage index in use on this
419. download. This may be None if there is no storage index (i.e. LIT
420. files)."""
421.
422. def check(verify=False, repair=False):
423. """Perform a file check. See IChecker.check for details.
424.
425. The default mode is named 'check' and simply asks each server whether
426. or not it has a share, and believes the answers. If verify=True, this
427. switches into the 'verify' mode, in which every byte of every share
428. is retrieved, and all hashes are verified. This uses much more
429. network and disk bandwidth than simple checking, especially for large
430. files, but will catch disk errors.
431.
432. The default repair=False argument means that files which are not
433. perfectly healthy will be reported, but not fixed. When repair=True,
434. the node will attempt to repair the file first. The results will
435. indicate the initial status of the file in either case. If repair was
436. attempted, the results will indicate that too.
437. """
438.
439. def is_readonly():
440. """Return True if this reference provides mutable access to the given
441. file or directory (i.e. if you can modify it), or False if not. Note
442. that even if this reference is read-only, someone else may hold a
443. read-write reference to it."""
444.
445. def is_mutable():
446. """Return True if this file or directory is mutable (by *somebody*,
447. not necessarily you), False if it is is immutable. Note that a file
448. might be mutable overall, but your reference to it might be
449. read-only. On the other hand, all references to an immutable file
450. will be read-only; there are no read-write references to an immutable
451. file.
452. """
453.
454. class IMutableFilesystemNode(IFilesystemNode):
455. pass
456.
457. class IFileNode(IFilesystemNode):
458. def download(target):
459. """Download the file's contents to a given IDownloadTarget"""
460.
461. def download_to_data():
462. """Download the file's contents. Return a Deferred that fires
463. with those contents."""
464.
465. def get_size():
466. """Return the length (in bytes) of the data this node represents."""
467.
468. class IMutableFileNode(IFileNode, IMutableFilesystemNode):
469. """I provide access to a 'mutable file', which retains its identity
470. regardless of what contents are put in it.
471.
472. The consistency-vs-availability problem means that there might be
473. multiple versions of a file present in the grid, some of which might be
474. unrecoverable (i.e. have fewer than 'k' shares). These versions are
475. loosely ordered: each has a sequence number and a hash, and any version
476. with seqnum=N was uploaded by a node which has seen at least one version
477. with seqnum=N-1.
478.
479. The 'servermap' (an instance of IMutableFileServerMap) is used to
480. describe the versions that are known to be present in the grid, and which
481. servers are hosting their shares. It is used to represent the 'state of
482. the world', and is used for this purpose by my test-and-set operations.
483. Downloading the contents of the mutable file will also return a
484. servermap. Uploading a new version into the mutable file requires a
485. servermap as input, and the semantics of the replace operation is
486. 'replace the file with my new version if it looks like nobody else has
487. changed the file since my previous download'. Because the file is
488. distributed, this is not a perfect test-and-set operation, but it will do
489. its best. If the replace process sees evidence of a simultaneous write,
490. it will signal an UncoordinatedWriteError, so that the caller can take
491. corrective action.
492.
493.
494. Most readers will want to use the 'best' current version of the file, and
495. should use my 'download_best_version()' method.
496.
497. To unconditionally replace the file, callers should use overwrite(). This
498. is the mode that user-visible mutable files will probably use.
499.
500. To apply some delta to the file, call modify() with a callable modifier
501. function that can apply the modification that you want to make. This is
502. the mode that dirnodes will use, since most directory modification
503. operations can be expressed in terms of deltas to the directory state.
504.
505.
506. Three methods are available for users who need to perform more complex
507. operations. The first is get_servermap(), which returns an up-to-date
508. servermap using a specified mode. The second is download_version(), which
509. downloads a specific version (not necessarily the 'best' one). The third
510. is 'upload', which accepts new contents and a servermap (which must have
511. been updated with MODE_WRITE). The upload method will attempt to apply
512. the new contents as long as no other node has modified the file since the
513. servermap was updated. This might be useful to a caller who wants to
514. merge multiple versions into a single new one.
515.
516. Note that each time the servermap is updated, a specific 'mode' is used,
517. which determines how many peers are queried. To use a servermap for my
518. replace() method, that servermap must have been updated in MODE_WRITE.
519. These modes are defined in allmydata.mutable.common, and consist of
520. MODE_READ, MODE_WRITE, MODE_ANYTHING, and MODE_CHECK. Please look in
521. allmydata/mutable/servermap.py for details about the differences.
522.
523. Mutable files are currently limited in size (about 3.5MB max) and can
524. only be retrieved and updated all-at-once, as a single big string. Future
525. versions of our mutable files will remove this restriction.
526. """
527.
528. def download_best_version():
529. """Download the 'best' available version of the file, meaning one of
530. the recoverable versions with the highest sequence number. If no
531. uncoordinated writes have occurred, and if enough shares are
532. available, then this will be the most recent version that has been
533. uploaded.
534.
535. I return a Deferred that fires with a (contents, servermap) pair. The
536. servermap is updated with MODE_READ. The contents will be the version
537. of the file indicated by servermap.best_recoverable_version(). If no
538. version is recoverable, the Deferred will errback with
539. UnrecoverableFileError.
540. """
541.
542. def get_size_of_best_version():
543. """Find the size of the version that would be downloaded with
544. download_best_version(), without actually downloading the whole file.
545.
546. I return a Deferred that fires with an integer.
547. """
548.
549. def overwrite(new_contents):
550. """Unconditionally replace the contents of the mutable file with new
551. ones. This simply chains get_servermap(MODE_WRITE) and upload(). This
552. is only appropriate to use when the new contents of the file are
553. completely unrelated to the old ones, and you do not care about other
554. clients' changes.
555.
556. I return a Deferred that fires (with a PublishStatus object) when the
557. update has completed.
558. """
559.
560. def modify(modifier_cb):
561. """Modify the contents of the file, by downloading the current
562. version, applying the modifier function (or bound method), then
563. uploading the new version. I return a Deferred that fires (with a
564. PublishStatus object) when the update is complete.
565.
566. The modifier callable will be given two arguments: a string (with the
567. old contents) and a servermap. As with download_best_version(), the
568. old contents will be from the best recoverable version, but the
569. modifier can use the servermap to make other decisions (such as
570. refusing to apply the delta if there are multiple parallel versions,
571. or if there is evidence of a newer unrecoverable version).
572.
573. The callable should return a string with the new contents. The
574. callable must be prepared to be called multiple times, and must
575. examine the input string to see if the change that it wants to make
576. is already present in the old version. If it does not need to make
577. any changes, it can either return None, or return its input string.
578.
579. If the modifier raises an exception, it will be returned in the
580. errback.
581. """
582.
583.
584. def get_servermap(mode):
585. """Return a Deferred that fires with an IMutableFileServerMap
586. instance, updated using the given mode.
587. """
588.
589. def download_version(servermap, version):
590. """Download a specific version of the file, using the servermap
591. as a guide to where the shares are located.
592.
593. I return a Deferred that fires with the requested contents, or
594. errbacks with UnrecoverableFileError. Note that a servermap which was
595. updated with MODE_ANYTHING or MODE_READ may not know about shares for
596. all versions (those modes stop querying servers as soon as they can
597. fulfil their goals), so you may want to use MODE_CHECK (which checks
598. everything) to get increased visibility.
599. """
600.
601. def upload(new_contents, servermap):
602. """Replace the contents of the file with new ones. This requires a
603. servermap that was previously updated with MODE_WRITE.
604.
605. I attempt to provide test-and-set semantics, in that I will avoid
606. modifying any share that is different than the version I saw in the
607. servermap. However, if another node is writing to the file at the
608. same time as me, I may manage to update some shares while they update
609. others. If I see any evidence of this, I will signal
610. UncoordinatedWriteError, and the file will be left in an inconsistent
611. state (possibly the version you provided, possibly the old version,
612. possibly somebody else's version, and possibly a mix of shares from
613. all of these).
614.
615. The recommended response to UncoordinatedWriteError is to either
616. return it to the caller (since they failed to coordinate their
617. writes), or to attempt some sort of recovery. It may be sufficient to
618. wait a random interval (with exponential backoff) and repeat your
619. operation. If I do not signal UncoordinatedWriteError, then I was
620. able to write the new version without incident.
621.
622. I return a Deferred that fires (with a PublishStatus object) when the
623. publish has completed. I will update the servermap in-place with the
624. location of all new shares.
625. """
626.
627. def get_writekey():
628. """Return this filenode's writekey, or None if the node does not have
629. write-capability. This may be used to assist with data structures
630. that need to make certain data available only to writers, such as the
631. read-write child caps in dirnodes. The recommended process is to have
632. reader-visible data be submitted to the filenode in the clear (where
633. it will be encrypted by the filenode using the readkey), but encrypt
634. writer-visible data using this writekey.
635. """
636.
637. class ExistingChildError(Exception):
638. """A directory node was asked to add or replace a child that already
639. exists, and overwrite= was set to False."""
640.
641. class IDirectoryNode(IMutableFilesystemNode):
642. """I represent a name-to-child mapping, holding the tahoe equivalent of a
643. directory. All child names are unicode strings, and all children are some
644. sort of IFilesystemNode (either files or subdirectories).
645. """
646.
647. def get_uri():
648. """
649. The dirnode ('1') URI returned by this method can be used in
650. set_uri() on a different directory ('2') to 'mount' a reference to
651. this directory ('1') under the other ('2'). This URI is just a
652. string, so it can be passed around through email or other out-of-band
653. protocol.
654. """
655.
656. def get_readonly_uri():
657. """
658. The dirnode ('1') URI returned by this method can be used in
659. set_uri() on a different directory ('2') to 'mount' a reference to
660. this directory ('1') under the other ('2'). This URI is just a
661. string, so it can be passed around through email or other out-of-band
662. protocol.
663. """
664.
665. def list():
666. """I return a Deferred that fires with a dictionary mapping child
667. name (a unicode string) to (node, metadata_dict) tuples, in which
668. 'node' is either an IFileNode or IDirectoryNode, and 'metadata_dict'
669. is a dictionary of metadata."""
670.
671. def has_child(name):
672. """I return a Deferred that fires with a boolean, True if there
673. exists a child of the given name, False if not. The child name must
674. be a unicode string."""
675.
676. def get(name):
677. """I return a Deferred that fires with a specific named child node,
678. either an IFileNode or an IDirectoryNode. The child name must be a
679. unicode string."""
680.
681. def get_metadata_for(name):
682. """I return a Deferred that fires with the metadata dictionary for a
683. specific named child node. This metadata is stored in the *edge*, not
684. in the child, so it is attached to the parent dirnode rather than the
685. child dir-or-file-node. The child name must be a unicode string."""
686.
687. def set_metadata_for(name, metadata):
688. """I replace any existing metadata for the named child with the new
689. metadata. The child name must be a unicode string. This metadata is
690. stored in the *edge*, not in the child, so it is attached to the
691. parent dirnode rather than the child dir-or-file-node. I return a
692. Deferred (that fires with this dirnode) when the operation is
693. complete."""
694.
695. def get_child_at_path(path):
696. """Transform a child path into an IDirectoryNode or IFileNode.
697.
698. I perform a recursive series of 'get' operations to find the named
699. descendant node. I return a Deferred that fires with the node, or
700. errbacks with IndexError if the node could not be found.
701.
702. The path can be either a single string (slash-separated) or a list of
703. path-name elements. All elements must be unicode strings.
704. """
705.
706. def set_uri(name, child_uri, metadata=None, overwrite=True):
707. """I add a child (by URI) at the specific name. I return a Deferred
708. that fires when the operation finishes. If overwrite= is True, I will
709. replace any existing child of the same name, otherwise an existing
710. child will cause me to return ExistingChildError. The child name must
711. be a unicode string.
712.
713. The child_uri could be for a file, or for a directory (either
714. read-write or read-only, using a URI that came from get_uri() ).
715.
716. If metadata= is provided, I will use it as the metadata for the named
717. edge. This will replace any existing metadata. If metadata= is left
718. as the default value of None, I will set ['mtime'] to the current
719. time, and I will set ['ctime'] to the current time if there was not
720. already a child by this name present. This roughly matches the
721. ctime/mtime semantics of traditional filesystems.
722.
723. If this directory node is read-only, the Deferred will errback with a
724. NotMutableError."""
725.
726. def set_children(entries, overwrite=True):
727. """Add multiple (name, child_uri) pairs (or (name, child_uri,
728. metadata) triples) to a directory node. Returns a Deferred that fires
729. (with None) when the operation finishes. This is equivalent to
730. calling set_uri() multiple times, but is much more efficient. All
731. child names must be unicode strings.
732. """
733.
734. def set_node(name, child, metadata=None, overwrite=True):
735. """I add a child at the specific name. I return a Deferred that fires
736. when the operation finishes. This Deferred will fire with the child
737. node that was just added. I will replace any existing child of the
738. same name. The child name must be a unicode string.
739.
740. If metadata= is provided, I will use it as the metadata for the named
741. edge. This will replace any existing metadata. If metadata= is left
742. as the default value of None, I will set ['mtime'] to the current
743. time, and I will set ['ctime'] to the current time if there was not
744. already a child by this name present. This roughly matches the
745. ctime/mtime semantics of traditional filesystems.
746.
747. If this directory node is read-only, the Deferred will errback with a
748. NotMutableError."""
749.
750. def set_nodes(entries, overwrite=True):
751. """Add multiple (name, child_node) pairs (or (name, child_node,
752. metadata) triples) to a directory node. Returns a Deferred that fires
753. (with None) when the operation finishes. This is equivalent to
754. calling set_node() multiple times, but is much more efficient. All
755. child names must be unicode strings."""
756.
757.
758. def add_file(name, uploadable, metadata=None, overwrite=True):
759. """I upload a file (using the given IUploadable), then attach the
760. resulting FileNode to the directory at the given name. I set metadata
761. the same way as set_uri and set_node. The child name must be a
762. unicode string.
763.
764. I return a Deferred that fires (with the IFileNode of the uploaded
765. file) when the operation completes."""
766.
767. def delete(name):
768. """I remove the child at the specific name. I return a Deferred that
769. fires when the operation finishes. The child name must be a unicode
770. string."""
771.
772. def create_empty_directory(name, overwrite=True):
773. """I create and attach an empty directory at the given name. The
774. child name must be a unicode string. I return a Deferred that fires
775. when the operation finishes."""
776.
777. def move_child_to(current_child_name, new_parent, new_child_name=None,
778. overwrite=True):
779. """I take one of my children and move them to a new parent. The child
780. is referenced by name. On the new parent, the child will live under
781. 'new_child_name', which defaults to 'current_child_name'. TODO: what
782. should we do about metadata? I return a Deferred that fires when the
783. operation finishes. The child name must be a unicode string."""
784.
785. def build_manifest():
786. """Return a Deferred that fires with a frozenset of
787. verifier-capability strings for all nodes (directories and files)
788. reachable from this one."""
789.
790. def deep_stats():
791. """Return a Deferred that fires with a dictionary of statistics
792. computed by examining all nodes (directories and files) reachable
793. from this one, with the following keys::
794.
795. count-immutable-files: count of how many CHK files are in the set
796. count-mutable-files: same, for mutable files (does not include
797. directories)
798. count-literal-files: same, for LIT files
799. count-files: sum of the above three
800.
801. count-directories: count of directories
802.
803. size-immutable-files: total bytes for all CHK files in the set
804. size-mutable-files (TODO): same, for current version of all mutable
805. files, does not include directories
806. size-literal-files: same, for LIT files
807. size-directories: size of mutable files used by directories
808.
809. largest-directory: number of bytes in the largest directory
810. largest-directory-children: number of children in the largest
811. directory
812. largest-immutable-file: number of bytes in the largest CHK file
813.
814. size-mutable-files is not yet implemented, because it would involve
815. even more queries than deep_stats does.
816.
817. This operation will visit every directory node underneath this one,
818. and can take a long time to run. On a typical workstation with good
819. bandwidth, this can examine roughly 15 directories per second (and
820. takes several minutes of 100% CPU for ~1700 directories).
821. """
822.
823. class ICodecEncoder(Interface):
824. def set_params(data_size, required_shares, max_shares):
825. """Set up the parameters of this encoder.
826.
827. This prepares the encoder to perform an operation that converts a
828. single block of data into a number of shares, such that a future
829. ICodecDecoder can use a subset of these shares to recover the
830. original data. This operation is invoked by calling encode(). Once
831. the encoding parameters are set up, the encode operation can be
832. invoked multiple times.
833.
834. set_params() prepares the encoder to accept blocks of input data that
835. are exactly 'data_size' bytes in length. The encoder will be prepared
836. to produce 'max_shares' shares for each encode() operation (although
837. see the 'desired_share_ids' to use less CPU). The encoding math will
838. be chosen such that the decoder can get by with as few as
839. 'required_shares' of these shares and still reproduce the original
840. data. For example, set_params(1000, 5, 5) offers no redundancy at
841. all, whereas set_params(1000, 1, 10) provides 10x redundancy.
842.
843. Numerical Restrictions: 'data_size' is required to be an integral
844. multiple of 'required_shares'. In general, the caller should choose
845. required_shares and max_shares based upon their reliability
846. requirements and the number of peers available (the total storage
847. space used is roughly equal to max_shares*data_size/required_shares),
848. then choose data_size to achieve the memory footprint desired (larger
849. data_size means more efficient operation, smaller data_size means
850. smaller memory footprint).
851.
852. In addition, 'max_shares' must be equal to or greater than
853. 'required_shares'. Of course, setting them to be equal causes
854. encode() to degenerate into a particularly slow form of the 'split'
855. utility.
856.
857. See encode() for more details about how these parameters are used.
858.
859. set_params() must be called before any other ICodecEncoder methods
860. may be invoked.
861. """
862.
863. def get_encoder_type():
864. """Return a short string that describes the type of this encoder.
865.
866. There is required to be a global table of encoder classes. This method
867. returns an index into this table; the value at this index is an
868. encoder class, and this encoder is an instance of that class.
869. """
870.
871. def get_serialized_params(): # TODO: maybe, maybe not
872. """Return a string that describes the parameters of this encoder.
873.
874. This string can be passed to the decoder to prepare it for handling
875. the encoded shares we create. It might contain more information than
876. was presented to set_params(), if there is some flexibility of
877. parameter choice.
878.
879. This string is intended to be embedded in the URI, so there are
880. several restrictions on its contents. At the moment I'm thinking that
881. this means it may contain hex digits and hyphens, and nothing else.
882. The idea is that the URI contains something like '%s:%s:%s' %
883. (encoder.get_encoder_name(), encoder.get_serialized_params(),
884. b2a(crypttext_hash)), and this is enough information to construct a
885. compatible decoder.
886. """
887.
888. def get_block_size():
889. """Return the length of the shares that encode() will produce.
890. """
891.
892. def encode_proposal(data, desired_share_ids=None):
893. """Encode some data.
894.
895. 'data' must be a string (or other buffer object), and len(data) must
896. be equal to the 'data_size' value passed earlier to set_params().
897.
898. This will return a Deferred that will fire with two lists. The first
899. is a list of shares, each of which is a string (or other buffer
900. object) such that len(share) is the same as what get_share_size()
901. returned earlier. The second is a list of shareids, in which each is
902. an integer. The lengths of the two lists will always be equal to each
903. other. The user should take care to keep each share closely
904. associated with its shareid, as one is useless without the other.
905.
906. The length of this output list will normally be the same as the value
907. provided to the 'max_shares' parameter of set_params(). This may be
908. different if 'desired_share_ids' is provided.
909.
910. 'desired_share_ids', if provided, is required to be a sequence of
911. ints, each of which is required to be >= 0 and < max_shares. If not
912. provided, encode() will produce 'max_shares' shares, as if
913. 'desired_share_ids' were set to range(max_shares). You might use this
914. if you initially thought you were going to use 10 peers, started
915. encoding, and then two of the peers dropped out: you could use
916. desired_share_ids= to skip the work (both memory and CPU) of
917. producing shares for the peers which are no longer available.
918.
919. """
920.
921. def encode(inshares, desired_share_ids=None):
922. """Encode some data. This may be called multiple times. Each call is
923. independent.
924.
925. inshares is a sequence of length required_shares, containing buffers
926. (i.e. strings), where each buffer contains the next contiguous
927. non-overlapping segment of the input data. Each buffer is required to
928. be the same length, and the sum of the lengths of the buffers is
929. required to be exactly the data_size promised by set_params(). (This
930. implies that the data has to be padded before being passed to
931. encode(), unless of course it already happens to be an even multiple
932. of required_shares in length.)
933.
934. ALSO: the requirement to break up your data into 'required_shares'
935. chunks before calling encode() feels a bit surprising, at least from
936. the point of view of a user who doesn't know how FEC works. It feels
937. like an implementation detail that has leaked outside the
938. abstraction barrier. Can you imagine a use case in which the data to
939. be encoded might already be available in pre-segmented chunks, such
940. that it is faster or less work to make encode() take a list rather
941. than splitting a single string?
942.
943. ALSO ALSO: I think 'inshares' is a misleading term, since encode()
944. is supposed to *produce* shares, so what it *accepts* should be
945. something other than shares. Other places in this interface use the
946. word 'data' for that-which-is-not-shares.. maybe we should use that
947. term?
948.
949. 'desired_share_ids', if provided, is required to be a sequence of
950. ints, each of which is required to be >= 0 and < max_shares. If not
951. provided, encode() will produce 'max_shares' shares, as if
952. 'desired_share_ids' were set to range(max_shares). You might use this
953. if you initially thought you were going to use 10 peers, started
954. encoding, and then two of the peers dropped out: you could use
955. desired_share_ids= to skip the work (both memory and CPU) of
956. producing shares for the peers which are no longer available.
957.
958. For each call, encode() will return a Deferred that fires with two
959. lists, one containing shares and the other containing the shareids.
960. The get_share_size() method can be used to determine the length of
961. the share strings returned by encode(). Each shareid is a small
962. integer, exactly as passed into 'desired_share_ids' (or
963. range(max_shares), if desired_share_ids was not provided).
964.
965. The shares and their corresponding shareids are required to be kept
966. together during storage and retrieval. Specifically, the share data is
967. useless by itself: the decoder needs to be told which share is which
968. by providing it with both the shareid and the actual share data.
969.
970. This function will allocate an amount of memory roughly equal to::
971.
972. (max_shares - required_shares) * get_share_size()
973.
974. When combined with the memory that the caller must allocate to
975. provide the input data, this leads to a memory footprint roughly
976. equal to the size of the resulting encoded shares (i.e. the expansion
977. factor times the size of the input segment).
978. """
979.
980. # rejected ideas:
981. #
982. # returning a list of (shareidN,shareN) tuples instead of a pair of
983. # lists (shareids..,shares..). Brian thought the tuples would
984. # encourage users to keep the share and shareid together throughout
985. # later processing, Zooko pointed out that the code to iterate
986. # through two lists is not really more complicated than using a list
987. # of tuples and there's also a performance improvement
988. #
989. # having 'data_size' not required to be an integral multiple of
990. # 'required_shares'. Doing this would require encode() to perform
991. # padding internally, and we'd prefer to have any padding be done
992. # explicitly by the caller. Yes, it is an abstraction leak, but
993. # hopefully not an onerous one.
994.
995.
996. class ICodecDecoder(Interface):
997. def set_serialized_params(params):
998. """Set up the parameters of this encoder, from a string returned by
999. encoder.get_serialized_params()."""
1000.
1001. def get_needed_shares():
1002. """Return the number of shares needed to reconstruct the data.
1003. set_serialized_params() is required to be called before this."""
1004.
1005. def decode(some_shares, their_shareids):
1006. """Decode a partial list of shares into data.
1007.
1008. 'some_shares' is required to be a sequence of buffers of sharedata, a
1009. subset of the shares returned by ICodecEncode.encode(). Each share is
1010. required to be of the same length. The i'th element of their_shareids
1011. is required to be the shareid of the i'th buffer in some_shares.
1012.
1013. This returns a Deferred which fires with a sequence of buffers. This
1014. sequence will contain all of the segments of the original data, in
1015. order. The sum of the lengths of all of the buffers will be the
1016. 'data_size' value passed into the original ICodecEncode.set_params()
1017. call. To get back the single original input block of data, use
1018. ''.join(output_buffers), or you may wish to simply write them in
1019. order to an output file.
1020.
1021. Note that some of the elements in the result sequence may be
1022. references to the elements of the some_shares input sequence. In
1023. particular, this means that if those share objects are mutable (e.g.
1024. arrays) and if they are changed, then both the input (the
1025. 'some_shares' parameter) and the output (the value given when the
1026. deferred is triggered) will change.
1027.
1028. The length of 'some_shares' is required to be exactly the value of
1029. 'required_shares' passed into the original ICodecEncode.set_params()
1030. call.
1031. """
1032.
1033. class IEncoder(Interface):
1034. """I take an object that provides IEncryptedUploadable, which provides
1035. encrypted data, and a list of shareholders. I then encode, hash, and
1036. deliver shares to those shareholders. I will compute all the necessary
1037. Merkle hash trees that are necessary to validate the crypttext that
1038. eventually comes back from the shareholders. I provide the URI Extension
1039. Block Hash, and the encoding parameters, both of which must be included
1040. in the URI.
1041.
1042. I do not choose shareholders, that is left to the IUploader. I must be
1043. given a dict of RemoteReferences to storage buckets that are ready and
1044. willing to receive data.
1045. """
1046.
1047. def set_size(size):
1048. """Specify the number of bytes that will be encoded. This must be
1049. peformed before get_serialized_params() can be called.
1050. """
1051. def set_params(params):
1052. """Override the default encoding parameters. 'params' is a tuple of
1053. (k,d,n), where 'k' is the number of required shares, 'd' is the
1054. shares_of_happiness, and 'n' is the total number of shares that will
1055. be created.
1056.
1057. Encoding parameters can be set in three ways. 1: The Encoder class
1058. provides defaults (3/7/10). 2: the Encoder can be constructed with
1059. an 'options' dictionary, in which the
1060. needed_and_happy_and_total_shares' key can be a (k,d,n) tuple. 3:
1061. set_params((k,d,n)) can be called.
1062.
1063. If you intend to use set_params(), you must call it before
1064. get_share_size or get_param are called.
1065. """
1066.
1067. def set_encrypted_uploadable(u):
1068. """Provide a source of encrypted upload data. 'u' must implement
1069. IEncryptedUploadable.
1070.
1071. When this is called, the IEncryptedUploadable will be queried for its
1072. length and the storage_index that should be used.
1073.
1074. This returns a Deferred that fires with this Encoder instance.
1075.
1076. This must be performed before start() can be called.
1077. """
1078.
1079. def get_param(name):
1080. """Return an encoding parameter, by name.
1081.
1082. 'storage_index': return a string with the (16-byte truncated SHA-256
1083. hash) storage index to which these shares should be
1084. pushed.
1085.
1086. 'share_counts': return a tuple describing how many shares are used:
1087. (needed_shares, shares_of_happiness, total_shares)
1088.
1089. 'num_segments': return an int with the number of segments that
1090. will be encoded.
1091.
1092. 'segment_size': return an int with the size of each segment.
1093.
1094. 'block_size': return the size of the individual blocks that will
1095. be delivered to a shareholder's put_block() method. By
1096. knowing this, the shareholder will be able to keep all
1097. blocks in a single file and still provide random access
1098. when reading them. # TODO: can we avoid exposing this?
1099.
1100. 'share_size': an int with the size of the data that will be stored
1101. on each shareholder. This is aggregate amount of data
1102. that will be sent to the shareholder, summed over all
1103. the put_block() calls I will ever make. It is useful to
1104. determine this size before asking potential
1105. shareholders whether they will grant a lease or not,
1106. since their answers will depend upon how much space we
1107. need. TODO: this might also include some amount of
1108. overhead, like the size of all the hashes. We need to
1109. decide whether this is useful or not.
1110.
1111. 'serialized_params': a string with a concise description of the
1112. codec name and its parameters. This may be passed
1113. into the IUploadable to let it make sure that
1114. the same file encoded with different parameters
1115. will result in different storage indexes.
1116.
1117. Once this is called, set_size() and set_params() may not be called.
1118. """
1119.
1120. def set_shareholders(shareholders):
1121. """Tell the encoder where to put the encoded shares. 'shareholders'
1122. must be a dictionary that maps share number (an integer ranging from
1123. 0 to n-1) to an instance that provides IStorageBucketWriter. This
1124. must be performed before start() can be called."""
1125.
1126. def start():
1127. """Begin the encode/upload process. This involves reading encrypted
1128. data from the IEncryptedUploadable, encoding it, uploading the shares
1129. to the shareholders, then sending the hash trees.
1130.
1131. set_encrypted_uploadable() and set_shareholders() must be called
1132. before this can be invoked.
1133.
1134. This returns a Deferred that fires with a tuple of
1135. (uri_extension_hash, needed_shares, total_shares, size) when the
1136. upload process is complete. This information, plus the encryption
1137. key, is sufficient to construct the URI.
1138. """
1139.
1140. class IDecoder(Interface):
1141. """I take a list of shareholders and some setup information, then
1142. download, validate, decode, and decrypt data from them, writing the
1143. results to an output file.
1144.
1145. I do not locate the shareholders, that is left to the IDownloader. I must
1146. be given a dict of RemoteReferences to storage buckets that are ready to
1147. send data.
1148. """
1149.
1150. def setup(outfile):
1151. """I take a file-like object (providing write and close) to which all
1152. the plaintext data will be written.
1153.
1154. TODO: producer/consumer . Maybe write() should return a Deferred that
1155. indicates when it will accept more data? But probably having the
1156. IDecoder be a producer is easier to glue to IConsumer pieces.
1157. """
1158.
1159. def set_shareholders(shareholders):
1160. """I take a dictionary that maps share identifiers (small integers)
1161. to RemoteReferences that provide RIBucketReader. This must be called
1162. before start()."""
1163.
1164. def start():
1165. """I start the download. This process involves retrieving data and
1166. hash chains from the shareholders, using the hashes to validate the
1167. data, decoding the shares into segments, decrypting the segments,
1168. then writing the resulting plaintext to the output file.
1169.
1170. I return a Deferred that will fire (with self) when the download is
1171. complete.
1172. """
1173.
1174. class IDownloadTarget(Interface):
1175. # Note that if the IDownloadTarget is also an IConsumable, the downloader
1176. # will register itself as a producer. This allows the target to invoke
1177. # downloader.pauseProducing, resumeProducing, and stopProducing.
1178. def open(size):
1179. """Called before any calls to write() or close(). If an error
1180. occurs before any data is available, fail() may be called without
1181. a previous call to open().
1182.
1183. 'size' is the length of the file being downloaded, in bytes."""
1184.
1185. def write(data):
1186. """Output some data to the target."""
1187. def close():
1188. """Inform the target that there is no more data to be written."""
1189. def fail(why):
1190. """fail() is called to indicate that the download has failed. 'why'
1191. is a Failure object indicating what went wrong. No further methods
1192. will be invoked on the IDownloadTarget after fail()."""
1193. def register_canceller(cb):
1194. """The FileDownloader uses this to register a no-argument function
1195. that the target can call to cancel the download. Once this canceller
1196. is invoked, no further calls to write() or close() will be made."""
1197. def finish():
1198. """When the FileDownloader is done, this finish() function will be
1199. called. Whatever it returns will be returned to the invoker of
1200. Downloader.download.
1201. """
1202.
1203. class IDownloader(Interface):
1204. def download(uri, target):
1205. """Perform a CHK download, sending the data to the given target.
1206. 'target' must provide IDownloadTarget.
1207.
1208. Returns a Deferred that fires (with the results of target.finish)
1209. when the download is finished, or errbacks if something went wrong."""
1210.
1211. class IEncryptedUploadable(Interface):
1212. def set_upload_status(upload_status):
1213. """Provide an IUploadStatus object that should be filled with status
1214. information. The IEncryptedUploadable is responsible for setting
1215. key-determination progress ('chk'), size, storage_index, and
1216. ciphertext-fetch progress. It may delegate some of this
1217. responsibility to others, in particular to the IUploadable."""
1218.
1219. def get_size():
1220. """This behaves just like IUploadable.get_size()."""
1221.
1222. def get_all_encoding_parameters():
1223. """Return a Deferred that fires with a tuple of
1224. (k,happy,n,segment_size). The segment_size will be used as-is, and
1225. must match the following constraints: it must be a multiple of k, and
1226. it shouldn't be unreasonably larger than the file size (if
1227. segment_size is larger than filesize, the difference must be stored
1228. as padding).
1229.
1230. This usually passes through to the IUploadable method of the same
1231. name.
1232.
1233. The encoder strictly obeys the values returned by this method. To
1234. make an upload use non-default encoding parameters, you must arrange
1235. to control the values that this method returns.
1236. """
1237.
1238. def get_storage_index():
1239. """Return a Deferred that fires with a 16-byte storage index.
1240. """
1241.
1242. def read_encrypted(length, hash_only):
1243. """This behaves just like IUploadable.read(), but returns crypttext
1244. instead of plaintext. If hash_only is True, then this discards the
1245. data (and returns an empty list); this improves efficiency when
1246. resuming an interrupted upload (where we need to compute the
1247. plaintext hashes, but don't need the redundant encrypted data)."""
1248.
1249. def get_plaintext_hashtree_leaves(first, last, num_segments):
1250. """Get the leaf nodes of a merkle hash tree over the plaintext
1251. segments, i.e. get the tagged hashes of the given segments. The
1252. segment size is expected to be generated by the IEncryptedUploadable
1253. before any plaintext is read or ciphertext produced, so that the
1254. segment hashes can be generated with only a single pass.
1255.
1256. This returns a Deferred which fires with a sequence of hashes, using:
1257.
1258. tuple(segment_hashes[first:last])
1259.
1260. 'num_segments' is used to assert that the number of segments that the
1261. IEncryptedUploadable handled matches the number of segments that the
1262. encoder was expecting.
1263.
1264. This method must not be called until the final byte has been read
1265. from read_encrypted(). Once this method is called, read_encrypted()
1266. can never be called again.
1267. """
1268.
1269. def get_plaintext_hash():
1270. """Get the hash of the whole plaintext.
1271.
1272. This returns a Deferred which fires with a tagged SHA-256 hash of the
1273. whole plaintext, obtained from hashutil.plaintext_hash(data).
1274. """
1275.
1276. def close():
1277. """Just like IUploadable.close()."""
1278.
1279. class IUploadable(Interface):
1280. def set_upload_status(upload_status):
1281. """Provide an IUploadStatus object that should be filled with status
1282. information. The IUploadable is responsible for setting
1283. key-determination progress ('chk')."""
1284.
1285. def set_default_encoding_parameters(params):
1286. """Set the default encoding parameters, which must be a dict mapping
1287. strings to ints. The meaningful keys are 'k', 'happy', 'n', and
1288. 'max_segment_size'. These might have an influence on the final
1289. encoding parameters returned by get_all_encoding_parameters(), if the
1290. Uploadable doesn't have more specific preferences.
1291.
1292. This call is optional: if it is not used, the Uploadable will use
1293. some built-in defaults. If used, this method must be called before
1294. any other IUploadable methods to have any effect.
1295. """
1296.
1297. def get_size():
1298. """Return a Deferred that will fire with the length of the data to be
1299. uploaded, in bytes. This will be called before the data is actually
1300. used, to compute encoding parameters.
1301. """
1302.
1303. def get_all_encoding_parameters():
1304. """Return a Deferred that fires with a tuple of
1305. (k,happy,n,segment_size). The segment_size will be used as-is, and
1306. must match the following constraints: it must be a multiple of k, and
1307. it shouldn't be unreasonably larger than the file size (if
1308. segment_size is larger than filesize, the difference must be stored
1309. as padding).
1310.
1311. The relative values of k and n allow some IUploadables to request
1312. better redundancy than others (in exchange for consuming more space
1313. in the grid).
1314.
1315. Larger values of segment_size reduce hash overhead, while smaller
1316. values reduce memory footprint and cause data to be delivered in
1317. smaller pieces (which may provide a smoother and more predictable
1318. download experience).
1319.
1320. The encoder strictly obeys the values returned by this method. To
1321. make an upload use non-default encoding parameters, you must arrange
1322. to control the values that this method returns. One way to influence
1323. them may be to call set_encoding_parameters() before calling
1324. get_all_encoding_parameters().
1325. """
1326.
1327. def get_encryption_key():
1328. """Return a Deferred that fires with a 16-byte AES key. This key will
1329. be used to encrypt the data. The key will also be hashed to derive
1330. the StorageIndex.
1331.
1332. Uploadables which want to achieve convergence should hash their file
1333. contents and the serialized_encoding_parameters to form the key
1334. (which of course requires a full pass over the data). Uploadables can
1335. use the upload.ConvergentUploadMixin class to achieve this
1336. automatically.
1337.
1338. Uploadables which do not care about convergence (or do not wish to
1339. make multiple passes over the data) can simply return a
1340. strongly-random 16 byte string.
1341.
1342. get_encryption_key() may be called multiple times: the IUploadable is
1343. required to return the same value each time.
1344. """
1345.
1346. def read(length):
1347. """Return a Deferred that fires with a list of strings (perhaps with
1348. only a single element) which, when concatenated together, contain the
1349. next 'length' bytes of data. If EOF is near, this may provide fewer
1350. than 'length' bytes. The total number of bytes provided by read()
1351. before it signals EOF must equal the size provided by get_size().
1352.
1353. If the data must be acquired through multiple internal read
1354. operations, returning a list instead of a single string may help to
1355. reduce string copies.
1356.
1357. 'length' will typically be equal to (min(get_size(),1MB)/req_shares),
1358. so a 10kB file means length=3kB, 100kB file means length=30kB,
1359. and >=1MB file means length=300kB.
1360.
1361. This method provides for a single full pass through the data. Later
1362. use cases may desire multiple passes or access to only parts of the
1363. data (such as a mutable file making small edits-in-place). This API
1364. will be expanded once those use cases are better understood.
1365. """
1366.
1367. def close():
1368. """The upload is finished, and whatever filehandle was in use may be
1369. closed."""
1370.
1371. class IUploadResults(Interface):
1372. """I am returned by upload() methods. I contain a number of public
1373. attributes which can be read to determine the results of the upload. Some
1374. of these are functional, some are timing information. All of these may be
1375. None.::
1376.
1377. .file_size : the size of the file, in bytes
1378. .uri : the CHK read-cap for the file
1379. .ciphertext_fetched : how many bytes were fetched by the helper
1380. .sharemap : dict mapping share number to placement string
1381. .servermap : dict mapping server peerid to a set of share numbers
1382. .timings : dict of timing information, mapping name to seconds (float)
1383. total : total upload time, start to finish
1384. storage_index : time to compute the storage index
1385. peer_selection : time to decide which peers will be used
1386. contacting_helper : initial helper query to upload/no-upload decision
1387. existence_check : helper pre-upload existence check
1388. helper_total : initial helper query to helper finished pushing
1389. cumulative_fetch : helper waiting for ciphertext requests
1390. total_fetch : helper start to last ciphertext response
1391. cumulative_encoding : just time spent in zfec
1392. cumulative_sending : just time spent waiting for storage servers
1393. hashes_and_close : last segment push to shareholder close
1394. total_encode_and_push : first encode to shareholder close
1395.
1396. """
1397.
1398. class IDownloadResults(Interface):
1399. """I am created internally by download() methods. I contain a number of
1400. public attributes which contain details about the download process.::
1401.
1402. .file_size : the size of the file, in bytes
1403. .servers_used : set of server peerids that were used during download
1404. .server_problems : dict mapping server peerid to a problem string. Only
1405. servers that had problems (bad hashes, disconnects) are
1406. listed here.
1407. .servermap : dict mapping server peerid to a set of share numbers. Only
1408. servers that had any shares are listed here.
1409. .timings : dict of timing information, mapping name to seconds (float)
1410. peer_selection : time to ask servers about shares
1411. servers_peer_selection : dict of peerid to DYHB-query time
1412. uri_extension : time to fetch a copy of the URI extension block
1413. hashtrees : time to fetch the hash trees
1414. segments : time to fetch, decode, and deliver segments
1415. cumulative_fetch : time spent waiting for storage servers
1416. cumulative_decode : just time spent in zfec
1417. cumulative_decrypt : just time spent in decryption
1418. total : total download time, start to finish
1419. fetch_per_server : dict of peerid to list of per-segment fetch times
1420.
1421. """
1422.
1423. class IUploader(Interface):
1424. def upload(uploadable):
1425. """Upload the file. 'uploadable' must impement IUploadable. This
1426. returns a Deferred which fires with an UploadResults instance, from
1427. which the URI of the file can be obtained as results.uri ."""
1428.
1429. def upload_ssk(write_capability, new_version, uploadable):
1430. """TODO: how should this work?"""
1431.
1432. class ICheckable(Interface):
1433. def check(verify=False, repair=False):
1434. """Check upon my health, optionally repairing any problems.
1435.
1436. This returns a Deferred that fires with an instance that provides
1437. ICheckerResults.
1438.
1439. Filenodes and dirnodes (which provide IFilesystemNode) are also
1440. checkable. Instances that represent verifier-caps will be checkable
1441. but not downloadable. Some objects (like LIT files) do not actually
1442. live in the grid, and their checkers indicate a healthy result.
1443.
1444. If verify=False, a relatively lightweight check will be performed: I
1445. will ask all servers if they have a share for me, and I will believe
1446. whatever they say. If there are at least N distinct shares on the
1447. grid, my results will indicate r.is_healthy()==True. This requires a
1448. roundtrip to each server, but does not transfer very much data, so
1449. the network bandwidth is fairly low.
1450.
1451. If verify=True, a more resource-intensive check will be performed:
1452. every share will be downloaded, and the hashes will be validated on
1453. every bit. I will ignore any shares that failed their hash checks. If
1454. there are at least N distinct valid shares on the grid, my results
1455. will indicate r.is_healthy()==True. This requires N/k times as much
1456. download bandwidth (and server disk IO) as a regular download. If a
1457. storage server is holding a corrupt share, or is experiencing memory
1458. failures during retrieval, or is malicious or buggy, then
1459. verification will detect the problem, but checking will not.
1460.
1461. If repair=True, then a non-healthy result will cause an immediate
1462. repair operation, to generate and upload new shares. After repair,
1463. the file will be as healthy as we can make it. Details about what
1464. sort of repair is done will be put in the checker results. My
1465. Deferred will not fire until the repair is complete.
1466.
1467. TODO: any problems seen during checking will be reported to the
1468. health-manager.furl, a centralized object which is responsible for
1469. figuring out why files are unhealthy so corrective action can be
1470. taken.
1471. """
1472.
1473. def deep_check(verify=False, repair=False):
1474. """Check upon the health of me and everything I can reach.
1475.
1476. This is a recursive form of check(), useable on dirnodes. (it can be
1477. called safely on filenodes too, but only checks the one object).
1478.
1479. I return a Deferred that fires with an IDeepCheckResults object.
1480. """
1481.
1482. class ICheckerResults(Interface):
1483. """I contain the detailed results of a check/verify/repair operation.
1484.
1485. The IFilesystemNode.check()/verify()/repair() methods all return
1486. instances that provide ICheckerResults.
1487. """
1488.
1489. def is_healthy():
1490. """Return a bool, True if the file is fully healthy, False if it is
1491. damaged in any way."""
1492.
1493. def get_storage_index():
1494. """Return a string with the (binary) storage index."""
1495. def get_storage_index_string():
1496. """Return a string with the (printable) abbreviated storage index."""
1497. def get_mutability_string():
1498. """Return a string with 'mutable' or 'immutable'."""
1499.
1500. def to_string():
1501. """Return a string that describes the detailed results of the
1502. check/verify operation. This string will be displayed on a page all
1503. by itself."""
1504.
1505. # The old checker results (for only immutable files) were described
1506. # with this:
1507. # For filenodes, this fires with a tuple of (needed_shares,
1508. # total_shares, found_shares, sharemap). The first three are ints. The
1509. # basic health of the file is found_shares / needed_shares: if less
1510. # than 1.0, the file is unrecoverable.
1511. #
1512. # The sharemap has a key for each sharenum. The value is a list of
1513. # (binary) nodeids who hold that share. If two shares are kept on the
1514. # same nodeid, they will fail as a pair, and overall reliability is
1515. # decreased.
1516.
1517. class IDeepCheckResults(Interface):
1518. """I contain the results of a deep-check operation.
1519.
1520. This is returned by a call to ICheckable.deep_check().
1521. """
1522.
1523. def get_root_storage_index_string():
1524. """Return the storage index (abbreviated human-readable string) of
1525. the first object checked."""
1526. def count_objects_checked():
1527. """Return the number of objects that were checked."""
1528. def count_objects_healthy():
1529. """Return the number of objects that were fully healthy."""
1530. def count_repairs_attempted():
1531. """Return the number of repair operations that were attempted."""
1532. def count_repairs_successful():
1533. """Return the number of repair operations that succeeded in bringing
1534. the object back up to full health."""
1535. def get_server_problems():
1536. """Return a dict, mapping server nodeid to a count of how many
1537. problems involved that server."""
1538. def get_problems():
1539. """Return a list of ICheckerResults, one for each object that
1540. was not fully healthy."""
1541. def get_all_results():
1542. """Return a dict mapping storage_index (a binary string) to an
1543. ICheckerResults instance, one for each object that was checked."""
1544.
1545. class IRepairable(Interface):
1546. def repair(checker_results):
1547. """Attempt to repair the given object. Returns a Deferred that fires
1548. with a IRepairResults object.
1549.
1550. I must be called with an object that implements ICheckerResults, as
1551. proof that you have actually discovered a problem with this file. I
1552. will use the data in the checker results to guide the repair process,
1553. such as which servers provided bad data and should therefore be
1554. avoided.
1555. """
1556.
1557. class IRepairResults(Interface):
1558. """I contain the results of a repair operation."""
1559.
1560.
1561. class IClient(Interface):
1562. def upload(uploadable):
1563. """Upload some data into a CHK, get back the UploadResults for it.
1564. @param uploadable: something that implements IUploadable
1565. @return: a Deferred that fires with the UploadResults instance.
1566. To get the URI for this file, use results.uri .
1567. """
1568.
1569. def create_mutable_file(contents=""):
1570. """Create a new mutable file with contents, get back the URI string.
1571. @param contents: the initial contents to place in the file.
1572. @return: a Deferred that fires with tne (string) SSK URI for the new
1573. file.
1574. """
1575.
1576. def create_empty_dirnode():
1577. """Create a new dirnode, empty and unattached.
1578. @return: a Deferred that fires with the new IDirectoryNode instance.
1579. """