source file: /home/buildslave/tahoe/edgy/build/src/allmydata/codec.py
file stats: 95 lines, 91 executed: 95.8% covered
1. # -*- test-case-name: allmydata.test.test_encode_share -*-
2.
3. from zope.interface import implements
4. from twisted.internet import defer
5. from allmydata.util import mathutil
6. from allmydata.util.assertutil import precondition
7. from allmydata.interfaces import ICodecEncoder, ICodecDecoder
8. import zfec
9.
10.
11. class ReplicatingEncoder(object):
12. implements(ICodecEncoder)
13. ENCODER_TYPE = "rep"
14.
15. def set_params(self, data_size, required_shares, max_shares):
16. assert data_size % required_shares == 0
17. assert required_shares <= max_shares
18. self.data_size = data_size
19. self.required_shares = required_shares
20. self.max_shares = max_shares
21.
22. def get_encoder_type(self):
23. return self.ENCODER_TYPE
24.
25. def get_serialized_params(self):
26. return "%d" % self.required_shares
27.
28. def get_block_size(self):
29. return self.data_size
30.
31. def encode(self, inshares, desired_shareids=None):
32. assert isinstance(inshares, list)
33. for inshare in inshares:
34. assert isinstance(inshare, str)
35. assert self.required_shares * len(inshare) == self.data_size
36. data = "".join(inshares)
37. if desired_shareids is None:
38. desired_shareids = range(self.max_shares)
39. shares = [data for i in desired_shareids]
40. return defer.succeed((shares, desired_shareids))
41.
42. class ReplicatingDecoder(object):
43. implements(ICodecDecoder)
44.
45. def set_serialized_params(self, params):
46. self.required_shares = int(params)
47.
48. def get_needed_shares(self):
49. return self.required_shares
50.
51. def decode(self, some_shares, their_shareids):
52. assert len(some_shares) == self.required_shares
53. assert len(some_shares) == len(their_shareids)
54. data = some_shares[0]
55.
56. chunksize = mathutil.div_ceil(len(data), self.required_shares)
57. numchunks = mathutil.div_ceil(len(data), chunksize)
58. l = [ data[i:i+chunksize] for i in range(0, len(data), chunksize) ]
59.
60. return defer.succeed(l)
61.
62.
63. class CRSEncoder(object):
64. implements(ICodecEncoder)
65. ENCODER_TYPE = "crs"
66.
67. def set_params(self, data_size, required_shares, max_shares):
68. assert required_shares <= max_shares
69. self.data_size = data_size
70. self.required_shares = required_shares
71. self.max_shares = max_shares
72. self.share_size = mathutil.div_ceil(data_size, required_shares)
73. self.last_share_padding = mathutil.pad_size(self.share_size, required_shares)
74. self.encoder = zfec.Encoder(required_shares, max_shares)
75.
76. def get_encoder_type(self):
77. return self.ENCODER_TYPE
78.
79. def get_serialized_params(self):
80. return "%d-%d-%d" % (self.data_size, self.required_shares,
81. self.max_shares)
82.
83. def get_block_size(self):
84. return self.share_size
85.
86. def encode(self, inshares, desired_share_ids=None):
87. precondition(desired_share_ids is None or len(desired_share_ids) <= self.max_shares, desired_share_ids, self.max_shares)
88.
89. if desired_share_ids is None:
90. desired_share_ids = range(self.max_shares)
91.
92. for inshare in inshares:
93. assert len(inshare) == self.share_size, (len(inshare), self.share_size, self.data_size, self.required_shares)
94. shares = self.encoder.encode(inshares, desired_share_ids)
95.
96. return defer.succeed((shares, desired_share_ids))
97.
98. class CRSDecoder(object):
99. implements(ICodecDecoder)
100.
101. def set_serialized_params(self, params):
102. pieces = params.split("-")
103. self.data_size = int(pieces[0])
104. self.required_shares = int(pieces[1])
105. self.max_shares = int(pieces[2])
106.
107. self.chunk_size = self.required_shares
108. self.num_chunks = mathutil.div_ceil(self.data_size, self.chunk_size)
109. self.share_size = self.num_chunks
110. self.decoder = zfec.Decoder(self.required_shares, self.max_shares)
111.
112. def get_needed_shares(self):
113. return self.required_shares
114.
115. def decode(self, some_shares, their_shareids):
116. precondition(len(some_shares) == len(their_shareids),
117. len(some_shares), len(their_shareids))
118. precondition(len(some_shares) == self.required_shares,
119. len(some_shares), self.required_shares)
120. data = self.decoder.decode(some_shares,
121. [int(s) for s in their_shareids])
122. return defer.succeed(data)
123.
124.
125. all_encoders = {
126. ReplicatingEncoder.ENCODER_TYPE: (ReplicatingEncoder, ReplicatingDecoder),
127. CRSEncoder.ENCODER_TYPE: (CRSEncoder, CRSDecoder),
128. }
129.
130. def get_decoder_by_name(name):
131. decoder_class = all_encoders[name][1]
132. return decoder_class()
133.