source file: /home/buildslave/tahoe/edgy/build/src/allmydata/scripts/tahoe_check.py
file stats: 235 lines, 207 executed: 88.1% covered
coverage versus previous test: 0 lines added, 0 lines removed
1.
2. import urllib
3. import simplejson
4. from twisted.protocols.basic import LineOnlyReceiver
5. from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path
6. from allmydata.scripts.common_http import do_http
7.
8. class Checker:
9. pass
10.
11. def check(options):
12. stdout = options.stdout
13. stderr = options.stderr
14. nodeurl = options['node-url']
15. if not nodeurl.endswith("/"):
16. nodeurl += "/"
17. where = options.where
18. rootcap, path = get_alias(options.aliases, where, DEFAULT_ALIAS)
19. if path == '/':
20. path = ''
21. url = nodeurl + "uri/%s" % urllib.quote(rootcap)
22. if path:
23. url += "/" + escape_path(path)
24. # todo: should it end with a slash?
25. url += "?t=check&output=JSON"
26. if options["verify"]:
27. url += "&verify=true"
28. if options["repair"]:
29. url += "&repair=true"
30. if options["add-lease"]:
31. url += "&add-lease=true"
32.
33. resp = do_http("POST", url)
34. if resp.status != 200:
35. print >>stderr, "ERROR", resp.status, resp.reason, resp.read()
36. return 1
37. jdata = resp.read()
38. if options.get("raw"):
39. stdout.write(jdata)
40. stdout.write("\n")
41. return 0
42. data = simplejson.loads(jdata)
43.
44. if options["repair"]:
45. # show repair status
46. if data["pre-repair-results"]["results"]["healthy"]:
47. summary = "healthy"
48. else:
49. summary = "not healthy"
50. stdout.write("Summary: %s\n" % summary)
51. cr = data["pre-repair-results"]["results"]
52. stdout.write(" storage index: %s\n" % data["storage-index"])
53. stdout.write(" good-shares: %d (encoding is %d-of-%d)\n"
54. % (cr["count-shares-good"],
55. cr["count-shares-needed"],
56. cr["count-shares-expected"]))
57. stdout.write(" wrong-shares: %d\n" % cr["count-wrong-shares"])
58. corrupt = cr["list-corrupt-shares"]
59. if corrupt:
60. stdout.write(" corrupt shares:\n")
61. for (serverid, storage_index, sharenum) in corrupt:
62. stdout.write(" server %s, SI %s, shnum %d\n" %
63. (serverid, storage_index, sharenum))
64. if data["repair-attempted"]:
65. if data["repair-successful"]:
66. stdout.write(" repair successful\n")
67. else:
68. stdout.write(" repair failed\n")
69. else:
70. stdout.write("Summary: %s\n" % data["summary"])
71. cr = data["results"]
72. stdout.write(" storage index: %s\n" % data["storage-index"])
73. stdout.write(" good-shares: %d (encoding is %d-of-%d)\n"
74. % (cr["count-shares-good"],
75. cr["count-shares-needed"],
76. cr["count-shares-expected"]))
77. stdout.write(" wrong-shares: %d\n" % cr["count-wrong-shares"])
78. corrupt = cr["list-corrupt-shares"]
79. if corrupt:
80. stdout.write(" corrupt shares:\n")
81. for (serverid, storage_index, sharenum) in corrupt:
82. stdout.write(" server %s, SI %s, shnum %d\n" %
83. (serverid, storage_index, sharenum))
84. return 0
85.
86.
87. class FakeTransport:
88. disconnecting = False
89.
90. class DeepCheckOutput(LineOnlyReceiver):
91. delimiter = "\n"
92. def __init__(self, streamer, options):
93. self.streamer = streamer
94. self.transport = FakeTransport()
95.
96. self.verbose = bool(options["verbose"])
97. self.stdout = options.stdout
98. self.stderr = options.stderr
99. self.num_objects = 0
100. self.files_healthy = 0
101. self.files_unhealthy = 0
102. self.in_error = False
103.
104. def lineReceived(self, line):
105. if self.in_error:
106. print >>self.stderr, line
107. return
108. if line.startswith("ERROR:"):
109. self.in_error = True
110. self.streamer.rc = 1
111. print >>self.stderr, line
112. return
113.
114. d = simplejson.loads(line)
115. stdout = self.stdout
116. if d["type"] not in ("file", "directory"):
117. return
118. self.num_objects += 1
119. # non-verbose means print a progress marker every 100 files
120. if self.num_objects % 100 == 0:
121. print >>stdout, "%d objects checked.." % self.num_objects
122. cr = d["check-results"]
123. if cr["results"]["healthy"]:
124. self.files_healthy += 1
125. else:
126. self.files_unhealthy += 1
127. if self.verbose:
128. # verbose means also print one line per file
129. path = d["path"]
130. if not path:
131. path = ["<root>"]
132. summary = cr.get("summary", "Healthy (LIT)")
133. try:
134. print >>stdout, "%s: %s" % ("/".join(path), summary)
135. except UnicodeEncodeError:
136. print >>stdout, "%s: %s" % ("/".join([p.encode("utf-8")
137. for p in path]),
138. summary)
139. # always print out corrupt shares
140. for shareloc in cr["results"].get("list-corrupt-shares", []):
141. (serverid, storage_index, sharenum) = shareloc
142. print >>stdout, " corrupt: server %s, SI %s, shnum %d" % \
143. (serverid, storage_index, sharenum)
144.
145. def done(self):
146. if self.in_error:
147. return
148. stdout = self.stdout
149. print >>stdout, "done: %d objects checked, %d healthy, %d unhealthy" \
150. % (self.num_objects, self.files_healthy, self.files_unhealthy)
151.
152. class DeepCheckAndRepairOutput(LineOnlyReceiver):
153. delimiter = "\n"
154. def __init__(self, streamer, options):
155. self.streamer = streamer
156. self.transport = FakeTransport()
157.
158. self.verbose = bool(options["verbose"])
159. self.stdout = options.stdout
160. self.stderr = options.stderr
161. self.num_objects = 0
162. self.pre_repair_files_healthy = 0
163. self.pre_repair_files_unhealthy = 0
164. self.repairs_attempted = 0
165. self.repairs_successful = 0
166. self.post_repair_files_healthy = 0
167. self.post_repair_files_unhealthy = 0
168. self.in_error = False
169.
170. def lineReceived(self, line):
171. if self.in_error:
172. print >>self.stderr, line
173. return
174. if line.startswith("ERROR:"):
175. self.in_error = True
176. self.streamer.rc = 1
177. print >>self.stderr, line
178. return
179.
180. d = simplejson.loads(line)
181. stdout = self.stdout
182. if d["type"] not in ("file", "directory"):
183. return
184. self.num_objects += 1
185. # non-verbose means print a progress marker every 100 files
186. if self.num_objects % 100 == 0:
187. print >>stdout, "%d objects checked.." % self.num_objects
188. crr = d["check-and-repair-results"]
189. if d["storage-index"]:
190. if crr["pre-repair-results"]["results"]["healthy"]:
191. was_healthy = True
192. self.pre_repair_files_healthy += 1
193. else:
194. was_healthy = False
195. self.pre_repair_files_unhealthy += 1
196. if crr["post-repair-results"]["results"]["healthy"]:
197. self.post_repair_files_healthy += 1
198. else:
199. self.post_repair_files_unhealthy += 1
200. else:
201. # LIT file
202. was_healthy = True
203. self.pre_repair_files_healthy += 1
204. self.post_repair_files_healthy += 1
205. if crr["repair-attempted"]:
206. self.repairs_attempted += 1
207. if crr["repair-successful"]:
208. self.repairs_successful += 1
209. if self.verbose:
210. # verbose means also print one line per file
211. path = d["path"]
212. if not path:
213. path = ["<root>"]
214. # we don't seem to have a summary available, so build one
215. if was_healthy:
216. summary = "healthy"
217. else:
218. summary = "not healthy"
219. try:
220. print >>stdout, "%s: %s" % ("/".join(path), summary)
221. except UnicodeEncodeError:
222. print >>stdout, "%s: %s" % ("/".join([p.encode("utf-8")
223. for p in path]),
224. summary)
225. # always print out corrupt shares
226. prr = crr.get("pre-repair-results", {})
227. for shareloc in prr.get("results", {}).get("list-corrupt-shares", []):
228. (serverid, storage_index, sharenum) = shareloc
229. print >>stdout, " corrupt: server %s, SI %s, shnum %d" % \
230. (serverid, storage_index, sharenum)
231.
232. # always print out repairs
233. if crr["repair-attempted"]:
234. if crr["repair-successful"]:
235. print >>stdout, " repair successful"
236. else:
237. print >>stdout, " repair failed"
238.
239. def done(self):
240. if self.in_error:
241. return
242. stdout = self.stdout
243. print >>stdout, "done: %d objects checked" % self.num_objects
244. print >>stdout, " pre-repair: %d healthy, %d unhealthy" \
245. % (self.pre_repair_files_healthy,
246. self.pre_repair_files_unhealthy)
247. print >>stdout, " %d repairs attempted, %d successful, %d failed" \
248. % (self.repairs_attempted,
249. self.repairs_successful,
250. (self.repairs_attempted - self.repairs_successful))
251. print >>stdout, " post-repair: %d healthy, %d unhealthy" \
252. % (self.post_repair_files_healthy,
253. self.post_repair_files_unhealthy)
254.
255. class DeepCheckStreamer(LineOnlyReceiver):
256.
257. def run(self, options):
258. stdout = options.stdout
259. stderr = options.stderr
260. self.rc = 0
261. self.options = options
262. nodeurl = options['node-url']
263. if not nodeurl.endswith("/"):
264. nodeurl += "/"
265. self.nodeurl = nodeurl
266. where = options.where
267. rootcap, path = get_alias(options.aliases, where, DEFAULT_ALIAS)
268. if path == '/':
269. path = ''
270. url = nodeurl + "uri/%s" % urllib.quote(rootcap)
271. if path:
272. url += "/" + escape_path(path)
273. # todo: should it end with a slash?
274. url += "?t=stream-deep-check"
275. if options["verify"]:
276. url += "&verify=true"
277. if options["repair"]:
278. url += "&repair=true"
279. output = DeepCheckAndRepairOutput(self, options)
280. else:
281. output = DeepCheckOutput(self, options)
282. if options["add-lease"]:
283. url += "&add-lease=true"
284. resp = do_http("POST", url)
285. if resp.status not in (200, 302):
286. print >>stderr, "ERROR", resp.status, resp.reason, resp.read()
287. return 1
288.
289. # use Twisted to split this into lines
290. while True:
291. chunk = resp.read(100)
292. if not chunk:
293. break
294. if self.options["raw"]:
295. stdout.write(chunk)
296. else:
297. output.dataReceived(chunk)
298. if not self.options["raw"]:
299. output.done()
300. return self.rc
301.
302. def deepcheck(options):
303. return DeepCheckStreamer().run(options)