source file: /home/buildslave/tahoe/edgy/build/src/allmydata/web/storage.py
file stats: 160 lines, 158 executed: 98.8% covered
coverage versus previous test: 0 lines added, 0 lines removed
1.
2. import time, simplejson
3. from nevow import rend, tags as T, inevow
4. from allmydata.web.common import getxmlfile, abbreviate_time, get_arg
5. from allmydata.util.abbreviate import abbreviate_space
6. from allmydata.util import time_format
7.
8. def remove_prefix(s, prefix):
9. if not s.startswith(prefix):
10. return None
11. return s[len(prefix):]
12.
13. class StorageStatus(rend.Page):
14. docFactory = getxmlfile("storage_status.xhtml")
15. # the default 'data' argument is the StorageServer instance
16.
17. def __init__(self, storage):
18. rend.Page.__init__(self, storage)
19. self.storage = storage
20.
21. def renderHTTP(self, ctx):
22. req = inevow.IRequest(ctx)
23. t = get_arg(req, "t")
24. if t == "json":
25. return self.render_JSON(req)
26. return rend.Page.renderHTTP(self, ctx)
27.
28. def render_JSON(self, req):
29. req.setHeader("content-type", "text/plain")
30. d = {"stats": self.storage.get_stats(),
31. "bucket-counter": self.storage.bucket_counter.get_state(),
32. "lease-checker": self.storage.lease_checker.get_state(),
33. "lease-checker-progress": self.storage.lease_checker.get_progress(),
34. }
35. return simplejson.dumps(d, indent=1) + "\n"
36.
37. def render_storage_running(self, ctx, storage):
38. if storage:
39. return ctx.tag
40. else:
41. return T.h1["No Storage Server Running"]
42.
43. def render_bool(self, ctx, data):
44. return {True: "Yes", False: "No"}[bool(data)]
45.
46. def render_abbrev_space(self, ctx, size):
47. if size is None:
48. return "?"
49. return abbreviate_space(size)
50.
51. def render_space(self, ctx, size):
52. if size is None:
53. return "?"
54. return "%d" % size
55.
56. def data_stats(self, ctx, data):
57. # FYI: 'data' appears to be self, rather than the StorageServer
58. # object in self.original that gets passed to render_* methods. I
59. # still don't understand Nevow.
60.
61. # Nevow has nevow.accessors.DictionaryContainer: Any data= directive
62. # that appears in a context in which the current data is a dictionary
63. # will be looked up as keys in that dictionary. So if data_stats()
64. # returns a dictionary, then we can use something like this:
65. #
66. # <ul n:data="stats">
67. # <li>disk_total: <span n:render="abbrev" n:data="disk_total" /></li>
68. # </ul>
69.
70. # to use get_stats()["storage_server.disk_total"] . However,
71. # DictionaryContainer does a raw d[] instead of d.get(), so any
72. # missing keys will cause an error, even if the renderer can tolerate
73. # None values. To overcome this, we either need a dict-like object
74. # that always returns None for unknown keys, or we must pre-populate
75. # our dict with those missing keys, or we should get rid of data_
76. # methods that return dicts (or find some way to override Nevow's
77. # handling of dictionaries).
78.
79. d = dict([ (remove_prefix(k, "storage_server."), v)
80. for k,v in self.storage.get_stats().items() ])
81. d.setdefault("disk_total", None)
82. d.setdefault("disk_used", None)
83. d.setdefault("disk_free_for_root", None)
84. d.setdefault("disk_free_for_nonroot", None)
85. d.setdefault("reserved_space", None)
86. d.setdefault("disk_avail", None)
87. return d
88.
89. def data_last_complete_bucket_count(self, ctx, data):
90. s = self.storage.bucket_counter.get_state()
91. count = s.get("last-complete-bucket-count")
92. if count is None:
93. return "Not computed yet"
94. return count
95.
96. def render_count_crawler_status(self, ctx, storage):
97. p = self.storage.bucket_counter.get_progress()
98. return ctx.tag[self.format_crawler_progress(p)]
99.
100. def format_crawler_progress(self, p):
101. cycletime = p["estimated-time-per-cycle"]
102. cycletime_s = ""
103. if cycletime is not None:
104. cycletime_s = " (estimated cycle time %s)" % abbreviate_time(cycletime)
105.
106. if p["cycle-in-progress"]:
107. pct = p["cycle-complete-percentage"]
108. soon = p["remaining-sleep-time"]
109.
110. eta = p["estimated-cycle-complete-time-left"]
111. eta_s = ""
112. if eta is not None:
113. eta_s = " (ETA %ds)" % eta
114.
115. return ["Current crawl %.1f%% complete" % pct,
116. eta_s,
117. " (next work in %s)" % abbreviate_time(soon),
118. cycletime_s,
119. ]
120. else:
121. soon = p["remaining-wait-time"]
122. return ["Next crawl in %s" % abbreviate_time(soon),
123. cycletime_s]
124.
125. def render_lease_expiration_enabled(self, ctx, data):
126. lc = self.storage.lease_checker
127. if lc.expiration_enabled:
128. return ctx.tag["Enabled: expired leases will be removed"]
129. else:
130. return ctx.tag["Disabled: scan-only mode, no leases will be removed"]
131.
132. def render_lease_expiration_mode(self, ctx, data):
133. lc = self.storage.lease_checker
134. if lc.mode == "age":
135. if lc.override_lease_duration is None:
136. ctx.tag["Leases will expire naturally, probably 31 days after "
137. "creation or renewal."]
138. else:
139. ctx.tag["Leases created or last renewed more than %s ago "
140. "will be considered expired."
141. % abbreviate_time(lc.override_lease_duration)]
142. else:
143. assert lc.mode == "cutoff-date"
144. localizedutcdate = time.strftime("%d-%b-%Y", time.gmtime(lc.cutoff_date))
145. isoutcdate = time_format.iso_utc_date(lc.cutoff_date)
146. ctx.tag["Leases created or last renewed before %s (%s) UTC "
147. "will be considered expired." % (isoutcdate, localizedutcdate, )]
148. if len(lc.mode) > 2:
149. ctx.tag[" The following sharetypes will be expired: ",
150. " ".join(sorted(lc.sharetypes_to_expire)), "."]
151. return ctx.tag
152.
153. def format_recovered(self, sr, a):
154. def maybe(d):
155. if d is None:
156. return "?"
157. return "%d" % d
158. return "%s shares, %s buckets (%s mutable / %s immutable), %s (%s / %s)" % \
159. (maybe(sr["%s-shares" % a]),
160. maybe(sr["%s-buckets" % a]),
161. maybe(sr["%s-buckets-mutable" % a]),
162. maybe(sr["%s-buckets-immutable" % a]),
163. abbreviate_space(sr["%s-diskbytes" % a]),
164. abbreviate_space(sr["%s-diskbytes-mutable" % a]),
165. abbreviate_space(sr["%s-diskbytes-immutable" % a]),
166. )
167.
168. def render_lease_current_cycle_progress(self, ctx, data):
169. lc = self.storage.lease_checker
170. p = lc.get_progress()
171. return ctx.tag[self.format_crawler_progress(p)]
172.
173. def render_lease_current_cycle_results(self, ctx, data):
174. lc = self.storage.lease_checker
175. p = lc.get_progress()
176. if not p["cycle-in-progress"]:
177. return ""
178. s = lc.get_state()
179. so_far = s["cycle-to-date"]
180. sr = so_far["space-recovered"]
181. er = s["estimated-remaining-cycle"]
182. esr = er["space-recovered"]
183. ec = s["estimated-current-cycle"]
184. ecr = ec["space-recovered"]
185.
186. p = T.ul()
187. pieces = []
188. def add(*pieces):
189. p[T.li[pieces]]
190.
191. def maybe(d):
192. if d is None:
193. return "?"
194. return "%d" % d
195. add("So far, this cycle has examined %d shares in %d buckets"
196. % (sr["examined-shares"], sr["examined-buckets"]),
197. " (%d mutable / %d immutable)"
198. % (sr["examined-buckets-mutable"], sr["examined-buckets-immutable"]),
199. " (%s / %s)" % (abbreviate_space(sr["examined-diskbytes-mutable"]),
200. abbreviate_space(sr["examined-diskbytes-immutable"])),
201. )
202. add("and has recovered: ", self.format_recovered(sr, "actual"))
203. if so_far["expiration-enabled"]:
204. add("The remainder of this cycle is expected to recover: ",
205. self.format_recovered(esr, "actual"))
206. add("The whole cycle is expected to examine %s shares in %s buckets"
207. % (maybe(ecr["examined-shares"]), maybe(ecr["examined-buckets"])))
208. add("and to recover: ", self.format_recovered(ecr, "actual"))
209.
210. else:
211. add("If expiration were enabled, we would have recovered: ",
212. self.format_recovered(sr, "configured"), " by now")
213. add("and the remainder of this cycle would probably recover: ",
214. self.format_recovered(esr, "configured"))
215. add("and the whole cycle would probably recover: ",
216. self.format_recovered(ecr, "configured"))
217.
218. add("if we were strictly using each lease's default 31-day lease lifetime "
219. "(instead of our configured behavior), "
220. "this cycle would be expected to recover: ",
221. self.format_recovered(ecr, "original"))
222.
223. if so_far["corrupt-shares"]:
224. add("Corrupt shares:",
225. T.ul[ [T.li[ ["SI %s shnum %d" % corrupt_share
226. for corrupt_share in so_far["corrupt-shares"] ]
227. ]]])
228.
229. return ctx.tag["Current cycle:", p]
230.
231. def render_lease_last_cycle_results(self, ctx, data):
232. lc = self.storage.lease_checker
233. h = lc.get_state()["history"]
234. if not h:
235. return ""
236. last = h[max(h.keys())]
237.
238. start, end = last["cycle-start-finish-times"]
239. ctx.tag["Last complete cycle (which took %s and finished %s ago)"
240. " recovered: " % (abbreviate_time(end-start),
241. abbreviate_time(time.time() - end)),
242. self.format_recovered(last["space-recovered"], "actual")
243. ]
244.
245. p = T.ul()
246. pieces = []
247. def add(*pieces):
248. p[T.li[pieces]]
249.
250. if not last["expiration-enabled"]:
251. rec = self.format_recovered(last["space-recovered"], "configured")
252. add("but expiration was not enabled. If it had been, "
253. "it would have recovered: ", rec)
254.
255. if last["corrupt-shares"]:
256. add("Corrupt shares:",
257. T.ul[ [T.li[ ["SI %s shnum %d" % corrupt_share
258. for corrupt_share in last["corrupt-shares"] ]
259. ]]])
260.
261. return ctx.tag[p]
262.