source file: /home/buildslave/tahoe/edgy/build/src/allmydata/scripts/tahoe_manifest.py
file stats: 106 lines, 99 executed: 93.4% covered
coverage versus previous test: 0 lines added, 0 lines removed
1.
2. import urllib, simplejson
3. from twisted.protocols.basic import LineOnlyReceiver
4. from allmydata.util.abbreviate import abbreviate_space_both
5. from allmydata.scripts.slow_operation import SlowOperationRunner
6. from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path
7. from allmydata.scripts.common_http import do_http
8.
9. class FakeTransport:
10. disconnecting = False
11.
12. class ManifestStreamer(LineOnlyReceiver):
13. delimiter = "\n"
14.
15. def __init__(self):
16. self.transport = FakeTransport()
17.
18. def run(self, options):
19. self.rc = 0
20. stdout = options.stdout
21. stderr = options.stderr
22. self.options = options
23. nodeurl = options['node-url']
24. if not nodeurl.endswith("/"):
25. nodeurl += "/"
26. self.nodeurl = nodeurl
27. where = options.where
28. rootcap, path = get_alias(options.aliases, where, DEFAULT_ALIAS)
29. if path == '/':
30. path = ''
31. url = nodeurl + "uri/%s" % urllib.quote(rootcap)
32. if path:
33. url += "/" + escape_path(path)
34. # todo: should it end with a slash?
35. url += "?t=stream-manifest"
36. resp = do_http("POST", url)
37. if resp.status not in (200, 302):
38. print >>stderr, "ERROR", resp.status, resp.reason, resp.read()
39. return 1
40. #print "RESP", dir(resp)
41. # use Twisted to split this into lines
42. self.in_error = False
43. while True:
44. chunk = resp.read(100)
45. if not chunk:
46. break
47. if self.options["raw"]:
48. stdout.write(chunk)
49. else:
50. self.dataReceived(chunk)
51. return self.rc
52.
53. def lineReceived(self, line):
54. stdout = self.options.stdout
55. stderr = self.options.stderr
56. if self.in_error:
57. print >>stderr, line
58. return
59. if line.startswith("ERROR:"):
60. self.in_error = True
61. self.rc = 1
62. print >>stderr, line
63. return
64.
65. d = simplejson.loads(line)
66. if d["type"] in ("file", "directory"):
67. if self.options["storage-index"]:
68. si = d["storage-index"]
69. if si:
70. print >>stdout, si
71. elif self.options["verify-cap"]:
72. vc = d["verifycap"]
73. if vc:
74. print >>stdout, vc
75. elif self.options["repair-cap"]:
76. vc = d["repaircap"]
77. if vc:
78. print >>stdout, vc
79. else:
80. try:
81. print >>stdout, d["cap"], "/".join(d["path"])
82. except UnicodeEncodeError:
83. print >>stdout, d["cap"], "/".join([p.encode("utf-8")
84. for p in d["path"]])
85.
86. def manifest(options):
87. return ManifestStreamer().run(options)
88.
89. class StatsGrabber(SlowOperationRunner):
90.
91. def make_url(self, base, ophandle):
92. return base + "?t=start-deep-stats&ophandle=" + ophandle
93.
94. def write_results(self, data):
95. stdout = self.options.stdout
96. stderr = self.options.stderr
97. keys = ("count-immutable-files",
98. "count-mutable-files",
99. "count-literal-files",
100. "count-files",
101. "count-directories",
102. "size-immutable-files",
103. "size-mutable-files",
104. "size-literal-files",
105. "size-directories",
106. "largest-directory",
107. "largest-immutable-file",
108. )
109. width = max([len(k) for k in keys])
110. print >>stdout, "Counts and Total Sizes:"
111. for k in keys:
112. fmt = "%" + str(width) + "s: %d"
113. if k in data:
114. value = data[k]
115. if not k.startswith("count-") and value > 1000:
116. absize = abbreviate_space_both(value)
117. print >>stdout, fmt % (k, data[k]), " ", absize
118. else:
119. print >>stdout, fmt % (k, data[k])
120. if data["size-files-histogram"]:
121. print >>stdout, "Size Histogram:"
122. prevmax = None
123. maxlen = max([len(str(maxsize))
124. for (minsize, maxsize, count)
125. in data["size-files-histogram"]])
126. maxcountlen = max([len(str(count))
127. for (minsize, maxsize, count)
128. in data["size-files-histogram"]])
129. minfmt = "%" + str(maxlen) + "d"
130. maxfmt = "%-" + str(maxlen) + "d"
131. countfmt = "%-" + str(maxcountlen) + "d"
132. linefmt = minfmt + "-" + maxfmt + " : " + countfmt + " %s"
133. for (minsize, maxsize, count) in data["size-files-histogram"]:
134. if prevmax is not None and minsize != prevmax+1:
135. print >>stdout, " "*(maxlen-1) + "..."
136. prevmax = maxsize
137. print >>stdout, linefmt % (minsize, maxsize, count,
138. abbreviate_space_both(maxsize))
139.
140. def stats(options):
141. return StatsGrabber().run(options)