Revision 798
Added by Borja Sotomayor almost 14 years ago
branches/1.1/src/haizea/cli/commands.py | ||
---|---|---|
25 | 25 |
from haizea.cli.optionparser import Option |
26 | 26 |
from haizea.cli import Command |
27 | 27 |
from haizea.lwf.generators import LWFGenerator, LWFAnnotationGenerator |
28 |
from haizea.lwf.analysis import LWFAnalyser |
|
28 | 29 |
from mx.DateTime import TimeDelta, Parser |
29 | 30 |
import haizea.common.defaults as defaults |
30 | 31 |
import sys |
... | ... | |
367 | 368 |
self.optparser.add_option(Option("-c", "--conf", action="store", type="string", dest="conf", |
368 | 369 |
help = """ |
369 | 370 |
... |
370 |
""")) |
|
371 |
"""))
|
|
371 | 372 |
|
372 | 373 |
def run(self): |
373 | 374 |
self.parse_options() |
... | ... | |
379 | 380 |
|
380 | 381 |
generator.generate() |
381 | 382 |
|
383 |
class haizea_lwf_stats(Command): |
|
384 |
|
|
385 |
name = "haizea-lwf-stats" |
|
386 |
|
|
387 |
def __init__(self, argv): |
|
388 |
Command.__init__(self, argv) |
|
389 |
self.optparser.add_option(Option("-i", "--in", action="store", type="string", dest="inf", required=True, |
|
390 |
help = """ |
|
391 |
Input file |
|
392 |
""")) |
|
393 |
self.optparser.add_option(Option("-l", "--utilization-length", action="store", type="string", dest="utilization_length", |
|
394 |
help = """ |
|
395 |
Length of the utilization interval in format DD:HH:MM:SS. Default is until |
|
396 |
the time the last lease is requested. |
|
397 |
""")) |
|
398 |
|
|
399 |
def run(self): |
|
400 |
self.parse_options() |
|
401 |
|
|
402 |
infile = self.opt.inf |
|
403 |
utilization_length = self.opt.utilization_length |
|
404 |
if utilization_length != None: |
|
405 |
utilization_length = Parser.DateTimeDeltaFromString(utilization_length) |
|
406 |
|
|
407 |
analyser = LWFAnalyser(infile, utilization_length) |
|
408 |
|
|
409 |
analyser.analyse() |
|
410 |
|
|
382 | 411 |
class haizea_lwf_annotate(Command): |
383 | 412 |
|
384 | 413 |
name = "haizea-lwf-annotate" |
branches/1.1/src/haizea/common/stats.py | ||
---|---|---|
137 | 137 |
DiscreteDistribution.__init__(self, values) |
138 | 138 |
self.__distribution = UniformDistribution(0,1) |
139 | 139 |
|
140 |
def seed(self, x): |
|
141 |
self.__distribution.seed(x) |
|
142 |
|
|
140 | 143 |
def get(self): |
141 | 144 |
return self._get_from_prob(self.__distribution.get()) |
142 | 145 |
|
branches/1.1/src/haizea/core/frontends/tracefile.py | ||
---|---|---|
40 | 40 |
# Requests is a list of lease requests |
41 | 41 |
self.logger.info("Loading tracefile %s" % tracefile) |
42 | 42 |
self.requests = None |
43 |
if tracefile.endswith(".swf"): |
|
44 |
self.requests = LeaseWorkload.from_swf_file(tracefile, self.starttime) |
|
45 |
elif tracefile.endswith(".lwf") or tracefile.endswith(".xml"): |
|
46 |
lease_workload = LeaseWorkload.from_xml_file(tracefile, self.starttime) |
|
47 |
self.requests = lease_workload.get_leases() |
|
43 |
lease_workload = LeaseWorkload.from_xml_file(tracefile, self.starttime) |
|
44 |
self.requests = lease_workload.get_leases() |
|
48 | 45 |
|
49 | 46 |
if injectfile != None: |
50 | 47 |
self.logger.info("Loading injection file %s" % injectfile) |
branches/1.1/src/haizea/core/leases.py | ||
---|---|---|
1573 | 1573 |
Argument: |
1574 | 1574 |
lwf_file -- LWF file. |
1575 | 1575 |
""" |
1576 |
return cls.__from_xml_element(ET.parse(lwf_file).getroot().find("site")) |
|
1576 |
site_elem = ET.parse(lwf_file).getroot().find("site") |
|
1577 |
if site_elem == None: |
|
1578 |
return None # LWF file does not contain a <site> element |
|
1579 |
else: |
|
1580 |
return cls.__from_xml_element(site_elem) |
|
1577 | 1581 |
|
1578 | 1582 |
@classmethod |
1579 | 1583 |
def __from_xml_element(cls, element): |
... | ... | |
1620 | 1624 |
|
1621 | 1625 |
resource_str = resource_str.split() |
1622 | 1626 |
numnodes = int(resource_str[0]) |
1623 |
resources = resource_str[1:]
|
|
1627 |
resources = resource_str[1] |
|
1624 | 1628 |
capacity = Capacity.from_resources_string(resources) |
1625 | 1629 |
|
1626 | 1630 |
nodes = Nodes([(numnodes,capacity)]) |
... | ... | |
1659 | 1663 |
|
1660 | 1664 |
return max_ninstances |
1661 | 1665 |
|
1666 |
def to_xml(self): |
|
1667 |
"""Returns an ElementTree XML representation of the nodes |
|
1668 |
|
|
1669 |
See the Haizea documentation for details on the |
|
1670 |
lease XML format. |
|
1671 |
|
|
1672 |
""" |
|
1673 |
site = ET.Element("site") |
|
1674 |
resource_types = ET.SubElement(site, "resource-types") |
|
1675 |
resource_types.set("names", " ".join(self.resource_types)) |
|
1676 |
site.append(self.nodes.to_xml()) |
|
1677 |
|
|
1678 |
return site |
|
1662 | 1679 |
|
1663 | 1680 |
|
1664 | 1681 |
class Nodes(object): |
branches/1.1/src/haizea/lwf/generators.py | ||
---|---|---|
19 | 19 |
import haizea.common.stats as stats |
20 | 20 |
from haizea.core.leases import LeaseWorkload, LeaseAnnotation, LeaseAnnotations, Timestamp, Lease,\ |
21 | 21 |
Capacity, Duration, UnmanagedSoftwareEnvironment,\ |
22 |
DiskImageSoftwareEnvironment |
|
22 |
DiskImageSoftwareEnvironment, Site
|
|
23 | 23 |
from haizea.common.utils import round_datetime_delta |
24 | 24 |
from mx.DateTime import DateTimeDelta, TimeDelta, Parser |
25 | 25 |
import ConfigParser |
... | ... | |
235 | 235 |
|
236 | 236 |
class LWFGenerator(FileGenerator): |
237 | 237 |
|
238 |
SITE_OPT = "site" |
|
239 |
|
|
238 | 240 |
NUMLEASES_SEC = "numleases" |
239 | 241 |
|
240 | 242 |
NUMLEASES_TYPE_UTILIZATION = "utilization" |
... | ... | |
249 | 251 |
|
250 | 252 |
self.numleases_type = self.config.get(LWFGenerator.NUMLEASES_SEC, LWFGenerator.TYPE_OPT) |
251 | 253 |
|
252 |
if self.numleases_type == LWFGenerator.NUMLEASES_TYPE_INTERVAL: |
|
253 |
self.interval_dist = self._get_dist(LWFGenerator.NUMLEASES_SEC) |
|
254 |
else: |
|
255 |
self.interval_dist = None |
|
256 |
|
|
254 |
self.interval_dist = self._get_dist(LWFGenerator.NUMLEASES_SEC) |
|
255 |
|
|
257 | 256 |
def _get_interval(self): |
258 | 257 |
if self.interval_dist == None: |
259 | 258 |
return None |
... | ... | |
290 | 289 |
description = ET.SubElement(lwf, "description") |
291 | 290 |
description.text = "Created with haizea-generate" |
292 | 291 |
|
293 |
#if self.opt.site != None: |
|
294 |
# site_elem = ET.parse(self.opt.site).getroot() |
|
295 |
# site_num_nodes = int(site_elem.find("nodes").find("node-set").get("numnodes")) |
|
296 |
# lwf.append(site_elem) |
|
292 |
site = self.config.get(LWFGenerator.GENERAL_SEC, LWFGenerator.SITE_OPT) |
|
293 |
if site.startswith("file:"): |
|
294 |
sitefile = site.split(":") |
|
295 |
site = Site.from_xml_file(sitefile[1]) |
|
296 |
else: |
|
297 |
site = Site.from_resources_string(site) |
|
298 |
|
|
299 |
lwf.append(site.to_xml()) |
|
297 | 300 |
|
298 | 301 |
time = TimeDelta(seconds=0) |
299 |
requests = ET.SubElement(lwf, "lease-requests")
|
|
302 |
requests = ET.SubElement(lwf, "lease-requests") |
|
300 | 303 |
|
301 | 304 |
if self.numleases_type == LWFGenerator.NUMLEASES_TYPE_INTERVAL: |
302 | 305 |
leases = [] |
... | ... | |
308 | 311 |
for l in leases: |
309 | 312 |
interval = TimeDelta(seconds=self._get_interval()) |
310 | 313 |
time += interval |
311 |
print interval, time |
|
312 | 314 |
l.start.requested += time |
313 | 315 |
lease_request = ET.SubElement(requests, "lease-request") |
314 | 316 |
lease_request.set("arrival", str(time)) |
... | ... | |
318 | 320 |
last_request = self.config.get(LWFGenerator.NUMLEASES_SEC, LWFGenerator.NUMLEASES_LAST_REQUEST_OPT) |
319 | 321 |
last_request = Parser.DateTimeDeltaFromString(last_request) |
320 | 322 |
|
321 |
# TODO |
|
322 |
target_utilization = utilization |
|
323 |
accum_utilization = None |
|
323 |
max_utilization = 0 |
|
324 |
for res in site.nodes.get_all_nodes().values(): |
|
325 |
for i in range(1,res.get_ninstances("CPU") + 1): |
|
326 |
max_utilization += res.get_quantity_instance("CPU", i) * last_request.seconds |
|
327 |
target_utilization = int(max_utilization * utilization) |
|
324 | 328 |
|
329 |
accum_utilization = 0 |
|
330 |
|
|
325 | 331 |
leases = [] |
326 | 332 |
|
327 | 333 |
while accum_utilization < target_utilization: |
328 |
pass |
|
329 |
self.__gen_lease() |
|
334 |
lease = self.__gen_lease() |
|
335 |
leases.append(lease) |
|
336 |
duration = lease.duration.requested.seconds |
|
337 |
lease_utilization = 0 |
|
338 |
for res in lease.requested_resources.values(): |
|
339 |
for i in range(1,res.get_ninstances("CPU") + 1): |
|
340 |
lease_utilization += res.get_quantity_instance("CPU", i) * duration |
|
341 |
accum_utilization += lease_utilization |
|
330 | 342 |
|
331 |
# TODO: Set arrival times so they are evenly spaced |
|
343 |
time = TimeDelta(seconds=0) |
|
344 |
avg_interval = int(last_request.seconds / len(leases)) |
|
332 | 345 |
for l in leases: |
346 |
interval = avg_interval + TimeDelta(seconds=self._get_interval()) |
|
347 |
time = max(time + interval, TimeDelta(seconds=0)) |
|
348 |
l.start.requested += time |
|
333 | 349 |
lease_request = ET.SubElement(requests, "lease-request") |
334 | 350 |
lease_request.set("arrival", str(time)) |
335 |
lease_request.append(l.to_xml()) |
|
351 |
lease_request.append(l.to_xml())
|
|
336 | 352 |
|
337 |
|
|
338 |
print ET.tostring(lwf) |
|
353 |
tree = ET.ElementTree(lwf) |
|
354 |
|
|
355 |
outfile = open(self.outfile, "w") |
|
356 |
tree.write(outfile) |
|
357 |
outfile.close() |
|
339 | 358 |
|
340 | 359 |
class LWFAnnotationGenerator(FileGenerator): |
341 | 360 |
|
branches/1.1/src/haizea/lwf/analysis.py | ||
---|---|---|
14 | 14 |
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # |
15 | 15 |
# See the License for the specific language governing permissions and # |
16 | 16 |
# limitations under the License. # |
17 |
# -------------------------------------------------------------------------- # |
|
17 |
# -------------------------------------------------------------------------- # |
|
18 |
from mx.DateTime import DateTime |
|
19 |
from haizea.core.leases import LeaseWorkload, Site |
|
20 |
|
|
21 |
|
|
22 |
class LWFAnalyser(object): |
|
23 |
|
|
24 |
|
|
25 |
def __init__(self, lwffile, utilization_length): |
|
26 |
# Arbitrary start time |
|
27 |
self.starttime = DateTime(2006,11,25,13) |
|
28 |
|
|
29 |
self.workload = LeaseWorkload.from_xml_file(lwffile, self.starttime) |
|
30 |
self.site = Site.from_lwf_file(lwffile) |
|
31 |
|
|
32 |
if utilization_length == None: |
|
33 |
self.utilization_length = self.workload.get_leases()[-1].submit_time - self.starttime |
|
34 |
else: |
|
35 |
self.utilization_length = utilization_length |
|
36 |
print self.utilization_length |
|
37 |
|
|
38 |
def analyse(self): |
|
39 |
utilization = 0 |
|
40 |
for lease in self.workload.get_leases(): |
|
41 |
if lease.start.requested + lease.duration.requested > self.starttime + self.utilization_length: |
|
42 |
duration = self.starttime + self.utilization_length - lease.start.requested |
|
43 |
else: |
|
44 |
duration = lease.duration.requested.seconds |
|
45 |
for res in lease.requested_resources.values(): |
|
46 |
for i in range(1,res.get_ninstances("CPU") + 1): |
|
47 |
utilization += res.get_quantity_instance("CPU", i) * duration |
|
48 |
|
|
49 |
if self.site != None: |
|
50 |
max_utilization = 0 |
|
51 |
duration = self.utilization_length.seconds |
|
52 |
for res in self.site.nodes.get_all_nodes().values(): |
|
53 |
for i in range(1,res.get_ninstances("CPU") + 1): |
|
54 |
max_utilization += res.get_quantity_instance("CPU", i) * duration |
|
55 |
|
|
56 |
|
|
57 |
print "Utilization: %.2f%%" % (utilization / max_utilization) |
|
58 |
|
branches/1.1/bin/haizea-lwf-generate | ||
---|---|---|
3 | 3 |
from haizea.cli import commands |
4 | 4 |
import sys |
5 | 5 |
|
6 |
c = commands.haizea_lwf_annotate(sys.argv)
|
|
6 |
c = commands.haizea_lwf_generate(sys.argv)
|
|
7 | 7 |
c.run() |
branches/1.1/bin/haizea-lwf-stats | ||
---|---|---|
1 |
#!/usr/bin/python |
|
2 |
|
|
3 |
from haizea.cli import commands |
|
4 |
import sys |
|
5 |
|
|
6 |
c = commands.haizea_lwf_stats(sys.argv) |
|
7 |
c.run() |
|
0 | 8 |
Also available in: Unified diff
Utilization-based LWF generation (still needs some work, but mostly done)