1
|
|
2
|
|
3
|
|
4
|
|
5
|
|
6
|
|
7
|
|
8
|
|
9
|
|
10
|
|
11
|
|
12
|
|
13
|
|
14
|
|
15
|
|
16
|
|
17
|
|
18
|
|
19
|
import haizea.common.stats as stats
|
20
|
from haizea.core.leases import LeaseWorkload, LeaseAnnotation, LeaseAnnotations, Timestamp, Lease,\
|
21
|
Capacity, Duration, UnmanagedSoftwareEnvironment,\
|
22
|
DiskImageSoftwareEnvironment, Site
|
23
|
from haizea.common.utils import round_datetime_delta
|
24
|
from mx.DateTime import DateTimeDelta, TimeDelta, Parser
|
25
|
import ConfigParser
|
26
|
|
27
|
try:
|
28
|
import xml.etree.ElementTree as ET
|
29
|
except ImportError:
|
30
|
|
31
|
import elementtree.ElementTree as ET
|
32
|
|
33
|
class FileGenerator(object):
|
34
|
|
35
|
GENERAL_SEC = "general"
|
36
|
|
37
|
TYPE_OPT = "type"
|
38
|
DISTRIBUTION_OPT = "distribution"
|
39
|
DISCRETE_OPT = "discrete"
|
40
|
MIN_OPT = "min"
|
41
|
MAX_OPT = "max"
|
42
|
VALUES_OPT = "values"
|
43
|
MEAN_OPT = "mu"
|
44
|
STDEV_OPT = "sigma"
|
45
|
ALPHA_OPT = "alpha"
|
46
|
SCALE_OPT = "scale"
|
47
|
INVERT_OPT = "invert"
|
48
|
SEED_OPT = "seed"
|
49
|
|
50
|
START_DELAY_SEC = "start-delay"
|
51
|
START_ABSOLUTE = "absolute"
|
52
|
START_DURATION = "multiple-of-duration"
|
53
|
|
54
|
DEADLINE_STRETCH_SEC = "deadline-stretch"
|
55
|
DEADLINE_DURATION = "multiple-of-duration"
|
56
|
DEADLINE_SLOWDOWN = "original-slowdown"
|
57
|
DEADLINE_ABSOLUTE = "absolute"
|
58
|
|
59
|
RATE_SEC = "user-rate"
|
60
|
|
61
|
NODES_SEC = "nodes"
|
62
|
RESOURCES_OPT = "resources"
|
63
|
|
64
|
DURATION_SEC = "duration"
|
65
|
|
66
|
SOFTWARE_SEC = "software"
|
67
|
|
68
|
|
69
|
def __init__(self, outfile, conffile):
|
70
|
self.outfile = outfile
|
71
|
self.conffile = conffile
|
72
|
|
73
|
conffile = open(conffile, "r")
|
74
|
self.config = ConfigParser.ConfigParser()
|
75
|
self.config.readfp(conffile)
|
76
|
|
77
|
self.startdelay_dist = self._get_dist(FileGenerator.START_DELAY_SEC)
|
78
|
self.deadlinestretch_dist = self._get_dist(FileGenerator.DEADLINE_STRETCH_SEC)
|
79
|
self.rate_dist = self._get_dist(FileGenerator.RATE_SEC)
|
80
|
self.duration_dist = self._get_dist(FileGenerator.DURATION_SEC)
|
81
|
self.numnodes_dist = self._get_dist(FileGenerator.NODES_SEC)
|
82
|
self.software_dist = self._get_dist(FileGenerator.SOFTWARE_SEC)
|
83
|
|
84
|
self.start_type = self.config.get(FileGenerator.START_DELAY_SEC, FileGenerator.TYPE_OPT)
|
85
|
self.deadline_type = self.config.get(FileGenerator.DEADLINE_STRETCH_SEC, FileGenerator.TYPE_OPT)
|
86
|
|
87
|
|
88
|
def _get_start(self, type, lease = None):
|
89
|
if self.startdelay_dist == None:
|
90
|
return None, None
|
91
|
else:
|
92
|
delta = self.startdelay_dist.get()
|
93
|
if type == FileGenerator.START_ABSOLUTE:
|
94
|
start = round_datetime_delta(TimeDelta(seconds=delta))
|
95
|
elif type == FileGenerator.START_DURATION:
|
96
|
start = round_datetime_delta(delta * lease.duration.requested)
|
97
|
return start, delta
|
98
|
|
99
|
def _get_numnodes(self, lease = None):
|
100
|
if self.numnodes_dist == None:
|
101
|
return None
|
102
|
else:
|
103
|
numnodes = int(self.numnodes_dist.get())
|
104
|
return numnodes
|
105
|
|
106
|
def _get_duration(self, lease = None):
|
107
|
if self.duration_dist == None:
|
108
|
return None
|
109
|
else:
|
110
|
numnodes = int(self.duration_dist.get())
|
111
|
return numnodes
|
112
|
|
113
|
def _get_deadline(self, type, lease, start):
|
114
|
if self.deadlinestretch_dist == None:
|
115
|
return None, None
|
116
|
else:
|
117
|
if type in (FileGenerator.DEADLINE_DURATION, FileGenerator.DEADLINE_SLOWDOWN):
|
118
|
if type == FileGenerator.DEADLINE_DURATION:
|
119
|
tau = self.deadlinestretch_dist.get()
|
120
|
|
121
|
elif type == FileGenerator.DEADLINE_SLOWDOWN:
|
122
|
runtime = float(lease.extras["SWF_runtime"])
|
123
|
waittime = float(lease.extras["SWF_waittime"])
|
124
|
if runtime < 10: runtime = 10
|
125
|
slowdown = (waittime + runtime) / runtime
|
126
|
|
127
|
min = self.deadlinestretch_dist.min
|
128
|
max = self.deadlinestretch_dist.max
|
129
|
tau = self.deadlinestretch_dist.get()
|
130
|
|
131
|
tau = (slowdown - 1)*((tau-min) / (max-min))
|
132
|
|
133
|
deadline = round_datetime_delta(start + (1 + tau)*lease.duration.requested)
|
134
|
elif type == FileGenerator.DEADLINE_ABSOLUTE:
|
135
|
wait = self.deadlinestretch_dist.get()
|
136
|
deadline = round_datetime_delta(start + TimeDelta(seconds=wait) + lease.duration.requested)
|
137
|
|
138
|
tau = ((deadline - start) / lease.duration.requested) - 1
|
139
|
|
140
|
return deadline, tau
|
141
|
|
142
|
def _get_software(self, lease = None):
|
143
|
if self.software_dist == None:
|
144
|
return UnmanagedSoftwareEnvironment()
|
145
|
else:
|
146
|
software = self.software_dist.get()
|
147
|
image_id, image_size = software.split("|")
|
148
|
return DiskImageSoftwareEnvironment(image_id, int(image_size))
|
149
|
|
150
|
|
151
|
def _get_rate(self, lease):
|
152
|
if self.rate_dist == None:
|
153
|
return None
|
154
|
else:
|
155
|
if lease.user_id == -1:
|
156
|
return self.rate_dist.get()
|
157
|
else:
|
158
|
if self.user_rates.has_key(lease.user_id):
|
159
|
return self.user_rates[lease.user_id]
|
160
|
else:
|
161
|
rate = self.rate_dist.get()
|
162
|
self.user_rates[lease.user_id] = rate
|
163
|
return rate
|
164
|
|
165
|
def _get_attributes(self):
|
166
|
attributes = {}
|
167
|
attrs = self.config.get(FileGenerator.GENERAL_SEC, FileGenerator.ATTRIBUTES_OPT)
|
168
|
attrs = attrs.split(",")
|
169
|
for attr in attrs:
|
170
|
(k,v) = attr.split("=")
|
171
|
attributes[k] = v
|
172
|
|
173
|
return attributes
|
174
|
|
175
|
def _get_dist(self, section):
|
176
|
if self.config.has_section(section):
|
177
|
return self.__create_distribution_from_section(section)
|
178
|
else:
|
179
|
return None
|
180
|
|
181
|
def __create_distribution_from_section(self, section):
|
182
|
dist_type = self.config.get(section, FileGenerator.DISTRIBUTION_OPT)
|
183
|
if self.config.has_option(section, FileGenerator.DISCRETE_OPT):
|
184
|
discrete = self.config.getboolean(section, FileGenerator.DISCRETE_OPT)
|
185
|
else:
|
186
|
discrete = False
|
187
|
|
188
|
if self.config.has_option(section, FileGenerator.MIN_OPT) and self.config.has_option(section, FileGenerator.MAX_OPT):
|
189
|
min = self.config.get(section, FileGenerator.MIN_OPT)
|
190
|
max = self.config.get(section, FileGenerator.MAX_OPT)
|
191
|
|
192
|
if min == "unbounded":
|
193
|
min = float("inf")
|
194
|
else:
|
195
|
min = float(min)
|
196
|
if max == "unbounded":
|
197
|
max = float("inf")
|
198
|
else:
|
199
|
max = float(max)
|
200
|
else:
|
201
|
min = max = None
|
202
|
|
203
|
if discrete:
|
204
|
if dist_type == "uniform":
|
205
|
if min != None:
|
206
|
values = range(int(min), int(max) + 1)
|
207
|
else:
|
208
|
values = self.config.get(section, FileGenerator.VALUES_OPT).split()
|
209
|
dist = stats.DiscreteUniformDistribution(values)
|
210
|
else:
|
211
|
if dist_type == "uniform":
|
212
|
dist = stats.UniformDistribution(min, max)
|
213
|
elif dist_type == "normal":
|
214
|
mu = self.config.getfloat(section, FileGenerator.MEAN_OPT)
|
215
|
sigma = self.config.getfloat(section, FileGenerator.STDEV_OPT)
|
216
|
dist = stats.BoundedNormalDistribution(min,max,mu,sigma)
|
217
|
elif dist_type == "bounded-pareto" or dist_type == "truncated-pareto":
|
218
|
alpha = self.config.getfloat(section, FileGenerator.ALPHA_OPT)
|
219
|
if self.config.has_option(section, FileGenerator.INVERT_OPT):
|
220
|
invert = self.config.getboolean(section, FileGenerator.INVERT_OPT)
|
221
|
else:
|
222
|
invert = False
|
223
|
if dist_type == "bounded-pareto":
|
224
|
dist = stats.BoundedParetoDistribution(min,max,alpha,invert)
|
225
|
else:
|
226
|
scale = self.config.getfloat(section, FileGenerator.SCALE_OPT)
|
227
|
dist = stats.TruncatedParetoDistribution(min,max,scale,alpha,invert)
|
228
|
|
229
|
|
230
|
if self.config.has_option(section, FileGenerator.SEED_OPT):
|
231
|
seed = self.config.getint(section, FileGenerator.SEED_OPT)
|
232
|
dist.seed(seed)
|
233
|
|
234
|
return dist
|
235
|
|
236
|
class LWFGenerator(FileGenerator):
|
237
|
|
238
|
SITE_OPT = "site"
|
239
|
|
240
|
NUMLEASES_SEC = "numleases"
|
241
|
|
242
|
NUMLEASES_TYPE_UTILIZATION = "utilization"
|
243
|
NUMLEASES_UTILIZATION_OPT = "utilization"
|
244
|
NUMLEASES_LAST_REQUEST_OPT = "last-request"
|
245
|
|
246
|
NUMLEASES_TYPE_INTERVAL = "interval"
|
247
|
NUMLEASES_OPT = "numleases"
|
248
|
|
249
|
def __init__(self, outfile, conffile):
|
250
|
FileGenerator.__init__(self, outfile, conffile)
|
251
|
|
252
|
self.numleases_type = self.config.get(LWFGenerator.NUMLEASES_SEC, LWFGenerator.TYPE_OPT)
|
253
|
|
254
|
self.interval_dist = self._get_dist(LWFGenerator.NUMLEASES_SEC)
|
255
|
|
256
|
def _get_interval(self):
|
257
|
if self.interval_dist == None:
|
258
|
return None
|
259
|
else:
|
260
|
interval = int(self.interval_dist.get())
|
261
|
return interval
|
262
|
|
263
|
def __gen_lease(self):
|
264
|
submit_time = None
|
265
|
user_id = None
|
266
|
|
267
|
res = self.config.get(LWFGenerator.NODES_SEC, LWFGenerator.RESOURCES_OPT)
|
268
|
res = Capacity.from_resources_string(res)
|
269
|
numnodes = self._get_numnodes(None)
|
270
|
requested_resources = dict([(i+1,res) for i in xrange(numnodes)])
|
271
|
|
272
|
start, delta = self._get_start(self.start_type, None)
|
273
|
start = Timestamp(TimeDelta(seconds=start))
|
274
|
|
275
|
duration = self._get_duration()
|
276
|
duration = Duration(TimeDelta(seconds=duration))
|
277
|
deadline = None
|
278
|
preemptible = False
|
279
|
software = self._get_software()
|
280
|
|
281
|
l = Lease.create_new(submit_time, user_id, requested_resources,
|
282
|
start, duration, deadline, preemptible, software)
|
283
|
|
284
|
return l
|
285
|
|
286
|
def generate(self):
|
287
|
lwf = ET.Element("lease-workload")
|
288
|
lwf.set("name", self.outfile)
|
289
|
description = ET.SubElement(lwf, "description")
|
290
|
description.text = "Created with haizea-generate"
|
291
|
|
292
|
site = self.config.get(LWFGenerator.GENERAL_SEC, LWFGenerator.SITE_OPT)
|
293
|
if site.startswith("file:"):
|
294
|
sitefile = site.split(":")
|
295
|
site = Site.from_xml_file(sitefile[1])
|
296
|
else:
|
297
|
site = Site.from_resources_string(site)
|
298
|
|
299
|
lwf.append(site.to_xml())
|
300
|
|
301
|
time = TimeDelta(seconds=0)
|
302
|
requests = ET.SubElement(lwf, "lease-requests")
|
303
|
|
304
|
if self.numleases_type == LWFGenerator.NUMLEASES_TYPE_INTERVAL:
|
305
|
leases = []
|
306
|
|
307
|
numleases = self.config.getint(LWFGenerator.NUMLEASES_SEC, LWFGenerator.NUMLEASES_OPT)
|
308
|
for i in xrange(numleases):
|
309
|
leases.append(self.__gen_lease())
|
310
|
|
311
|
for l in leases:
|
312
|
interval = TimeDelta(seconds=self._get_interval())
|
313
|
time += interval
|
314
|
l.start.requested += time
|
315
|
lease_request = ET.SubElement(requests, "lease-request")
|
316
|
lease_request.set("arrival", str(time))
|
317
|
lease_request.append(l.to_xml())
|
318
|
elif self.numleases_type == LWFGenerator.NUMLEASES_TYPE_UTILIZATION:
|
319
|
utilization = self.config.getfloat(LWFGenerator.NUMLEASES_SEC, LWFGenerator.NUMLEASES_UTILIZATION_OPT)
|
320
|
last_request = self.config.get(LWFGenerator.NUMLEASES_SEC, LWFGenerator.NUMLEASES_LAST_REQUEST_OPT)
|
321
|
last_request = Parser.DateTimeDeltaFromString(last_request)
|
322
|
|
323
|
max_utilization = 0
|
324
|
for res in site.nodes.get_all_nodes().values():
|
325
|
for i in range(1,res.get_ninstances("CPU") + 1):
|
326
|
max_utilization += res.get_quantity_instance("CPU", i) * last_request.seconds
|
327
|
target_utilization = int(max_utilization * utilization)
|
328
|
|
329
|
accum_utilization = 0
|
330
|
|
331
|
leases = []
|
332
|
|
333
|
while accum_utilization < target_utilization:
|
334
|
lease = self.__gen_lease()
|
335
|
leases.append(lease)
|
336
|
duration = lease.duration.requested.seconds
|
337
|
lease_utilization = 0
|
338
|
for res in lease.requested_resources.values():
|
339
|
for i in range(1,res.get_ninstances("CPU") + 1):
|
340
|
lease_utilization += res.get_quantity_instance("CPU", i) * duration
|
341
|
accum_utilization += lease_utilization
|
342
|
|
343
|
time = TimeDelta(seconds=0)
|
344
|
avg_interval = int(last_request.seconds / len(leases))
|
345
|
for l in leases:
|
346
|
interval = avg_interval + TimeDelta(seconds=self._get_interval())
|
347
|
time = max(time + interval, TimeDelta(seconds=0))
|
348
|
l.start.requested += time
|
349
|
lease_request = ET.SubElement(requests, "lease-request")
|
350
|
lease_request.set("arrival", str(time))
|
351
|
lease_request.append(l.to_xml())
|
352
|
|
353
|
tree = ET.ElementTree(lwf)
|
354
|
|
355
|
outfile = open(self.outfile, "w")
|
356
|
tree.write(outfile)
|
357
|
outfile.close()
|
358
|
|
359
|
class LWFAnnotationGenerator(FileGenerator):
|
360
|
|
361
|
def __init__(self, lwffile, outfile, conffile):
|
362
|
FileGenerator.__init__(self, outfile, conffile)
|
363
|
self.lwffile = lwffile
|
364
|
|
365
|
def generate(self):
|
366
|
lease_workload = LeaseWorkload.from_xml_file(self.lwffile)
|
367
|
leases = lease_workload.get_leases()
|
368
|
annotations = {}
|
369
|
|
370
|
for lease in leases:
|
371
|
lease_id = lease.id
|
372
|
extra = {}
|
373
|
|
374
|
start, delta = self._get_start(self.start_type, lease)
|
375
|
if start != None:
|
376
|
start = Timestamp(start)
|
377
|
extra["simul_start_delta"] = "%.2f" % delta
|
378
|
|
379
|
deadline, tau = self._get_deadline(self.deadline_type, lease, start.requested)
|
380
|
if deadline != None:
|
381
|
extra["simul_deadline_tau"] = "%.2f" % tau
|
382
|
|
383
|
software = self._get_software(lease)
|
384
|
|
385
|
rate = self._get_rate(lease)
|
386
|
|
387
|
if rate != None:
|
388
|
extra["simul_userrate"] = "%.2f" % rate
|
389
|
|
390
|
annotation = LeaseAnnotation(lease_id, start, deadline, software, extra)
|
391
|
annotations[lease_id] = annotation
|
392
|
|
393
|
attributes = self._get_attributes()
|
394
|
|
395
|
annotations = LeaseAnnotations(annotations, attributes)
|
396
|
|
397
|
tree = ET.ElementTree(annotations.to_xml())
|
398
|
outfile = open(self.outfile, "w")
|
399
|
tree.write(outfile)
|
400
|
outfile.close()
|