Project

General

Profile

Revision 798

Utilization-based LWF generation (still needs some work, but mostly done)

View differences:

branches/1.1/src/haizea/cli/commands.py
25 25
from haizea.cli.optionparser import Option
26 26
from haizea.cli import Command
27 27
from haizea.lwf.generators import LWFGenerator, LWFAnnotationGenerator
28
from haizea.lwf.analysis import LWFAnalyser
28 29
from mx.DateTime import TimeDelta, Parser
29 30
import haizea.common.defaults as defaults
30 31
import sys
......
367 368
        self.optparser.add_option(Option("-c", "--conf", action="store", type="string", dest="conf",
368 369
                                         help = """
369 370
                                         ...
370
                                         """))
371
                                         """))      
371 372
        
372 373
    def run(self):
373 374
        self.parse_options()      
......
379 380
        
380 381
        generator.generate()
381 382

  
383
class haizea_lwf_stats(Command):
384
    
385
    name = "haizea-lwf-stats"
386
    
387
    def __init__(self, argv):
388
        Command.__init__(self, argv)
389
        self.optparser.add_option(Option("-i", "--in", action="store",  type="string", dest="inf", required=True,
390
                                         help = """
391
                                         Input file
392
                                         """))
393
        self.optparser.add_option(Option("-l", "--utilization-length", action="store", type="string", dest="utilization_length",
394
                                         help = """
395
                                         Length of the utilization interval in format DD:HH:MM:SS. Default is until
396
                                         the time the last lease is requested.
397
                                         """))          
398
        
399
    def run(self):
400
        self.parse_options()      
401
        
402
        infile = self.opt.inf
403
        utilization_length = self.opt.utilization_length
404
        if utilization_length != None:
405
            utilization_length = Parser.DateTimeDeltaFromString(utilization_length)
406

  
407
        analyser = LWFAnalyser(infile, utilization_length)
408
        
409
        analyser.analyse()
410

  
382 411
class haizea_lwf_annotate(Command):
383 412
    
384 413
    name = "haizea-lwf-annotate"
branches/1.1/src/haizea/common/stats.py
137 137
        DiscreteDistribution.__init__(self, values)
138 138
        self.__distribution = UniformDistribution(0,1)
139 139
        
140
    def seed(self, x):
141
        self.__distribution.seed(x)
142
        
140 143
    def get(self):
141 144
        return self._get_from_prob(self.__distribution.get())      
142 145
    
branches/1.1/src/haizea/core/frontends/tracefile.py
40 40
        # Requests is a list of lease requests
41 41
        self.logger.info("Loading tracefile %s" % tracefile)
42 42
        self.requests = None
43
        if tracefile.endswith(".swf"):
44
            self.requests = LeaseWorkload.from_swf_file(tracefile, self.starttime)
45
        elif tracefile.endswith(".lwf") or tracefile.endswith(".xml"):
46
            lease_workload = LeaseWorkload.from_xml_file(tracefile, self.starttime)
47
            self.requests = lease_workload.get_leases()
43
        lease_workload = LeaseWorkload.from_xml_file(tracefile, self.starttime)
44
        self.requests = lease_workload.get_leases()
48 45
    
49 46
        if injectfile != None:
50 47
            self.logger.info("Loading injection file %s" % injectfile)
branches/1.1/src/haizea/core/leases.py
1573 1573
        Argument:
1574 1574
        lwf_file -- LWF file.
1575 1575
        """                
1576
        return cls.__from_xml_element(ET.parse(lwf_file).getroot().find("site"))        
1576
        site_elem = ET.parse(lwf_file).getroot().find("site")
1577
        if site_elem == None:
1578
            return None # LWF file does not contain a <site> element
1579
        else:
1580
            return cls.__from_xml_element(site_elem)        
1577 1581
        
1578 1582
    @classmethod
1579 1583
    def __from_xml_element(cls, element):     
......
1620 1624

  
1621 1625
        resource_str = resource_str.split()
1622 1626
        numnodes = int(resource_str[0])
1623
        resources = resource_str[1:]
1627
        resources = resource_str[1]
1624 1628
        capacity = Capacity.from_resources_string(resources)
1625 1629
        
1626 1630
        nodes = Nodes([(numnodes,capacity)])
......
1659 1663

  
1660 1664
        return max_ninstances
1661 1665
    
1666
    def to_xml(self):
1667
        """Returns an ElementTree XML representation of the nodes
1668
        
1669
        See the Haizea documentation for details on the
1670
        lease XML format.
1671
        
1672
        """   
1673
        site = ET.Element("site")
1674
        resource_types = ET.SubElement(site, "resource-types")
1675
        resource_types.set("names", " ".join(self.resource_types))
1676
        site.append(self.nodes.to_xml())
1677
            
1678
        return site    
1662 1679

  
1663 1680

  
1664 1681
class Nodes(object):
branches/1.1/src/haizea/lwf/generators.py
19 19
import haizea.common.stats as stats
20 20
from haizea.core.leases import LeaseWorkload, LeaseAnnotation, LeaseAnnotations, Timestamp, Lease,\
21 21
    Capacity, Duration, UnmanagedSoftwareEnvironment,\
22
    DiskImageSoftwareEnvironment
22
    DiskImageSoftwareEnvironment, Site
23 23
from haizea.common.utils import round_datetime_delta
24 24
from mx.DateTime import DateTimeDelta, TimeDelta, Parser
25 25
import ConfigParser 
......
235 235

  
236 236
class LWFGenerator(FileGenerator):
237 237
    
238
    SITE_OPT = "site"    
239

  
238 240
    NUMLEASES_SEC = "numleases"    
239 241
    
240 242
    NUMLEASES_TYPE_UTILIZATION = "utilization"    
......
249 251

  
250 252
        self.numleases_type = self.config.get(LWFGenerator.NUMLEASES_SEC, LWFGenerator.TYPE_OPT)
251 253
        
252
        if self.numleases_type == LWFGenerator.NUMLEASES_TYPE_INTERVAL:
253
            self.interval_dist = self._get_dist(LWFGenerator.NUMLEASES_SEC)
254
        else:
255
            self.interval_dist = None
256
            
254
        self.interval_dist = self._get_dist(LWFGenerator.NUMLEASES_SEC)
255

  
257 256
    def _get_interval(self):
258 257
        if self.interval_dist == None:
259 258
            return None
......
290 289
        description = ET.SubElement(lwf, "description")
291 290
        description.text = "Created with haizea-generate"
292 291

  
293
        #if self.opt.site != None:
294
        #    site_elem = ET.parse(self.opt.site).getroot()
295
        #    site_num_nodes = int(site_elem.find("nodes").find("node-set").get("numnodes"))
296
        #    lwf.append(site_elem)
292
        site = self.config.get(LWFGenerator.GENERAL_SEC, LWFGenerator.SITE_OPT)
293
        if site.startswith("file:"):
294
            sitefile = site.split(":")
295
            site = Site.from_xml_file(sitefile[1])
296
        else:
297
            site = Site.from_resources_string(site)
298

  
299
        lwf.append(site.to_xml())
297 300
            
298 301
        time = TimeDelta(seconds=0)
299
        requests = ET.SubElement(lwf, "lease-requests")            
302
        requests = ET.SubElement(lwf, "lease-requests")   
300 303
        
301 304
        if self.numleases_type == LWFGenerator.NUMLEASES_TYPE_INTERVAL:
302 305
            leases = []            
......
308 311
            for l in leases:
309 312
                interval = TimeDelta(seconds=self._get_interval())
310 313
                time += interval
311
                print interval, time
312 314
                l.start.requested += time
313 315
                lease_request = ET.SubElement(requests, "lease-request")
314 316
                lease_request.set("arrival", str(time))            
......
318 320
            last_request = self.config.get(LWFGenerator.NUMLEASES_SEC, LWFGenerator.NUMLEASES_LAST_REQUEST_OPT)
319 321
            last_request = Parser.DateTimeDeltaFromString(last_request)
320 322
            
321
            # TODO
322
            target_utilization = utilization
323
            accum_utilization = None
323
            max_utilization = 0
324
            for res in site.nodes.get_all_nodes().values():
325
                for i in range(1,res.get_ninstances("CPU") + 1):
326
                    max_utilization += res.get_quantity_instance("CPU", i) * last_request.seconds
327
            target_utilization = int(max_utilization * utilization)
324 328
            
329
            accum_utilization = 0
330
            
325 331
            leases = []            
326 332

  
327 333
            while accum_utilization < target_utilization:
328
                pass
329
                self.__gen_lease()
334
                lease = self.__gen_lease()
335
                leases.append(lease)
336
                duration = lease.duration.requested.seconds
337
                lease_utilization = 0
338
                for res in lease.requested_resources.values():
339
                    for i in range(1,res.get_ninstances("CPU") + 1):
340
                        lease_utilization += res.get_quantity_instance("CPU", i) * duration                
341
                accum_utilization += lease_utilization
330 342
                
331
            # TODO: Set arrival times so they are evenly spaced
343
            time = TimeDelta(seconds=0)            
344
            avg_interval = int(last_request.seconds / len(leases))
332 345
            for l in leases:
346
                interval = avg_interval + TimeDelta(seconds=self._get_interval())
347
                time = max(time + interval, TimeDelta(seconds=0))
348
                l.start.requested += time
333 349
                lease_request = ET.SubElement(requests, "lease-request")
334 350
                lease_request.set("arrival", str(time))            
335
                lease_request.append(l.to_xml())            
351
                lease_request.append(l.to_xml())                            
336 352
        
337
            
338
        print ET.tostring(lwf)
353
        tree = ET.ElementTree(lwf)
354
        
355
        outfile = open(self.outfile, "w")
356
        tree.write(outfile)
357
        outfile.close()
339 358
    
340 359
class LWFAnnotationGenerator(FileGenerator):
341 360
    
branches/1.1/src/haizea/lwf/analysis.py
14 14
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.   #
15 15
# See the License for the specific language governing permissions and        #
16 16
# limitations under the License.                                             #
17
# -------------------------------------------------------------------------- #
17
# -------------------------------------------------------------------------- #
18
from mx.DateTime import DateTime
19
from haizea.core.leases import LeaseWorkload, Site
20

  
21

  
22
class LWFAnalyser(object):
23
    
24
    
25
    def __init__(self, lwffile, utilization_length):
26
        # Arbitrary start time
27
        self.starttime = DateTime(2006,11,25,13)
28
        
29
        self.workload = LeaseWorkload.from_xml_file(lwffile, self.starttime)
30
        self.site = Site.from_lwf_file(lwffile)
31
        
32
        if utilization_length == None:
33
            self.utilization_length = self.workload.get_leases()[-1].submit_time - self.starttime
34
        else:
35
            self.utilization_length = utilization_length
36
        print self.utilization_length
37
        
38
    def analyse(self):
39
        utilization = 0
40
        for lease in self.workload.get_leases():
41
            if lease.start.requested + lease.duration.requested > self.starttime + self.utilization_length:
42
                duration = self.starttime + self.utilization_length - lease.start.requested
43
            else: 
44
                duration = lease.duration.requested.seconds
45
            for res in lease.requested_resources.values():
46
                for i in range(1,res.get_ninstances("CPU") + 1):
47
                    utilization += res.get_quantity_instance("CPU", i) * duration
48
        
49
        if self.site != None:
50
            max_utilization = 0
51
            duration = self.utilization_length.seconds
52
            for res in self.site.nodes.get_all_nodes().values():
53
                for i in range(1,res.get_ninstances("CPU") + 1):
54
                    max_utilization += res.get_quantity_instance("CPU", i) * duration
55
            
56
        
57
        print "Utilization: %.2f%%" % (utilization / max_utilization)
58
        
branches/1.1/bin/haizea-lwf-generate
3 3
from haizea.cli import commands
4 4
import sys
5 5
	
6
c = commands.haizea_lwf_annotate(sys.argv)
6
c = commands.haizea_lwf_generate(sys.argv)
7 7
c.run()
branches/1.1/bin/haizea-lwf-stats
1
#!/usr/bin/python
2

  
3
from haizea.cli import commands
4
import sys
5
	
6
c = commands.haizea_lwf_stats(sys.argv)
7
c.run()
0 8

  

Also available in: Unified diff