root / trunk / src / haizea / core / scheduler / mapper.py @ 641
1 |
# -------------------------------------------------------------------------- #
|
---|---|
2 |
# Copyright 2006-2009, University of Chicago #
|
3 |
# Copyright 2008-2009, Distributed Systems Architecture Group, Universidad #
|
4 |
# Complutense de Madrid (dsa-research.org) #
|
5 |
# #
|
6 |
# Licensed under the Apache License, Version 2.0 (the "License"); you may #
|
7 |
# not use this file except in compliance with the License. You may obtain #
|
8 |
# a copy of the License at #
|
9 |
# #
|
10 |
# http://www.apache.org/licenses/LICENSE-2.0 #
|
11 |
# #
|
12 |
# Unless required by applicable law or agreed to in writing, software #
|
13 |
# distributed under the License is distributed on an "AS IS" BASIS, #
|
14 |
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
|
15 |
# See the License for the specific language governing permissions and #
|
16 |
# limitations under the License. #
|
17 |
# -------------------------------------------------------------------------- #
|
18 |
|
19 |
"""This module provides the base class for writing custom "mappers" and the
|
20 |
default greedy mapper used in Haizea. A mapper is a class with a single function
|
21 |
"map" that takes a set of requested resources (typically corresponding to
|
22 |
VMs) and maps them to physical nodes (if such a mapping exists).
|
23 |
"""
|
24 |
|
25 |
from haizea.common.utils import abstract |
26 |
from haizea.core.scheduler.slottable import ResourceTuple, AvailabilityWindow |
27 |
import haizea.common.constants as constants |
28 |
import operator |
29 |
|
30 |
# This dictionary provides a shorthand notation for any mappers
|
31 |
# included in this module (this shorthand notation can be used in
|
32 |
# the configuration file)
|
33 |
class_mappings = {"greedy": "haizea.core.scheduler.mapper.GreedyMapper"} |
34 |
|
35 |
class Mapper(object): |
36 |
"""Base class for mappers
|
37 |
|
38 |
"""
|
39 |
|
40 |
def __init__(self, slottable, policy): |
41 |
"""Constructor
|
42 |
|
43 |
Arguments
|
44 |
slottable -- A fully constructed SlotTable
|
45 |
policy -- A fully constructed PolicyManager
|
46 |
"""
|
47 |
self.slottable = slottable
|
48 |
self.policy = policy
|
49 |
|
50 |
|
51 |
def map(self, requested_resources, start, end, strictend, onlynodes = None): |
52 |
"""The mapping function
|
53 |
|
54 |
The mapping function takes a set of requested resources and maps
|
55 |
them to physical resources (based on the availability
|
56 |
in the slot table) in a specified time interval. The mapper
|
57 |
may return a mapping that only satisfies part of the specified
|
58 |
time interval.
|
59 |
|
60 |
Arguments:
|
61 |
requested_resources -- A dictionary mapping lease nodes (integers) to
|
62 |
ResourceTuples (representing the desired amount of resources for
|
63 |
that lease node)
|
64 |
start -- Starting time of the interval during which the resources
|
65 |
are required
|
66 |
end -- Ending time of the interval
|
67 |
strictend -- If True, the only valid mappings are those that span
|
68 |
the entire requested interval. If False, the mapper is allowed to
|
69 |
return mappings that only span part of the interval (this reduced
|
70 |
interval must always start at "start"; the earlier end time is
|
71 |
returned as a return value)
|
72 |
onlynodes -- List of physical nodes. Only look for a mapping in
|
73 |
these nodes.
|
74 |
|
75 |
Returns:
|
76 |
mapping -- A dictionary mapping lease nodes to physical nodes
|
77 |
maxend -- The end of the interval for which a mapping was found.
|
78 |
As noted in argument "strictend", this return value might not
|
79 |
be the same as "end"
|
80 |
preempting -- Leases that would have to be preempted for the
|
81 |
mapping to be valid.
|
82 |
|
83 |
If no mapping is found, the three return values are set to None
|
84 |
"""
|
85 |
abstract() |
86 |
|
87 |
|
88 |
class GreedyMapper(Mapper): |
89 |
"""Haizea's default greedy mapper
|
90 |
|
91 |
Haizea uses a greedy algorithm to determine how VMs are mapped to
|
92 |
physical resources at a specific point in time (determining that point
|
93 |
in time, when using best-effort scheduling, is determined in the lease
|
94 |
and VM scheduling classes).
|
95 |
|
96 |
The way the algorithm works is by, first, greedily ordering the
|
97 |
physical nodes from "most desirable" to "least desirable". For example,
|
98 |
a physical node with no leases scheduled on it in the future is preferable
|
99 |
to one with leases (since this reduces the probability of having to
|
100 |
preempt leases to obtain a mapping). This ordering, however, is done by the
|
101 |
policy engine (see the GreedyPolicy class in the host_selection module) so,
|
102 |
to be a truly greedy algorithm, this mapper must be used in conjunction with
|
103 |
the "greedy" host selection policy).
|
104 |
|
105 |
Then, the algorithm traverses the list of nodes and tries to map as many
|
106 |
lease nodes into each physical node before moving on to the next. If
|
107 |
the list of physical nodes is exhausted without finding a mapping for all
|
108 |
the lease nodes, then the algorithm tries to find a mapping by preempting
|
109 |
other leases.
|
110 |
|
111 |
Before doing this, the mapper must first determine what leases could be
|
112 |
preempted. This decision is delegated to the policy engine, which returns
|
113 |
a list of leases ordered from "most preemptable" to "least preemptable".
|
114 |
The mapper attempts a mapping assuming that the first lease is going
|
115 |
to be preempted, then assuming the first and the second, etc.
|
116 |
|
117 |
If no mapping is found with preemption, then there is no mapping at the
|
118 |
requested time.
|
119 |
|
120 |
"""
|
121 |
|
122 |
def __init__(self, slottable, policy): |
123 |
"""Constructor
|
124 |
|
125 |
Arguments
|
126 |
slottable -- A fully constructed SlotTable
|
127 |
policy -- A fully constructed PolicyManager
|
128 |
"""
|
129 |
Mapper.__init__(self, slottable, policy)
|
130 |
|
131 |
def map(self, lease, requested_resources, start, end, strictend, onlynodes=None): |
132 |
"""The mapping function
|
133 |
|
134 |
See documentation in Mapper for more details
|
135 |
"""
|
136 |
|
137 |
# Generate an availability window at time "start"
|
138 |
aw = self.slottable.get_availability_window(start)
|
139 |
|
140 |
nodes = aw.get_nodes_at(start) |
141 |
if onlynodes != None: |
142 |
nodes = list(set(nodes) & onlynodes) |
143 |
|
144 |
# Get an ordered list of physical nodes
|
145 |
pnodes = self.policy.sort_hosts(nodes, start, lease)
|
146 |
|
147 |
# Get an ordered list of lease nodes
|
148 |
vnodes = self.__sort_vnodes(requested_resources)
|
149 |
|
150 |
# Get the leases that intersect with the requested interval.
|
151 |
leases = aw.get_leases_until(end) |
152 |
# Ask the policy engine to sort the leases based on their
|
153 |
# preemptability
|
154 |
leases = self.policy.sort_leases(lease, leases, start)
|
155 |
|
156 |
preemptable_leases = leases |
157 |
preempting = [] |
158 |
|
159 |
# Try to find a mapping. Each iteration of this loop goes through
|
160 |
# all the lease nodes and tries to find a mapping. The first
|
161 |
# iteration assumes no leases can be preempted, and each successive
|
162 |
# iteration assumes one more lease can be preempted.
|
163 |
mapping = {} |
164 |
done = False
|
165 |
while not done: |
166 |
# Start at the first lease node
|
167 |
vnodes_pos = 0
|
168 |
cur_vnode = vnodes[vnodes_pos] |
169 |
cur_vnode_capacity = requested_resources[cur_vnode] |
170 |
maxend = end |
171 |
|
172 |
# Go through all the physical nodes.
|
173 |
# In each iteration, we try to map as many lease nodes
|
174 |
# as possible into the physical nodes.
|
175 |
# "cur_vnode_capacity" holds the capacity of the vnode we are currently
|
176 |
# trying to map. "need_to_map" is the amount of resources we are
|
177 |
# trying to map into the current physical node (which might be
|
178 |
# more than one lease node).
|
179 |
for pnode in pnodes: |
180 |
# need_to_map is initialized to the capacity of whatever
|
181 |
# lease node we are trying to map now.
|
182 |
need_to_map = self.slottable.create_empty_resource_tuple()
|
183 |
need_to_map.incr(cur_vnode_capacity) |
184 |
avail=aw.get_availability_at_node(start, pnode, preempted_leases = preempting) |
185 |
|
186 |
# Try to fit as many lease nodes as we can into this physical node
|
187 |
pnode_done = False
|
188 |
while not pnode_done: |
189 |
if avail.fits(need_to_map, until = maxend):
|
190 |
# In this case, we can fit "need_to_map" into the
|
191 |
# physical node.
|
192 |
mapping[cur_vnode] = pnode |
193 |
vnodes_pos += 1
|
194 |
if vnodes_pos >= len(vnodes): |
195 |
# No more lease nodes to map, we're done.
|
196 |
done = True
|
197 |
break
|
198 |
else:
|
199 |
# Advance to the next lease node, and add its
|
200 |
# capacity to need_to_map
|
201 |
cur_vnode = vnodes[vnodes_pos] |
202 |
cur_vnode_capacity = requested_resources[cur_vnode] |
203 |
need_to_map.incr(cur_vnode_capacity) |
204 |
else:
|
205 |
# We couldn't fit the lease node. If we need to
|
206 |
# find a mapping that spans the entire requested
|
207 |
# interval, then we're done checking this physical node.
|
208 |
if strictend:
|
209 |
pnode_done = True
|
210 |
else:
|
211 |
# Otherwise, check what the longest interval
|
212 |
# we could fit in this physical node
|
213 |
latest = avail.latest_fit(need_to_map) |
214 |
if latest == None: |
215 |
pnode_done = True
|
216 |
else:
|
217 |
maxend = latest |
218 |
|
219 |
if done:
|
220 |
break
|
221 |
|
222 |
# If there's no more leases that we could preempt,
|
223 |
# we're done.
|
224 |
if len(preemptable_leases) == 0: |
225 |
done = True
|
226 |
elif not done: |
227 |
# Otherwise, add another lease to the list of
|
228 |
# leases we are preempting
|
229 |
preempting.append(preemptable_leases.pop()) |
230 |
|
231 |
if len(mapping) != len(requested_resources): |
232 |
# No mapping found
|
233 |
return None, None, None |
234 |
else:
|
235 |
return mapping, maxend, preempting
|
236 |
|
237 |
def __sort_vnodes(self, requested_resources): |
238 |
"""Sorts the lease nodes
|
239 |
|
240 |
Greedily sorts the lease nodes so the mapping algorithm
|
241 |
will first try to map those that require the highest
|
242 |
capacity.
|
243 |
"""
|
244 |
|
245 |
# Find the maximum requested resources for each resource type
|
246 |
max_res = self.slottable.create_empty_resource_tuple()
|
247 |
for res in requested_resources.values(): |
248 |
for i in range(len(res._res)): |
249 |
if res._res[i] > max_res._res[i]:
|
250 |
max_res._res[i] = res._res[i] |
251 |
|
252 |
# Normalize the capacities of the lease nodes (divide each
|
253 |
# requested amount of a resource type by the maximum amount)
|
254 |
norm_res = {} |
255 |
for k,v in requested_resources.items(): |
256 |
norm_capacity = 0
|
257 |
for i in range(len(max_res._res)): |
258 |
if max_res._res[i] > 0: |
259 |
norm_capacity += v._res[i] / float(max_res._res[i])
|
260 |
norm_res[k] = norm_capacity |
261 |
|
262 |
vnodes = norm_res.items() |
263 |
vnodes.sort(key=operator.itemgetter(1), reverse = True) |
264 |
vnodes = [k for k,v in vnodes] |
265 |
return vnodes
|
266 |
|