Changeset 1650:e53bd66d0178

Timestamp:

06/09/08 14:54:03 (3 months ago)

Author:

hadoopsandholm@…

Branch:

default

Message:

added initial rebidding and usage accounting features

Location:

src/grid/hadoop/bin

Files:

• Predictor.py (modified) (3 diffs)
• SpendingTracker.py (added)
• createvm (modified) (2 diffs)
• monitorcluster (modified) (1 diff)
• nodeperformance (modified) (1 diff)
• predict (modified) (1 diff)
• th_get-capacity (modified) (1 diff)
• th_get-performance (modified) (1 diff)
• th_get-usage (added)
• th_versions (added)

src/grid/hadoop/bin/Predictor.py

@@ -120 +120 @@
 exec "ppf=" +  sys.argv[8] + "_ppf"
 exec "err_ppf=" + sys.argv[9] + "_ppf"
-resource_factor = float(sys.argv[10])
+spending_file = sys.argv[10]
+discount_spending = sys.argv[11]
 log = open(".hadooptrace","w")
 f = open(filename,'r')
 price_list = []
+
+discount_map = {}
+if discount_spending == "yes":
+    try:
+        spending = open(spending_file,'r')
+        for line in spending:
+            cols = line.split(' ')
+            discount_map[cols[0]] = {'CPU':float(cols[1]),'memory':float(cols[2]),'disk':float(cols[3])}
+    except:
+        pass
 
 for line in f:
@@ -142 +153 @@
        price[resource] = {}
        price[resource]['capacity'] = cols[start_index]-cols[start_index+1]  
-       price[resource]['moms'] = [[cols[start_index+3], cols[start_index+4]],[cols[start_index+5],cols[start_index+6]],[cols[start_index+7],cols[start_index+8]],[cols[start_index+9],cols[start_index+10]]]
+       if discount_map.has_key(address):
+           discount = discount_map[address][resource]
+       else:
+           discount = 0.0
+       price[resource]['moms'] = [[cols[start_index+3]-discount, cols[start_index+4]],[cols[start_index+5]-discount,cols[start_index+6]],[cols[start_index+7]-discount,cols[start_index+8]],[cols[start_index+9]-discount,cols[start_index+10]]]
        if sys.argv[8] != "current":  
            price[resource]['predict'] = prediction_bound(confidence, guarantee, price[resource]['moms'], ppf, err_ppf)  
@@ -148 +163 @@
            if cols[start_index+2] < 1e-20:
                cols[start_index+2] = 1e-20
-           price[resource]['predict'] = cols[start_index+2]
+           price[resource]['predict'] = cols[start_index+2]-discount
        start_index = start_index + 11
 

src/grid/hadoop/bin/createvm

@@ -7 +7 @@
 IMAGE=$5
 
-
+MAXBID=`cat ${HADOOP_ROOT}/hadoop.conf | grep bid | awk '{print $2}'`
 
 IPBID=
@@ -42 +42 @@
 fi
 
-echo tycoon -u ${USER} host create_account $HOST $BID $TIME $PREFS $IPBID file_system:http://tycoon.hpl.hp.com/~tycoon/dl/images/$IMAGE >>${HADOOP_ROOT}/.createtrace
-OUT=`tycoon -u ${USER} host create_account $HOST $BID $TIME $PREFS $IPBID file_system:http://tycoon.hpl.hp.com/~tycoon/dl/images/$IMAGE`
+echo tycoon -u ${USER} host create_account $HOST ${BID},${MAXBID} $TIME $PREFS $IPBID file_system:http://tycoon.hpl.hp.com/~tycoon/dl/images/$IMAGE >>${HADOOP_ROOT}/.createtrace
+OUT=`tycoon -u ${USER} host create_account $HOST ${BID},${MAXBID} $TIME $PREFS $IPBID file_system:http://tycoon.hpl.hp.com/~tycoon/dl/images/$IMAGE`
 echo $OUT
 echo $OUT | grep "Refunded" > /dev/null && exit 1 

src/grid/hadoop/bin/monitorcluster

@@ -79 +79 @@
   done
 
+  cat ${HADOOP_ROOT}/.performance | python ${HADOOP_INSTALL}/bin/SpendingTracker.py ${HADOOP_ROOT} 60 > ${HADOOP_ROOT}/.spendingavg.hour
   sleep 60
   MONITOR=`cat ${HADOOP_ROOT}/.monitor 2>/dev/null`

src/grid/hadoop/bin/nodeperformance

@@ -57 +57 @@
 IP=`cat ${HADOOP_ROOT}/.hosts | grep ${HOST} | awk '{print $2}'`
 
-echo -e "${IP} ${CPUVAL} ${CPUAVAIL} ${CPUSHARE} ${CPUSPEND} ${MEMVAL} ${MEMAVAIL} ${MEMSHARE} ${MEMSPEND} ${DISKVAL} ${DISKAVAIL} ${DISKSHARE} ${DISKSPEND}"
+echo -e "${IP} ${CPUVAL} ${CPUAVAIL} ${CPUSHARE} ${CPUSPEND} ${MEMVAL} ${MEMAVAIL} ${MEMSHARE} ${MEMSPEND} ${DISKVAL} ${DISKAVAIL} ${DISKSHARE} ${DISKSPEND} ${HOST}"
 
 rm -f ${HADOOP_ROOT}/.auctioneer_stat${HOST}.$$

src/grid/hadoop/bin/predict

@@ -8 +8 @@
 PPF=`cat ${HADOOP_ROOT}/hadoop.conf | grep ppf_model | awk '{print $2}'`
 PPFERR=`cat ${HADOOP_ROOT}/hadoop.conf | grep ppf_err_model | awk '{print $2}'`
-RESOURCEFACT=`cat ${HADOOP_ROOT}/hadoop.conf | grep resource_factor | awk '{print $2}'`
+WINDOW=`cat ${HADOOP_ROOT}/hadoop.conf | grep prediction_window | awk '{print $2}'`
+DISCOUNT=`cat ${HADOOP_ROOT}/hadoop.conf | grep discount_spending | awk '{print $2}'`
 
-python ${HADOOP_INSTALL}/bin/Predictor.py ${HADOOP_ROOT}/.stats $GUARANTEE $CONFIDENCE $CPULIMIT $MEMORYLIMIT $DISKLIMIT $TIME $PPF $PPFERR $RESOURCEFACT > ${HADOOP_ROOT}/.prices
+python ${HADOOP_INSTALL}/bin/Predictor.py ${HADOOP_ROOT}/.stats $GUARANTEE $CONFIDENCE $CPULIMIT $MEMORYLIMIT $DISKLIMIT $TIME $PPF $PPFERR ${HADOOP_ROOT}/.spendingavg.${WINDOW} $DISCOUNT > ${HADOOP_ROOT}/.prices

src/grid/hadoop/bin/th_get-capacity

@@ -2 +2 @@
 USER=`id -un`
 SLAVES=`cat ${HADOOP_ROOT}/.hosts | awk '{print $1}'`
-#ip cpu_cap cpu_avail cpu_share cpu_spend mem_cap mem_avail mem_share mem_spend disk_cap disk_avail disk_share disk_spend
+#ip cpu_cap cpu_avail cpu_share cpu_spend mem_cap mem_avail mem_share mem_spend disk_cap disk_avail disk_share disk_spend host
 tycoon_hadoop get-performance -h > ${HADOOP_ROOT}/.performance
 CPU_CAP=`cat ${HADOOP_ROOT}/.performance | awk '{print $2}'`

src/grid/hadoop/bin/th_get-performance

@@ -3 +3 @@
 vxargs.py -p -y -a ${HADOOP_ROOT}/.ips -o ${HADOOP_ROOT}/logs/command${MYPROC} ${HADOOP_INSTALL}/bin/nodeperformance {} >/dev/null
 if [ "$1" != "-h" ]; then
-    echo "ip cpu_cap cpu_avail cpu_share cpu_spend mem_cap mem_avail mem_share mem_spend disk_cap disk_avail disk_share disk_spend"
+    echo "ip cpu_cap cpu_avail cpu_share cpu_spend mem_cap mem_avail mem_share mem_spend disk_cap disk_avail disk_share disk_spend host"
 fi