Changeset 1660:592058a16c4d

Timestamp:

06/13/08 17:13:29 (2 months ago)

Author:

hadoopsandholm@…

Branch:

default

Message:

fixed corr reduce code

Location:

src/grid/hadoop

Files:

• benchmarks/digg/topic-correlation-reduce (modified) (2 diffs)
• bin/createvm (modified) (1 diff)

src/grid/hadoop/benchmarks/digg/topic-correlation-reduce

@@ -1 +1 @@
 #! /usr/bin/python
 import sys
+from math import sqrt
+
+topic_desc = ['offbeat_news','gadgets','videos_comedy','world_news','tech_news','apple','health','videos_music','playable_web_games','security','environment','space','linux_unix','videos_educational','videos_people','politics','xbox_360','music','videos_animation','hardware','business_finance','2008_us_elections','programming','design','software','general_sciences','gaming_news','other_sports','movies','pc_games','mods','political_opinion','baseball','nintendo_wii','extreme_sports','microsoft','celebrity','playstation_3','television','tennis','videos_gaming','football','tech_deals','videos_sports','motorsport','basketball','hockey','golf','soccer','comics_animation','odd_stuff','educational','xbox','comedy','people','nintendo','playstation','pets_animals','travel_places','arts_culture','food_drink','autos']
+topic_map = {}
+
+for i in range(0, len(topic_desc)):
+    topic_map["%d" % (i+1)] = topic_desc[i]
+topic_map['0'] = topic_map['1']
 
 def add_value(key, value, map):
@@ -7 +15 @@
     map[key] += value
 
-users = 0
-key_map = {}
+def corr_sort(x, y):
+    if x['corr'] < y['corr']:
+        return -1    
+    elif x['corr'] > y['corr']:
+        return 1    
+    return 0
+
+sums = {}
+topics = {}
 for line in sys.stdin:
-    users += 1
     line = line.strip()
     cols = line.split('\t')
-    i = 1
-    topic_map = {}
-    while i < len(cols):
-        topic = cols[i]
-        prob = float(cols[i+1]) 
-        #print "%s_mean\t%.20f" % (topic, prob)
-        add_value("%s_mean" % topic, prob, key_map)
-        topic_map[topic] = prob
-        #print "%s_square\t%.20f" % (topic, prob**2)
-        add_value("%s_square" % topic, prob**2, key_map)
-        i += 2
-    for topic, val in topic_map.iteritems():
-        for topic2, val2 in topic_map.iteritems():
-            if topic != topic2:
-                t1 = long(topic)
-                t2 = long(topic2)
-                t_1 = min(t1,t2)
-                t_2 = max(t1,t2)
-                #print "%d_%d_cross\t%.20f" % (t_1,t_2,val*val2)
-                add_value("%d_%d_cross" % (t_1,t_2),val*val2, key_map)
-print "%s\t%d" % ("users",users)
-for key, value in key_map.iteritems():
-    print "%s\t%.20f" % (key, value)
-    #print >>sys.stderr,"%s\t%.20f" % (key, value)
+    if cols[0].endswith("_mean"):
+        topics[cols[0].replace('_mean','')] = 0
+    add_value(cols[0], float(cols[1]), sums)
+topic_names = topics.keys()
+topic_names.sort(lambda x,y: int(x)-int(y))
+corr_list = []
+n = float(sums['users'])
+for topic1 in topic_names:
+    for topic2 in topic_names:
+        if topic1 == topic2:
+            continue
+        if not sums.has_key(topic1 + "_" + topic2 + "_cross"):
+            continue
+        sum1 = sums[topic1 + "_mean"]
+        sum2 = sums[topic2 + "_mean"]
+        sum12 = sums[topic1 + "_" + topic2 + "_cross"]
+        sqr1 = sums[topic1 + "_sqr"]
+        sqr2 = sums[topic2 + "_sqr"]
+        corr = (n * sum12 - sum1 * sum2)/(sqrt(n * sqr1-sum1**2)*sqrt(n*sqr2-sum2**2))
+        corr_list.append({'t1':topic1,'t2':topic2,'corr': corr})
+corr_list.sort(corr_sort)
+for corr in corr_list:
+    print "%.5f %s %s %s %s" % (corr['corr'],corr['t1'],topic_map[corr['t1']],corr['t2'],topic_map[corr['t2']])        

src/grid/hadoop/bin/createvm

@@ -32 +32 @@
 MEMORY=`cat ${HADOOP_ROOT}/hadoop.conf | grep memory_limit | awk '{print $2}'`
 DISK=`cat ${HADOOP_ROOT}/hadoop.conf | grep disk_limit | awk '{print $2}'`
+CPU=`cat ${HADOOP_ROOT}/hadoop.conf | grep cpu_limit | awk '{print $2}'`
 MEM=`${HADOOP_INSTALL}/bin/mult ${MEMORY} 1e-6`
 DIS=`${HADOOP_INSTALL}/bin/mult ${DISK} 1e-9`
-PREFS="CPU:${CPUWEIGHT},0.01,1.0 disk:${DISKWEIGHT},${DIS}GB,${DIS}GB memory:${MEMWEIGHT},${MEM}MB,${MEM}MB"
+CPU=`${HADOOP_INSTALL}/bin/mult ${CPU} 1e-6`
+
+PREFS="CPU:${CPUWEIGHT},${CPU}MHz,3.0GHz disk:${DISKWEIGHT},${DIS}GB,${DIS}GB memory:${MEMWEIGHT},${MEM}MB,${MEM}MB"
 
 MASTER=`cat ${HADOOP_ROOT}/.master 2>/dev/null`