Changeset 1659:4cbf5a4c527a

Timestamp:

06/13/08 14:17:11 (2 months ago)

Author:

hadoopsandholm@…

Branch:

default

Message:

digg benchmark

Location:

src/grid/hadoop

Files:

• benchmarks/digg/digg-analysis (added)
• benchmarks/digg/topic-correlation-map (added)
• benchmarks/digg/topic-correlation-reduce (added)
• benchmarks/digg/user-topic-map (added)
• benchmarks/digg/user-topic-prob-map (added)
• benchmarks/digg/user-topic-prob-reduce (added)
• benchmarks/digg/user-topic-reduce (added)
• bin/th_stream-raw (added)
• bin/th_upload (modified) (1 diff)
• lib/stream-job-wrapper (modified) (1 diff)

src/grid/hadoop/bin/th_upload

@@ -4 +4 @@
 ${SSHCMD} ${MASTER} "rm -rf /tmp/${STREAMINDIR}; mkdir /tmp/${STREAMINDIR}"
 ${SCPCMD} ${HADOOP_ROOT}/input/${1} ${MASTER}:/tmp/${STREAMINDIR}/upload.tar.gz
-${SSHCMD} ${MASTER} "tar -C /tmp/${STREAMINDIR} -xvf /tmp/${STREAMINDIR}/upload.tar.gz; rm /tmp/${STREAMINDIR}/upload.tar.gz; hadoop dfs -put /tmp/${STREAMINDIR} /${STREAMINDIR}; rm -rf /tmp/${STREAMINDIR};"
+${SSHCMD} ${MASTER} "tar -C /tmp/${STREAMINDIR} -xvf /tmp/${STREAMINDIR}/upload.tar.gz; rm /tmp/${STREAMINDIR}/upload.tar.gz; hadoop dfs -mkdir /${STREAMINDIR}; hadoop dfs -copyFromLocal /tmp/${STREAMINDIR}/* /${STREAMINDIR}/; rm -rf /tmp/${STREAMINDIR};"
 

src/grid/hadoop/lib/stream-job-wrapper

@@ -4 +4 @@
 rm -rf out
 mkdir out
-hadoop dfs -copyToLocal /${INDIR}/$INPUT $INPUT
+#hadoop dfs -copyToLocal /${INDIR}/$INPUT $INPUT
 START=`date +%s`
-/bin/sh stream-job $INPUT 
+/bin/sh stream-job "$INPUT"
 END=`date +%s`
 ELAPSED=`expr $END - $START`
-dir=${INPUT}.out
-hadoop dfs -put out /${OUTDIR}/${dir}
-echo -e "${INPUT}\t${ELAPSED}"
+#dir=${INPUT}.out
+#hadoop dfs -put out /${OUTDIR}/${dir}
+#echo -e "${INPUT}\t${ELAPSED}"
 read INPUT
 done