Mercurial > hg > Members > nobuyasu > jungle_benchmark

changeset 25:726b2deadbe3

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
calc_sd.rb
author Nobuyasu
date Fri, 31 Jan 2014 04:51:45 +0900
parents 4bc7b8835f56
children cfae346a232e
files distributed/weighttp/calc_sd.rb distributed/weighttp/capistrano_weighttp.rb distributed/weighttp/weighttp_jungle.sh
diffstat 3 files changed, 138 insertions(+), 25 deletions(-) [+]
line wrap: on
line diff
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/distributed/weighttp/calc_sd.rb	Fri Jan 31 04:51:45 2014 +0900
@@ -0,0 +1,82 @@
+#!/usr/local/bin/ruby
+
+class Array
+  def average
+    inject(0.0) { |sum, i| sum += i } / size
+  end
+
+  def variance
+    ave = average
+    inject(0.0) { |sum, i| sum += (i - ave)**2 } / size
+  end
+
+  def standard_devitation
+    Math::sqrt(variance)
+  end
+end
+
+
+ def log_select(filename) 
+   return logArray = File.open(filename, "r") do |file|
+      file.readlines.select{|line| line =~ /sec/ }
+   end 
+ end
+
+ def create_table(logArray)
+    totalFailed = 0
+    timeArray = []
+    logArray.each{|line|
+     tokens = line.split(' ')
+     sec = tokens[0].to_i * 1000
+     millsec = tokens[2].to_i
+     time = sec + millsec
+     timeArray.push(time)
+     totalFailed = totalFailed + tokens[4].to_i
+    }
+   size = timeArray.size().to_s
+   return {size => timeArray}
+ end
+
+
+ def add_table(table1, table2)
+   t = {}
+   table1.each {|key, value|
+     if table2.key?(key) then
+	t = {key => value.concat(table2[key])}
+     else
+        t = {key => value}
+     end
+   }
+   table2.each {|key, value|
+     if table1.key?(key) then
+       # nop continue
+     else
+        t = {key => value}     
+     end
+    }
+   return t
+ end
+
+ def print_log(nodes, array)
+#    puts array.size()
+#    puts array.variance
+#    puts array.standard_devitation
+    puts nodes.to_s + " " + array.average.to_s + " " + array.standard_devitation.to_s
+ end
+
+ def start(files)
+   totalTable = {}
+   files.each {|filename| 
+     logArray = log_select(filename)
+     table = create_table(logArray)
+     totalTable = add_table(table, totalTable)
+    }
+
+    totalTable.each {|key, value|
+	print_log(key, value)
+    }
+   
+ end
+
+start(ARGV)
+
--- a/distributed/weighttp/capistrano_weighttp.rb	Fri Jan 31 00:51:16 2014 +0900
+++ b/distributed/weighttp/capistrano_weighttp.rb	Fri Jan 31 04:51:45 2014 +0900
@@ -8,10 +8,13 @@
 
 
 # Server for BBS
-role :pall23, "133.13.62.39","133.13.62.40","133.13.62.41","133.13.62.42","133.13.62.43","133.13.62.44","133.13.62.45","133.13.62.46","133.13.62.47", "133.13.62.48", "133.13.62.29","133.13.62.30","133.13.62.31","133.13.62.32","133.13.62.33","133.13.62.34","133.13.62.35","133.13.62.36","133.13.62.37","133.13.62.38","133.13.62.1","133.13.62.20","133.13.62.21","133.13.62.22","133.13.62.23","133.13.62.24","133.13.62.25","133.13.62.26","133.13.62.27","133.13.62.28"
+role :pall23, "133.13.62.39","133.13.62.40","133.13.62.41","133.13.62.42","133.13.62.43","133.13.62.44","133.13.62.45","133.13.62.46","133.13.62.47", "133.13.62.48", "133.13.62.29","133.13.62.30","133.13.62.31","133.13.62.32","133.13.62.33","133.13.62.34","133.13.62.35","133.13.62.36","133.13.62.37","133.13.62.38","133.13.62.1","133.13.62.20","133.13.62.21","133.13.62.22","133.13.62.23"
+
+role :pall225, "133.13.62.24","133.13.62.25","133.13.62.26","133.13.62.27","133.13.62.28","133.13.62.29","133.13.62.30","133.13.62.31","133.13.62.32","133.13.62.33","133.13.62.34","133.13.62.35","133.13.62.36","133.13.62.37","133.13.62.38","133.13.62.39","133.13.62.40","133.13.62.41","133.13.62.42","133.13.62.43","133.13.62.44","133.13.62.45","133.13.62.46","133.13.62.47", "133.13.62.48"
 
-role :pall22, "133.13.62.39","133.13.62.40","133.13.62.41","133.13.62.42","133.13.62.43","133.13.62.44","133.13.62.45","133.13.62.46","133.13.62.47", "133.13.62.48", "133.13.62.29","133.13.62.30","133.13.62.31","133.13.62.32","133.13.62.33","133.13.62.34","133.13.62.35","133.13.62.36","133.13.62.37","133.13.62.38"
+role :pall22, "133.13.62.29","133.13.62.30","133.13.62.31","133.13.62.32","133.13.62.33","133.13.62.34","133.13.62.35","133.13.62.36","133.13.62.37","133.13.62.38","133.13.62.39","133.13.62.40","133.13.62.41","133.13.62.42","133.13.62.43","133.13.62.44","133.13.62.45","133.13.62.46","133.13.62.47", "133.13.62.48"
 
+role :pall215, "133.13.62.39","133.13.62.40","133.13.62.41","133.13.62.42","133.13.62.43","133.13.62.44","133.13.62.45","133.13.62.46","133.13.62.47", "133.13.62.48","133.13.62.34","133.13.62.35","133.13.62.36","133.13.62.37","133.13.62.38"
 role :pall21, "133.13.62.39","133.13.62.40","133.13.62.41","133.13.62.42","133.13.62.43","133.13.62.44","133.13.62.45","133.13.62.46","133.13.62.47", "133.13.62.48"
 
 
@@ -48,15 +51,23 @@
 end
 
 task :run_jungle30, :roles =>:pall23 do 
-  run "export MAVEN_OPTS='-Xmx6g -Xms4g'  && /usr/local/apache-maven/apache-maven-3.0.5/bin/mvn -f /home/mass/student/k128576/hg/jungle-network/pom.xml exec:java -Dexec.mainClass='app.bbs.DistributeApp' -Dexec.args='-level fatal -host bldsv12.cr.ie.u-ryukyu.ac.jp -port 10000 -p 20000 -bp 8080' "
+  run "export MAVEN_OPTS='-Xmx6g -Xms4g'  && /usr/local/apache-maven/apache-maven-3.0.5/bin/mvn -f /home/mass/student/k128576/hg/jungle-network/pom.xml exec:java -Dexec.mainClass='app.bbs.DistributeApp' -Dexec.args='-level fatal -host mass00.cs.ie.u-ryukyu.ac.jp -port 10000 -p 20000 -bp 8080' "
+end
+
+task :run_jungle25, :roles =>:pall225 do 
+  run "export MAVEN_OPTS='-Xmx6g -Xms4g'  && /usr/local/apache-maven/apache-maven-3.0.5/bin/mvn -f /home/mass/student/k128576/hg/jungle-network/pom.xml exec:java -Dexec.mainClass='app.bbs.DistributeApp' -Dexec.args='-level fatal -host mass00.cs.ie.u-ryukyu.ac.jp -port 10000 -p 20000 -bp 8080' "
 end
 
 task :run_jungle20, :roles =>:pall22 do 
-  run "export MAVEN_OPTS='-Xmx6g -Xms4g'  && /usr/local/apache-maven/apache-maven-3.0.5/bin/mvn -f /home/mass/student/k128576/hg/jungle-network/pom.xml exec:java -Dexec.mainClass='app.bbs.DistributeApp' -Dexec.args='-level fatal -host bldsv12.cr.ie.u-ryukyu.ac.jp -port 10000 -p 20000 -bp 8080' "
+  run "export MAVEN_OPTS='-Xmx6g -Xms4g'  && /usr/local/apache-maven/apache-maven-3.0.5/bin/mvn -f /home/mass/student/k128576/hg/jungle-network/pom.xml exec:java -Dexec.mainClass='app.bbs.DistributeApp' -Dexec.args='-level fatal -host mass00.cs.ie.u-ryukyu.ac.jp -port 10000 -p 20000 -bp 8080' "
+end
+
+task :run_jungle15, :roles =>:pall215 do 
+  run "export MAVEN_OPTS='-Xmx6g -Xms4g'  && /usr/local/apache-maven/apache-maven-3.0.5/bin/mvn -f /home/mass/student/k128576/hg/jungle-network/pom.xml exec:java -Dexec.mainClass='app.bbs.DistributeApp' -Dexec.args='-level fatal -host mass00.cs.ie.u-ryukyu.ac.jp -port 10000 -p 20000 -bp 8080' "
 end
 
 task :run_jungle10, :roles =>:pall21 do 
-  run "export MAVEN_OPTS='-Xmx6g -Xms4g'  && /usr/local/apache-maven/apache-maven-3.0.5/bin/mvn -f /home/mass/student/k128576/hg/jungle-network/pom.xml exec:java -Dexec.mainClass='app.bbs.DistributeApp' -Dexec.args='-level fatal -host bldsv12.cr.ie.u-ryukyu.ac.jp -port 10000 -p 20000 -bp 8080' "
+  run "export MAVEN_OPTS='-Xmx6g -Xms4g'  && /usr/local/apache-maven/apache-maven-3.0.5/bin/mvn -f /home/mass/student/k128576/hg/jungle-network/pom.xml exec:java -Dexec.mainClass='app.bbs.DistributeApp' -Dexec.args='-level fatal -host mass00.cs.ie.u-ryukyu.ac.jp -port 10000 -p 20000 -bp 8080' "
 end
 
 
@@ -65,15 +76,29 @@
   run "cd /home/mass/student/k128576/cassandra/ &&  /home/mass/student/k128576/cassandra/apache-cassandra-2.0.4/bin/cassandra -f"
 end
 
+task :start_cassandra25, :roles =>:pall225 do 
+  run "cd /home/mass/student/k128576/cassandra/ &&  /home/mass/student/k128576/cassandra/apache-cassandra-2.0.4/bin/cassandra -f"
+end
+
 task :start_cassandra20, :roles =>:pall22 do 
   run "cd /home/mass/student/k128576/cassandra/ &&  /home/mass/student/k128576/cassandra/apache-cassandra-2.0.4/bin/cassandra -f"
 end
 
+task :start_cassandra15, :roles =>:pall215 do 
+  run "cd /home/mass/student/k128576/cassandra/ &&  /home/mass/student/k128576/cassandra/apache-cassandra-2.0.4/bin/cassandra -f"
+end
+
 task :start_cassandra10, :roles =>:pall21 do 
   run "cd /home/mass/student/k128576/cassandra/ &&  /home/mass/student/k128576/cassandra/apache-cassandra-2.0.4/bin/cassandra -f"
 end
 
-task :one_bbs10, :roles =>:pall21 do 
+
+
+task :one_bbs30, :roles =>:pall23 do 
+ run "/usr/local/apache-maven/apache-maven-3.0.5/bin/mvn -f  /home/mass/student/k128576/cassandra/bulletinboard/pom.xml exec:java -Dexec.mainClass='jp.ac.u_ryukyu.ie.cr.shoshi.jungle.bbs.App' -Dexec.args='-one' "
+end
+
+task :one_bbs25, :roles =>:pall225 do 
  run "/usr/local/apache-maven/apache-maven-3.0.5/bin/mvn -f  /home/mass/student/k128576/cassandra/bulletinboard/pom.xml exec:java -Dexec.mainClass='jp.ac.u_ryukyu.ie.cr.shoshi.jungle.bbs.App' -Dexec.args='-one' "
 end
 
@@ -81,30 +106,33 @@
  run "/usr/local/apache-maven/apache-maven-3.0.5/bin/mvn -f  /home/mass/student/k128576/cassandra/bulletinboard/pom.xml exec:java -Dexec.mainClass='jp.ac.u_ryukyu.ie.cr.shoshi.jungle.bbs.App' -Dexec.args='-one' "
 end
 
-task :one_bbs30, :roles =>:pall23 do 
+task :one_bbs15, :roles =>:pall215 do 
  run "/usr/local/apache-maven/apache-maven-3.0.5/bin/mvn -f  /home/mass/student/k128576/cassandra/bulletinboard/pom.xml exec:java -Dexec.mainClass='jp.ac.u_ryukyu.ie.cr.shoshi.jungle.bbs.App' -Dexec.args='-one' "
 end
 
-task :all_bbs10, :roles =>:pall21 do 
- run "/usr/local/apache-maven/apache-maven-3.0.5/bin/mvn -f  /home/mass/student/k128576/cassandra/bulletinboard/pom.xml exec:java -Dexec.mainClass='jp.ac.u_ryukyu.ie.cr.shoshi.jungle.bbs.App' -Dexec.args='-all -rep 10' "
+task :one_bbs10, :roles =>:pall21 do 
+ run "/usr/local/apache-maven/apache-maven-3.0.5/bin/mvn -f  /home/mass/student/k128576/cassandra/bulletinboard/pom.xml exec:java -Dexec.mainClass='jp.ac.u_ryukyu.ie.cr.shoshi.jungle.bbs.App' -Dexec.args='-one' "
 end
 
-task :all_bbs20, :roles =>:pall22 do 
- run "/usr/local/apache-maven/apache-maven-3.0.5/bin/mvn -f  /home/mass/student/k128576/cassandra/bulletinboard/pom.xml exec:java -Dexec.mainClass='jp.ac.u_ryukyu.ie.cr.shoshi.jungle.bbs.App' -Dexec.args='-all -rep 20' "
-end
 
-task :all_bbs30, :roles =>:pall23 do 
- run "/usr/local/apache-maven/apache-maven-3.0.5/bin/mvn -f  /home/mass/student/k128576/cassandra/bulletinboard/pom.xml exec:java -Dexec.mainClass='jp.ac.u_ryukyu.ie.cr.shoshi.jungle.bbs.App' -Dexec.args='-all -rep 30' "
-end
+
 
 task :quorum_bbs30, :roles =>:pall23 do 
  run "/usr/local/apache-maven/apache-maven-3.0.5/bin/mvn -f  /home/mass/student/k128576/cassandra/bulletinboard/pom.xml exec:java -Dexec.mainClass='jp.ac.u_ryukyu.ie.cr.shoshi.jungle.bbs.App' -Dexec.args='-quorum -rep 5' "
 end
 
+task :quorum_bbs25, :roles =>:pall225 do 
+ run "/usr/local/apache-maven/apache-maven-3.0.5/bin/mvn -f  /home/mass/student/k128576/cassandra/bulletinboard/pom.xml exec:java -Dexec.mainClass='jp.ac.u_ryukyu.ie.cr.shoshi.jungle.bbs.App' -Dexec.args='-quorum -rep 5' "
+end
+
 task :quorum_bbs20, :roles =>:pall22 do 
  run "/usr/local/apache-maven/apache-maven-3.0.5/bin/mvn -f  /home/mass/student/k128576/cassandra/bulletinboard/pom.xml exec:java -Dexec.mainClass='jp.ac.u_ryukyu.ie.cr.shoshi.jungle.bbs.App' -Dexec.args='-quorum -rep 5' "
 end
 
+task :quorum_bbs15, :roles =>:pall215 do 
+ run "/usr/local/apache-maven/apache-maven-3.0.5/bin/mvn -f  /home/mass/student/k128576/cassandra/bulletinboard/pom.xml exec:java -Dexec.mainClass='jp.ac.u_ryukyu.ie.cr.shoshi.jungle.bbs.App' -Dexec.args='-quorum -rep 5' "
+end
+
 task :quorum_bbs10, :roles =>:pall21 do 
  run "/usr/local/apache-maven/apache-maven-3.0.5/bin/mvn -f  /home/mass/student/k128576/cassandra/bulletinboard/pom.xml exec:java -Dexec.mainClass='jp.ac.u_ryukyu.ie.cr.shoshi.jungle.bbs.App' -Dexec.args='-quorum -rep 5' "
 end
--- a/distributed/weighttp/weighttp_jungle.sh	Fri Jan 31 00:51:16 2014 +0900
+++ b/distributed/weighttp/weighttp_jungle.sh	Fri Jan 31 04:51:45 2014 +0900
@@ -6,8 +6,13 @@
     echo "[mode]: read write"
     exit 1
 }
-if [ $# -lt 2 ]; then
-	usage
+loopnum=1
+if [ $# -lt 2 ]
+    then
+    usage
+elif [ $# -gt 2 ]
+    then
+    loopnum=$3
 fi
 readFlag=1
 if [ $2 == "read" ]
@@ -20,18 +25,15 @@
     usage
 fi
 
-#requestNum=50000
-#concurrentNum=200
-#threadNum=2
-
-requestNum=5000
-concurrentNum=1
-threadNum=1
+requestNum=50000
+concurrentNum=200
+threadNum=2
 
 echo "number of loop : $loopnum"
 echo "requests:$requestNum concurrent:$concurrentNum thread:$threadNum"
 nodesFile=/home/mass/share/student/k128576/jungle_benchmark/distributed/weighttp/nodes30.txt
 nodenum=$1
+
 for i in `seq 1 $loopnum`
   do
   echo "current count = $i"
@@ -41,9 +43,9 @@
   fi
   echo "nodes = $nodenum"
   if [ $readFlag -eq 0 ]
+  then
       echo "write benchmark"
       qsub -e  /home/mass/share/student/k128576/jungle_benchmark/distributed/weighttp/jungle_log/error_log/  -o /home/mass/share/student/k128576/jungle_benchmark/distributed/weighttp/jungle_log/log/ -v nodesFile=$nodesFile,requests=$requestNum,schedule=`expr $(date +%s) + 10`,concurrent=$concurrentNum,thread=$threadNum -l nodes=$nodenum:ppn=4 /home/mass/share/student/k128576/jungle_benchmark/distributed/weighttp/weighttp_write_jungle.sh 
-      then
   else
       echo "read benchmark"
       qsub -e  /home/mass/share/student/k128576/jungle_benchmark/distributed/weighttp/jungle_log/error_log/  -o /home/mass/share/student/k128576/jungle_benchmark/distributed/weighttp/jungle_log/log/ -v nodesFile=$nodesFile,requests=$requestNum,schedule=`expr $(date +%s) + 10`,concurrent=$concurrentNum,thread=$threadNum -l nodes=$nodenum:ppn=4 /home/mass/share/student/k128576/jungle_benchmark/distributed/weighttp/weighttp_read_jungle.sh 
@@ -55,4 +57,5 @@
     sleep 5
   done 
   echo "job complete"
+  sleep 10
 done