Mercurial > hg > Members > nobuyasu > jungle_benchmark

--- a/distributed/get_node_list.rb	Wed Jan 29 16:22:02 2014 +0900
+++ b/distributed/get_node_list.rb	Thu Jan 30 21:48:09 2014 +0900
@@ -1,12 +1,9 @@
 module GetNodeList
   def get_node_list(filename)
-    f = open(filename,"r")
-    nodes = []
-    f.each { |line|
-      if /.+jp/ =~ line then
-        nodes.push(line.chomp)
-      end
-    }
+    nodesArray = File.open(filename,"r") do |file|
+	file.readlines.select{|line| line =~ /.+jp/ }
+    end
+    nodes = nodesArray.collect{|node| node.chomp()}
     hostname = `hostname`
     mynum = 0
     if /[0-9]+/ =~ hostname then
--- a/distributed/test_cluster_get.rb	Wed Jan 29 16:22:02 2014 +0900
+++ b/distributed/test_cluster_get.rb	Thu Jan 30 21:48:09 2014 +0900
@@ -6,14 +6,21 @@
 require '/home/mass/share/student/k128576/jungle_benchmark/distributed/get_node_list.rb'
 include GetNodeList

-nodelist = get_node_list("/home/mass/share/student/k128576/jungle_benchmark/distributed/nodes.txt")
+filename = "/home/mass/share/student/k128576/jungle_benchmark/distributed/nodes.txt"
+if ARGV.size() > 0 then
+  filename = ARGV[0]
+end
+nodelist = get_node_list(filename)

+count=0
 nodelist.each { |nodename|
     puts "connect " + nodename
     begin
       ret = Net::HTTP.get(nodename,'/showBoardMessage?bname=hello%20world',8080)
+      count = count + 1
     rescue Errno::ECONNREFUSED
       puts "Failed : get request to " + nodename
     end
 }
+puts "connect success : " + count.to_s
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/distributed/weighttp/calc_sd_from_log.rb	Thu Jan 30 21:48:09 2014 +0900
@@ -0,0 +1,48 @@
+#!/usr/local/bin/ruby
+include Math
+
+if ARGV.size() < 1 then
+  puts "Usage:ruby parse_log.rb filename"
+  exit
+end
+
+def get_log_time(filename)
+  logArray = File.open(filename, "r") do |file|
+     file.readlines.select{|line| !(line =~ /mass/) }
+  end
+  totalFailed = 0
+  totalTime = 0
+  timeArray = []
+  logArray.each {|line|
+    tokens = line.split(' ')
+    float = sprintf( "%.0f",tokens[2].to_i)
+    time = tokens[0]. + "." + float.to_s
+    timeArray.push(time.to_f)
+    totalTime = totalTime + time.to_f
+    totalFailed = totalFailed + tokens[4].to_i
+  }
+  avarage = totalTime / timeArray.size()
+  avarage = sprintf("%.3f", avarage).to_f
+  totalN = 0
+  puts "avarage: "+ avarage.to_s
+  timeArray.each {|time|
+    totalN = totalN + (time.to_f - avarage)
+    totalN = sprintf("%.3f", totalN).to_f
+    puts "time: "+ time.to_s
+    puts "time - avarage :" + sprintf("%.3f",(time - avarage).to_s)
+    puts "totalN: "+totalN.to_s
+  }
+  variance = totalN / timeArray.size()
+  variance = sprintf("%.3f", variance).to_f
+  puts "variance: " + variance.to_s
+  sigma = Math.sqrt(variance)
+  puts sigma
+  if totalFailed != 0 then puts "Failed " + totalFailed.to_s end
+end
+
+ARGV.each {|filename|
+ get_log_time(filename)
+}
+
+
+
--- a/distributed/weighttp/capistrano_weighttp.rb	Wed Jan 29 16:22:02 2014 +0900
+++ b/distributed/weighttp/capistrano_weighttp.rb	Thu Jan 30 21:48:09 2014 +0900
@@ -1,22 +1,19 @@
 set :user, "mass"


-# pall10 , 39 ~ 48
-#role :pall10, "133.13.62.39","133.13.62.40","133.13.62.41","133.13.62.42","133.13.62.43","133.13.62.44","133.13.62.45","133.13.62.46","133.13.62.47","133.13.62.48"
-#
-#role :pall20, "133.13.62.34","133.13.62.35","133.13.62.36","133.13.62.37","133.13.62.38","133.13.62.29","133.13.62.30","133.13.62.31","133.13.62.32","133.13.62.33"
-#
-#role :pall30, "133.13.62.19","133.13.62.20","133.13.62.21","133.13.62.22","133.13.62.23","133.13.62.24","133.13.62.25","133.13.62.26","133.13.62.27","133.13.62.28"
-#
+role :pall13,"133.13.62.9","133.13.62.10","133.13.62.11","133.13.62.12","133.13.62.13","133.13.62.14","133.13.62.15","133.13.62.16","133.13.62.17","133.13.62.18","133.13.62.19","133.13.62.2","133.13.62.3","133.13.62.4","133.13.62.5","133.13.62.6","133.13.62.7","133.13.62.8","133.13.62.101", "133.13.62.102", "133.13.62.103","133.13.62.104","133.13.62.105","133.13.62.106","133.13.62.107","133.13.62.108","133.13.62.109","133.13.62.110","133.13.62.111","133.13.62.112"
+
+role :crpall, "133.13.62.101", "133.13.62.102", "133.13.62.103","133.13.62.104","133.13.62.105","133.13.62.106","133.13.62.107","133.13.62.108","133.13.62.109","133.13.62.110","133.13.62.111","133.13.62.112"
+


-role :pall40, "133.13.62.9","133.13.62.10","133.13.62.11","133.13.62.12","133.13.62.13","133.13.62.14","133.13.62.15","133.13.62.16","133.13.62.17","133.13.62.18"
+# Server for BBS
+role :pall23, "133.13.62.39","133.13.62.40","133.13.62.41","133.13.62.42","133.13.62.43","133.13.62.44","133.13.62.45","133.13.62.46","133.13.62.47", "133.13.62.48", "133.13.62.29","133.13.62.30","133.13.62.31","133.13.62.32","133.13.62.33","133.13.62.34","133.13.62.35","133.13.62.36","133.13.62.37","133.13.62.38","133.13.62.1","133.13.62.20","133.13.62.21","133.13.62.22","133.13.62.23","133.13.62.24","133.13.62.25","133.13.62.26","133.13.62.27","133.13.62.28"

-# Care plz, 101 and 102 is CR cluster.
-role :pall50, "133.13.62.1","133.13.62.2","133.13.62.3","133.13.62.4","133.13.62.5","133.13.62.6","133.13.62.7","133.13.62.8","133.13.62.101", "133.13.62.102"
+role :pall22, "133.13.62.39","133.13.62.40","133.13.62.41","133.13.62.42","133.13.62.43","133.13.62.44","133.13.62.45","133.13.62.46","133.13.62.47", "133.13.62.48", "133.13.62.29","133.13.62.30","133.13.62.31","133.13.62.32","133.13.62.33","133.13.62.34","133.13.62.35","133.13.62.36","133.13.62.37","133.13.62.38"

-#*** CR ***
-role :pall08, "133.13.62.103","133.13.62.104","133.13.62.105","133.13.62.106","133.13.62.107","133.13.62.108","133.13.62.109","133.13.62.110","133.13.62.111","133.13.62.112"
+role :pall21, "133.13.62.39","133.13.62.40","133.13.62.41","133.13.62.42","133.13.62.43","133.13.62.44","133.13.62.45","133.13.62.46","133.13.62.47", "133.13.62.48"
+


@@ -32,53 +29,6 @@
   run "hostname"
 end

-task :rsync_project do
-  run "rsync -av /home/mass/share/student/k128576/hg/jungle-network  /home/mass/student/k128576/hg/"
-end
-
-task :clean_project do
-  run "/usr/local/apache-maven/apache-maven-3.0.5/bin/mvn -f /home/mass/student/k128576/hg/jungle-network/pom.xml clean"
-end
-
-task :compile_project do
-  run "/usr/local/apache-maven/apache-maven-3.0.5/bin/mvn -f /home/mass/student/k128576/hg/jungle-network/pom.xml compile"
-end
-
-task :run_jungle do
-  run "export MAVEN_OPTS='-Xmx6g -Xms4g'  && /usr/local/apache-maven/apache-maven-3.0.5/bin/mvn -f /home/mass/student/k128576/hg/jungle-network/pom.xml exec:java -Dexec.mainClass='app.bbs.DistributeApp' -Dexec.args='-level fatal -host bldsv12.cr.ie.u-ryukyu.ac.jp -port 10000 -p 20000 -bp 8080' "
-end
-
-task :cp_cassandra2 do
-  run "cp -r /home/mass/share/student/k128576/cassandra/apache-cassandra-2.0.4 /home/mass/student/k128576/cassandra"
-end
-
-task :start_cassandra do
-  run "cd /home/mass/student/k128576/cassandra/ &&  /home/mass/student/k128576/cassandra/apache-cassandra-2.0.4/bin/cassandra -f"
-end
-
-task :rsync_bbs do
-  run "rsync -av /home/mass/share/student/k128576/hg/bulletinboard /home/mass/student/k128576/cassandra/"
-end
-
-task :clean_bbs do
-  run "/usr/local/apache-maven/apache-maven-3.0.5/bin/mvn -f /home/mass/student/k128576/cassandra/bulletinboard/pom.xml clean"
-end
-
-task :compile_bbs do
- run "/usr/local/apache-maven/apache-maven-3.0.5/bin/mvn -f  /home/mass/student/k128576/cassandra/bulletinboard/pom.xml compile"
-end
-
-task :one_bbs do
- run "/usr/local/apache-maven/apache-maven-3.0.5/bin/mvn -f  /home/mass/student/k128576/cassandra/bulletinboard/pom.xml exec:java -Dexec.mainClass='jp.ac.u_ryukyu.ie.cr.shoshi.jungle.bbs.App' -Dexec.args='-one' "
-end
-
-task :quorum_bbs do
- run "/usr/local/apache-maven/apache-maven-3.0.5/bin/mvn -f  /home/mass/student/k128576/cassandra/bulletinboard/pom.xml exec:java -Dexec.mainClass='jp.ac.u_ryukyu.ie.cr.shoshi.jungle.bbs.App' -Dexec.args='-quorum' "
-end
-
-task :quorum_bbs5 do
- run "/usr/local/apache-maven/apache-maven-3.0.5/bin/mvn -f  /home/mass/student/k128576/cassandra/bulletinboard/pom.xml exec:java -Dexec.mainClass='jp.ac.u_ryukyu.ie.cr.shoshi.jungle.bbs.App' -Dexec.args='-quorum -rep 5' "
-end

 task :delete_var do
  run "rm -rf /home/mass/student/k128576/cassandra/var"
@@ -88,7 +38,121 @@
    run "ps aux |grep maven |grep -v grep |awk '{print $2}'| xargs kill -9"
 end

-task :get_host do
-   run "sh /home/mass/share/student/k128576/jungle_benchmark/distributed/weighttp/test.sh "
+
+task :rm_log do
+   run "rm /home/mass/student/k128576/hg/jungle-network/log/*"
+end
+
+task :hostname01, :roles =>:pall13 do
+  run "hostname"
+end
+
+task :run_jungle30, :roles =>:pall23 do
+  run "export MAVEN_OPTS='-Xmx6g -Xms4g'  && /usr/local/apache-maven/apache-maven-3.0.5/bin/mvn -f /home/mass/student/k128576/hg/jungle-network/pom.xml exec:java -Dexec.mainClass='app.bbs.DistributeApp' -Dexec.args='-level fatal -host bldsv12.cr.ie.u-ryukyu.ac.jp -port 10000 -p 20000 -bp 8080' "
+end
+
+task :run_jungle20, :roles =>:pall22 do
+  run "export MAVEN_OPTS='-Xmx6g -Xms4g'  && /usr/local/apache-maven/apache-maven-3.0.5/bin/mvn -f /home/mass/student/k128576/hg/jungle-network/pom.xml exec:java -Dexec.mainClass='app.bbs.DistributeApp' -Dexec.args='-level fatal -host bldsv12.cr.ie.u-ryukyu.ac.jp -port 10000 -p 20000 -bp 8080' "
+end
+
+task :run_jungle10, :roles =>:pall21 do
+  run "export MAVEN_OPTS='-Xmx6g -Xms4g'  && /usr/local/apache-maven/apache-maven-3.0.5/bin/mvn -f /home/mass/student/k128576/hg/jungle-network/pom.xml exec:java -Dexec.mainClass='app.bbs.DistributeApp' -Dexec.args='-level fatal -host bldsv12.cr.ie.u-ryukyu.ac.jp -port 10000 -p 20000 -bp 8080' "
+end
+
+
+
+task :start_cassandra30, :roles =>:pall23 do
+  run "cd /home/mass/student/k128576/cassandra/ &&  /home/mass/student/k128576/cassandra/apache-cassandra-2.0.4/bin/cassandra -f"
+end
+
+task :start_cassandra20, :roles =>:pall22 do
+  run "cd /home/mass/student/k128576/cassandra/ &&  /home/mass/student/k128576/cassandra/apache-cassandra-2.0.4/bin/cassandra -f"
+end
+
+task :start_cassandra10, :roles =>:pall21 do
+  run "cd /home/mass/student/k128576/cassandra/ &&  /home/mass/student/k128576/cassandra/apache-cassandra-2.0.4/bin/cassandra -f"
+end
+
+task :one_bbs10, :roles =>:pall21 do
+ run "/usr/local/apache-maven/apache-maven-3.0.5/bin/mvn -f  /home/mass/student/k128576/cassandra/bulletinboard/pom.xml exec:java -Dexec.mainClass='jp.ac.u_ryukyu.ie.cr.shoshi.jungle.bbs.App' -Dexec.args='-one' "
+end
+
+task :one_bbs20, :roles =>:pall22 do
+ run "/usr/local/apache-maven/apache-maven-3.0.5/bin/mvn -f  /home/mass/student/k128576/cassandra/bulletinboard/pom.xml exec:java -Dexec.mainClass='jp.ac.u_ryukyu.ie.cr.shoshi.jungle.bbs.App' -Dexec.args='-one' "
+end
+
+task :one_bbs30, :roles =>:pall23 do
+ run "/usr/local/apache-maven/apache-maven-3.0.5/bin/mvn -f  /home/mass/student/k128576/cassandra/bulletinboard/pom.xml exec:java -Dexec.mainClass='jp.ac.u_ryukyu.ie.cr.shoshi.jungle.bbs.App' -Dexec.args='-one' "
+end
+
+task :all_bbs10, :roles =>:pall21 do
+ run "/usr/local/apache-maven/apache-maven-3.0.5/bin/mvn -f  /home/mass/student/k128576/cassandra/bulletinboard/pom.xml exec:java -Dexec.mainClass='jp.ac.u_ryukyu.ie.cr.shoshi.jungle.bbs.App' -Dexec.args='-all -rep 10' "
+end
+
+task :all_bbs20, :roles =>:pall22 do
+ run "/usr/local/apache-maven/apache-maven-3.0.5/bin/mvn -f  /home/mass/student/k128576/cassandra/bulletinboard/pom.xml exec:java -Dexec.mainClass='jp.ac.u_ryukyu.ie.cr.shoshi.jungle.bbs.App' -Dexec.args='-all -rep 20' "
+end
+
+task :all_bbs30, :roles =>:pall23 do
+ run "/usr/local/apache-maven/apache-maven-3.0.5/bin/mvn -f  /home/mass/student/k128576/cassandra/bulletinboard/pom.xml exec:java -Dexec.mainClass='jp.ac.u_ryukyu.ie.cr.shoshi.jungle.bbs.App' -Dexec.args='-all -rep 30' "
 end

+task :quorum_bbs30, :roles =>:pall23 do
+ run "/usr/local/apache-maven/apache-maven-3.0.5/bin/mvn -f  /home/mass/student/k128576/cassandra/bulletinboard/pom.xml exec:java -Dexec.mainClass='jp.ac.u_ryukyu.ie.cr.shoshi.jungle.bbs.App' -Dexec.args='-quorum -rep 5' "
+end
+
+task :quorum_bbs20, :roles =>:pall22 do
+ run "/usr/local/apache-maven/apache-maven-3.0.5/bin/mvn -f  /home/mass/student/k128576/cassandra/bulletinboard/pom.xml exec:java -Dexec.mainClass='jp.ac.u_ryukyu.ie.cr.shoshi.jungle.bbs.App' -Dexec.args='-quorum -rep 5' "
+end
+
+task :quorum_bbs10, :roles =>:pall21 do
+ run "/usr/local/apache-maven/apache-maven-3.0.5/bin/mvn -f  /home/mass/student/k128576/cassandra/bulletinboard/pom.xml exec:java -Dexec.mainClass='jp.ac.u_ryukyu.ie.cr.shoshi.jungle.bbs.App' -Dexec.args='-quorum -rep 5' "
+end
+
+
+
+
+task :pbs_start30, :roles =>:pall13 do
+  run "sudo /etc/init.d/torque start"
+end
+
+task :pbs_stop30, :roles =>:pall13 do
+  run "sudo /etc/init.d/torque stop"
+end
+
+task :pbs_restart30, :roles =>:pall13 do
+  run "sudo /etc/init.d/torque restart"
+end
+
+task :pbs_stop do
+  run "sudo /etc/init.d/torque stop"
+end
+
+task :pbs_start do
+  run "sudo /etc/init.d/torque start"
+end
+
+task :change_server_name, :roles =>:crpall do
+  run "sudo mv /var/spool/torque/server_name /home/mass/student/k128576/"
+  run "sudo cp /home/mass/share/student/k128576/jungle_benchmark/distributed/weighttp/torque_conf/server_name /var/spool/torque/"
+end
+
+task :change_config, :roles =>:crpall do
+  run "sudo mv /var/spool/torque/mom_priv/config /home/mass/student/k128576/"
+  run "sudo cp /home/mass/share/student/k128576/jungle_benchmark/distributed/weighttp/torque_conf/config  /var/spool/torque/mom_priv/"
+end
+
+task :cat_config, :roles =>:crpall do
+  run "sudo cat /var/spool/torque/mom_priv/config"
+end
+
+task :default_config, :roles =>:crpall do
+  run "sudo rm /var/spool/torque/mom_priv/config"
+  run "sudo cp /home/mass/share/student/k128576/jungle_benchmark/distributed/weighttp/torque_conf/CR_torque/config  /var/spool/torque/mom_priv/"
+end
+
+task :default_server_name, :roles =>:crpall do
+  run "sudo rm /var/spool/torque/server_name"
+  run "sudo cp /home/mass/share/student/k128576/jungle_benchmark/distributed/weighttp/torque_conf/CR_torque/server_name /var/spool/torque/server_name"
+end
+
--- a/distributed/weighttp/loop_qsub.sh	Wed Jan 29 16:22:02 2014 +0900
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,24 +0,0 @@
-#!/bin/sh
-
-nodesFile=/home/mass/share/student/k128576/jungle_benchmark/distributed/weighttp/nodes30.txt
-
-if [ $# -lt 1 ]; then
-  echo "Please set node number"
-  echo "Usage:./loop_qsub.sh nodenum"
-  exit 1
-fi
-
-nodenum=$1
-if [ $nodenum -lt 10 ]
-then
-   nodenum=0$nodenum
- fi
- echo "nodes = $nodenum"
- qsub -e  /home/mass/share/student/k128576/jungle_benchmark/distributed/weighttp/error_log/  -o /home/mass/share/student/k128576/jungle_benchmark/distributed/weighttp/log/ -v nodesFile=$nodesFile,requests=100000,schedule=`expr $(date +%s) + 5`,concurrent=200,thread=2 -l nodes=$nodenum:ppn=4 /home/mass/share/student/k128576/jungle_benchmark/distributed/weighttp/weighttp_write_jungle.sh
- while [ "x`qstat`" != "x" ]
- do
-  echo "waiting for job to complete"
-  sleep 5
- done
-echo "job complete"
-
--- a/distributed/weighttp/stress_weighttp.sh	Wed Jan 29 16:22:02 2014 +0900
+++ b/distributed/weighttp/stress_weighttp.sh	Thu Jan 30 21:48:09 2014 +0900
@@ -21,8 +21,8 @@
     echo ${nodeArray[$num]}
 }

+uri=':8080/showBoardMessage?bname=hello%20world'
 nodeUrl=`get_node_name < $nodesFile $myNum`
-uri=":8080/editMessageUseGet?bname=hello%20world&uuid=0&author=oshiro&msg=write_test&key="
 now=$(date +%s)
 sleepTime=`expr $schedule - $now`
 sleep $sleepTime
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/distributed/weighttp/weighttp_jungle.sh	Thu Jan 30 21:48:09 2014 +0900
@@ -0,0 +1,46 @@
+#!/bin/sh
+
+function usage() {
+  echo "Please set number of node and mode"
+  echo "Usage:./loop_qsub.sh nodenum [mode]"
+  echo "[mode]: read write"
+  exit 1
+}
+if [ $# -lt 2 ]; then
+  usage
+fi
+readFlag=1
+if [ $2 == "read" ]
+then
+   readFlag=1
+elif [ $2 == "write" ]
+then
+   readFlag=0
+else
+   usage
+fi
+
+nodesFile=/home/mass/share/student/k128576/jungle_benchmark/distributed/weighttp/nodes30.txt
+
+nodenum=$1
+if [ $nodenum -lt 10 ]
+then
+  nodenum=0$nodenum
+fi
+echo "nodes = $nodenum"
+if [ $readFlag -eq 0 ]
+  echo "write benchmark"
+  qsub -e  /home/mass/share/student/k128576/jungle_benchmark/distributed/weighttp/jungle_log/error_log/  -o /home/mass/share/student/k128576/jungle_benchmark/distributed/weighttp/jungle_log/log/ -v nodesFile=$nodesFile,requests=50000,schedule=`expr $(date +%s) + 10`,concurrent=200,thread=2 -l nodes=$nodenum:ppn=4 /home/mass/share/student/k128576/jungle_benchmark/distributed/weighttp/weighttp_write_jungle.sh
+then
+else
+  echo "read benchmark"
+  qsub -e  /home/mass/share/student/k128576/jungle_benchmark/distributed/weighttp/jungle_log/error_log/  -o /home/mass/share/student/k128576/jungle_benchmark/distributed/weighttp/jungle_log/log/ -v nodesFile=$nodesFile,requests=50000,schedule=`expr $(date +%s) + 10`,concurrent=200,thread=2 -l nodes=$nodenum:ppn=4 /home/mass/share/student/k128576/jungle_benchmark/distributed/weighttp/weighttp_read_jungle.sh
+fi
+
+while [ "x`qstat`" != "x" ]
+do
+  echo "waiting for job to complete"
+  sleep 5
+done
+echo "job complete"
+
--- a/distributed/weighttp/weighttp_write_jungle.sh	Wed Jan 29 16:22:02 2014 +0900
+++ b/distributed/weighttp/weighttp_write_jungle.sh	Thu Jan 30 21:48:09 2014 +0900
@@ -4,7 +4,7 @@
 #PBS -N write_jungle
 #PBS -l walltime=00:05:00

-command=/home/mass/share/student/k128576/jungle_benchmark/distributed/weighttp/stress_weighttp.sh
+command=/home/mass/share/student/k128576/jungle_benchmark/distributed/weighttp/stress_write_jungle.sh

 function run() {
  i=0