# HG changeset patch # User Nobuyasu # Date 1391086089 -32400 # Node ID d6d3b7180e6d6680ceb9c66d464e4f7603addf4b # Parent a972a5814b0be23948a2722d10a503c369373ba7 Added calc_sd_from_log.rb diff -r a972a5814b0b -r d6d3b7180e6d distributed/get_node_list.rb --- a/distributed/get_node_list.rb Wed Jan 29 16:22:02 2014 +0900 +++ b/distributed/get_node_list.rb Thu Jan 30 21:48:09 2014 +0900 @@ -1,12 +1,9 @@ module GetNodeList def get_node_list(filename) - f = open(filename,"r") - nodes = [] - f.each { |line| - if /.+jp/ =~ line then - nodes.push(line.chomp) - end - } + nodesArray = File.open(filename,"r") do |file| + file.readlines.select{|line| line =~ /.+jp/ } + end + nodes = nodesArray.collect{|node| node.chomp()} hostname = `hostname` mynum = 0 if /[0-9]+/ =~ hostname then diff -r a972a5814b0b -r d6d3b7180e6d distributed/test_cluster_get.rb --- a/distributed/test_cluster_get.rb Wed Jan 29 16:22:02 2014 +0900 +++ b/distributed/test_cluster_get.rb Thu Jan 30 21:48:09 2014 +0900 @@ -6,14 +6,21 @@ require '/home/mass/share/student/k128576/jungle_benchmark/distributed/get_node_list.rb' include GetNodeList -nodelist = get_node_list("/home/mass/share/student/k128576/jungle_benchmark/distributed/nodes.txt") +filename = "/home/mass/share/student/k128576/jungle_benchmark/distributed/nodes.txt" +if ARGV.size() > 0 then + filename = ARGV[0] +end +nodelist = get_node_list(filename) +count=0 nodelist.each { |nodename| puts "connect " + nodename begin ret = Net::HTTP.get(nodename,'/showBoardMessage?bname=hello%20world',8080) + count = count + 1 rescue Errno::ECONNREFUSED puts "Failed : get request to " + nodename end } +puts "connect success : " + count.to_s diff -r a972a5814b0b -r d6d3b7180e6d distributed/weighttp/calc_sd_from_log.rb --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/distributed/weighttp/calc_sd_from_log.rb Thu Jan 30 21:48:09 2014 +0900 @@ -0,0 +1,48 @@ +#!/usr/local/bin/ruby +include Math + +if ARGV.size() < 1 then + puts "Usage:ruby parse_log.rb filename" + exit +end + +def get_log_time(filename) + logArray = File.open(filename, "r") do |file| + file.readlines.select{|line| !(line =~ /mass/) } + end + totalFailed = 0 + totalTime = 0 + timeArray = [] + logArray.each {|line| + tokens = line.split(' ') + float = sprintf( "%.0f",tokens[2].to_i) + time = tokens[0]. + "." + float.to_s + timeArray.push(time.to_f) + totalTime = totalTime + time.to_f + totalFailed = totalFailed + tokens[4].to_i + } + avarage = totalTime / timeArray.size() + avarage = sprintf("%.3f", avarage).to_f + totalN = 0 + puts "avarage: "+ avarage.to_s + timeArray.each {|time| + totalN = totalN + (time.to_f - avarage) + totalN = sprintf("%.3f", totalN).to_f + puts "time: "+ time.to_s + puts "time - avarage :" + sprintf("%.3f",(time - avarage).to_s) + puts "totalN: "+totalN.to_s + } + variance = totalN / timeArray.size() + variance = sprintf("%.3f", variance).to_f + puts "variance: " + variance.to_s + sigma = Math.sqrt(variance) + puts sigma + if totalFailed != 0 then puts "Failed " + totalFailed.to_s end +end + +ARGV.each {|filename| + get_log_time(filename) +} + + + diff -r a972a5814b0b -r d6d3b7180e6d distributed/weighttp/capistrano_weighttp.rb --- a/distributed/weighttp/capistrano_weighttp.rb Wed Jan 29 16:22:02 2014 +0900 +++ b/distributed/weighttp/capistrano_weighttp.rb Thu Jan 30 21:48:09 2014 +0900 @@ -1,22 +1,19 @@ set :user, "mass" -# pall10 , 39 ~ 48 -#role :pall10, "133.13.62.39","133.13.62.40","133.13.62.41","133.13.62.42","133.13.62.43","133.13.62.44","133.13.62.45","133.13.62.46","133.13.62.47","133.13.62.48" -# -#role :pall20, "133.13.62.34","133.13.62.35","133.13.62.36","133.13.62.37","133.13.62.38","133.13.62.29","133.13.62.30","133.13.62.31","133.13.62.32","133.13.62.33" -# -#role :pall30, "133.13.62.19","133.13.62.20","133.13.62.21","133.13.62.22","133.13.62.23","133.13.62.24","133.13.62.25","133.13.62.26","133.13.62.27","133.13.62.28" -# +role :pall13,"133.13.62.9","133.13.62.10","133.13.62.11","133.13.62.12","133.13.62.13","133.13.62.14","133.13.62.15","133.13.62.16","133.13.62.17","133.13.62.18","133.13.62.19","133.13.62.2","133.13.62.3","133.13.62.4","133.13.62.5","133.13.62.6","133.13.62.7","133.13.62.8","133.13.62.101", "133.13.62.102", "133.13.62.103","133.13.62.104","133.13.62.105","133.13.62.106","133.13.62.107","133.13.62.108","133.13.62.109","133.13.62.110","133.13.62.111","133.13.62.112" + +role :crpall, "133.13.62.101", "133.13.62.102", "133.13.62.103","133.13.62.104","133.13.62.105","133.13.62.106","133.13.62.107","133.13.62.108","133.13.62.109","133.13.62.110","133.13.62.111","133.13.62.112" + -role :pall40, "133.13.62.9","133.13.62.10","133.13.62.11","133.13.62.12","133.13.62.13","133.13.62.14","133.13.62.15","133.13.62.16","133.13.62.17","133.13.62.18" +# Server for BBS +role :pall23, "133.13.62.39","133.13.62.40","133.13.62.41","133.13.62.42","133.13.62.43","133.13.62.44","133.13.62.45","133.13.62.46","133.13.62.47", "133.13.62.48", "133.13.62.29","133.13.62.30","133.13.62.31","133.13.62.32","133.13.62.33","133.13.62.34","133.13.62.35","133.13.62.36","133.13.62.37","133.13.62.38","133.13.62.1","133.13.62.20","133.13.62.21","133.13.62.22","133.13.62.23","133.13.62.24","133.13.62.25","133.13.62.26","133.13.62.27","133.13.62.28" -# Care plz, 101 and 102 is CR cluster. -role :pall50, "133.13.62.1","133.13.62.2","133.13.62.3","133.13.62.4","133.13.62.5","133.13.62.6","133.13.62.7","133.13.62.8","133.13.62.101", "133.13.62.102" +role :pall22, "133.13.62.39","133.13.62.40","133.13.62.41","133.13.62.42","133.13.62.43","133.13.62.44","133.13.62.45","133.13.62.46","133.13.62.47", "133.13.62.48", "133.13.62.29","133.13.62.30","133.13.62.31","133.13.62.32","133.13.62.33","133.13.62.34","133.13.62.35","133.13.62.36","133.13.62.37","133.13.62.38" -#*** CR *** -role :pall08, "133.13.62.103","133.13.62.104","133.13.62.105","133.13.62.106","133.13.62.107","133.13.62.108","133.13.62.109","133.13.62.110","133.13.62.111","133.13.62.112" +role :pall21, "133.13.62.39","133.13.62.40","133.13.62.41","133.13.62.42","133.13.62.43","133.13.62.44","133.13.62.45","133.13.62.46","133.13.62.47", "133.13.62.48" + @@ -32,53 +29,6 @@ run "hostname" end -task :rsync_project do - run "rsync -av /home/mass/share/student/k128576/hg/jungle-network /home/mass/student/k128576/hg/" -end - -task :clean_project do - run "/usr/local/apache-maven/apache-maven-3.0.5/bin/mvn -f /home/mass/student/k128576/hg/jungle-network/pom.xml clean" -end - -task :compile_project do - run "/usr/local/apache-maven/apache-maven-3.0.5/bin/mvn -f /home/mass/student/k128576/hg/jungle-network/pom.xml compile" -end - -task :run_jungle do - run "export MAVEN_OPTS='-Xmx6g -Xms4g' && /usr/local/apache-maven/apache-maven-3.0.5/bin/mvn -f /home/mass/student/k128576/hg/jungle-network/pom.xml exec:java -Dexec.mainClass='app.bbs.DistributeApp' -Dexec.args='-level fatal -host bldsv12.cr.ie.u-ryukyu.ac.jp -port 10000 -p 20000 -bp 8080' " -end - -task :cp_cassandra2 do - run "cp -r /home/mass/share/student/k128576/cassandra/apache-cassandra-2.0.4 /home/mass/student/k128576/cassandra" -end - -task :start_cassandra do - run "cd /home/mass/student/k128576/cassandra/ && /home/mass/student/k128576/cassandra/apache-cassandra-2.0.4/bin/cassandra -f" -end - -task :rsync_bbs do - run "rsync -av /home/mass/share/student/k128576/hg/bulletinboard /home/mass/student/k128576/cassandra/" -end - -task :clean_bbs do - run "/usr/local/apache-maven/apache-maven-3.0.5/bin/mvn -f /home/mass/student/k128576/cassandra/bulletinboard/pom.xml clean" -end - -task :compile_bbs do - run "/usr/local/apache-maven/apache-maven-3.0.5/bin/mvn -f /home/mass/student/k128576/cassandra/bulletinboard/pom.xml compile" -end - -task :one_bbs do - run "/usr/local/apache-maven/apache-maven-3.0.5/bin/mvn -f /home/mass/student/k128576/cassandra/bulletinboard/pom.xml exec:java -Dexec.mainClass='jp.ac.u_ryukyu.ie.cr.shoshi.jungle.bbs.App' -Dexec.args='-one' " -end - -task :quorum_bbs do - run "/usr/local/apache-maven/apache-maven-3.0.5/bin/mvn -f /home/mass/student/k128576/cassandra/bulletinboard/pom.xml exec:java -Dexec.mainClass='jp.ac.u_ryukyu.ie.cr.shoshi.jungle.bbs.App' -Dexec.args='-quorum' " -end - -task :quorum_bbs5 do - run "/usr/local/apache-maven/apache-maven-3.0.5/bin/mvn -f /home/mass/student/k128576/cassandra/bulletinboard/pom.xml exec:java -Dexec.mainClass='jp.ac.u_ryukyu.ie.cr.shoshi.jungle.bbs.App' -Dexec.args='-quorum -rep 5' " -end task :delete_var do run "rm -rf /home/mass/student/k128576/cassandra/var" @@ -88,7 +38,121 @@ run "ps aux |grep maven |grep -v grep |awk '{print $2}'| xargs kill -9" end -task :get_host do - run "sh /home/mass/share/student/k128576/jungle_benchmark/distributed/weighttp/test.sh " + +task :rm_log do + run "rm /home/mass/student/k128576/hg/jungle-network/log/*" +end + +task :hostname01, :roles =>:pall13 do + run "hostname" +end + +task :run_jungle30, :roles =>:pall23 do + run "export MAVEN_OPTS='-Xmx6g -Xms4g' && /usr/local/apache-maven/apache-maven-3.0.5/bin/mvn -f /home/mass/student/k128576/hg/jungle-network/pom.xml exec:java -Dexec.mainClass='app.bbs.DistributeApp' -Dexec.args='-level fatal -host bldsv12.cr.ie.u-ryukyu.ac.jp -port 10000 -p 20000 -bp 8080' " +end + +task :run_jungle20, :roles =>:pall22 do + run "export MAVEN_OPTS='-Xmx6g -Xms4g' && /usr/local/apache-maven/apache-maven-3.0.5/bin/mvn -f /home/mass/student/k128576/hg/jungle-network/pom.xml exec:java -Dexec.mainClass='app.bbs.DistributeApp' -Dexec.args='-level fatal -host bldsv12.cr.ie.u-ryukyu.ac.jp -port 10000 -p 20000 -bp 8080' " +end + +task :run_jungle10, :roles =>:pall21 do + run "export MAVEN_OPTS='-Xmx6g -Xms4g' && /usr/local/apache-maven/apache-maven-3.0.5/bin/mvn -f /home/mass/student/k128576/hg/jungle-network/pom.xml exec:java -Dexec.mainClass='app.bbs.DistributeApp' -Dexec.args='-level fatal -host bldsv12.cr.ie.u-ryukyu.ac.jp -port 10000 -p 20000 -bp 8080' " +end + + + +task :start_cassandra30, :roles =>:pall23 do + run "cd /home/mass/student/k128576/cassandra/ && /home/mass/student/k128576/cassandra/apache-cassandra-2.0.4/bin/cassandra -f" +end + +task :start_cassandra20, :roles =>:pall22 do + run "cd /home/mass/student/k128576/cassandra/ && /home/mass/student/k128576/cassandra/apache-cassandra-2.0.4/bin/cassandra -f" +end + +task :start_cassandra10, :roles =>:pall21 do + run "cd /home/mass/student/k128576/cassandra/ && /home/mass/student/k128576/cassandra/apache-cassandra-2.0.4/bin/cassandra -f" +end + +task :one_bbs10, :roles =>:pall21 do + run "/usr/local/apache-maven/apache-maven-3.0.5/bin/mvn -f /home/mass/student/k128576/cassandra/bulletinboard/pom.xml exec:java -Dexec.mainClass='jp.ac.u_ryukyu.ie.cr.shoshi.jungle.bbs.App' -Dexec.args='-one' " +end + +task :one_bbs20, :roles =>:pall22 do + run "/usr/local/apache-maven/apache-maven-3.0.5/bin/mvn -f /home/mass/student/k128576/cassandra/bulletinboard/pom.xml exec:java -Dexec.mainClass='jp.ac.u_ryukyu.ie.cr.shoshi.jungle.bbs.App' -Dexec.args='-one' " +end + +task :one_bbs30, :roles =>:pall23 do + run "/usr/local/apache-maven/apache-maven-3.0.5/bin/mvn -f /home/mass/student/k128576/cassandra/bulletinboard/pom.xml exec:java -Dexec.mainClass='jp.ac.u_ryukyu.ie.cr.shoshi.jungle.bbs.App' -Dexec.args='-one' " +end + +task :all_bbs10, :roles =>:pall21 do + run "/usr/local/apache-maven/apache-maven-3.0.5/bin/mvn -f /home/mass/student/k128576/cassandra/bulletinboard/pom.xml exec:java -Dexec.mainClass='jp.ac.u_ryukyu.ie.cr.shoshi.jungle.bbs.App' -Dexec.args='-all -rep 10' " +end + +task :all_bbs20, :roles =>:pall22 do + run "/usr/local/apache-maven/apache-maven-3.0.5/bin/mvn -f /home/mass/student/k128576/cassandra/bulletinboard/pom.xml exec:java -Dexec.mainClass='jp.ac.u_ryukyu.ie.cr.shoshi.jungle.bbs.App' -Dexec.args='-all -rep 20' " +end + +task :all_bbs30, :roles =>:pall23 do + run "/usr/local/apache-maven/apache-maven-3.0.5/bin/mvn -f /home/mass/student/k128576/cassandra/bulletinboard/pom.xml exec:java -Dexec.mainClass='jp.ac.u_ryukyu.ie.cr.shoshi.jungle.bbs.App' -Dexec.args='-all -rep 30' " end +task :quorum_bbs30, :roles =>:pall23 do + run "/usr/local/apache-maven/apache-maven-3.0.5/bin/mvn -f /home/mass/student/k128576/cassandra/bulletinboard/pom.xml exec:java -Dexec.mainClass='jp.ac.u_ryukyu.ie.cr.shoshi.jungle.bbs.App' -Dexec.args='-quorum -rep 5' " +end + +task :quorum_bbs20, :roles =>:pall22 do + run "/usr/local/apache-maven/apache-maven-3.0.5/bin/mvn -f /home/mass/student/k128576/cassandra/bulletinboard/pom.xml exec:java -Dexec.mainClass='jp.ac.u_ryukyu.ie.cr.shoshi.jungle.bbs.App' -Dexec.args='-quorum -rep 5' " +end + +task :quorum_bbs10, :roles =>:pall21 do + run "/usr/local/apache-maven/apache-maven-3.0.5/bin/mvn -f /home/mass/student/k128576/cassandra/bulletinboard/pom.xml exec:java -Dexec.mainClass='jp.ac.u_ryukyu.ie.cr.shoshi.jungle.bbs.App' -Dexec.args='-quorum -rep 5' " +end + + + + +task :pbs_start30, :roles =>:pall13 do + run "sudo /etc/init.d/torque start" +end + +task :pbs_stop30, :roles =>:pall13 do + run "sudo /etc/init.d/torque stop" +end + +task :pbs_restart30, :roles =>:pall13 do + run "sudo /etc/init.d/torque restart" +end + +task :pbs_stop do + run "sudo /etc/init.d/torque stop" +end + +task :pbs_start do + run "sudo /etc/init.d/torque start" +end + +task :change_server_name, :roles =>:crpall do + run "sudo mv /var/spool/torque/server_name /home/mass/student/k128576/" + run "sudo cp /home/mass/share/student/k128576/jungle_benchmark/distributed/weighttp/torque_conf/server_name /var/spool/torque/" +end + +task :change_config, :roles =>:crpall do + run "sudo mv /var/spool/torque/mom_priv/config /home/mass/student/k128576/" + run "sudo cp /home/mass/share/student/k128576/jungle_benchmark/distributed/weighttp/torque_conf/config /var/spool/torque/mom_priv/" +end + +task :cat_config, :roles =>:crpall do + run "sudo cat /var/spool/torque/mom_priv/config" +end + +task :default_config, :roles =>:crpall do + run "sudo rm /var/spool/torque/mom_priv/config" + run "sudo cp /home/mass/share/student/k128576/jungle_benchmark/distributed/weighttp/torque_conf/CR_torque/config /var/spool/torque/mom_priv/" +end + +task :default_server_name, :roles =>:crpall do + run "sudo rm /var/spool/torque/server_name" + run "sudo cp /home/mass/share/student/k128576/jungle_benchmark/distributed/weighttp/torque_conf/CR_torque/server_name /var/spool/torque/server_name" +end + diff -r a972a5814b0b -r d6d3b7180e6d distributed/weighttp/loop_qsub.sh --- a/distributed/weighttp/loop_qsub.sh Wed Jan 29 16:22:02 2014 +0900 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,24 +0,0 @@ -#!/bin/sh - -nodesFile=/home/mass/share/student/k128576/jungle_benchmark/distributed/weighttp/nodes30.txt - -if [ $# -lt 1 ]; then - echo "Please set node number" - echo "Usage:./loop_qsub.sh nodenum" - exit 1 -fi - -nodenum=$1 -if [ $nodenum -lt 10 ] -then - nodenum=0$nodenum - fi - echo "nodes = $nodenum" - qsub -e /home/mass/share/student/k128576/jungle_benchmark/distributed/weighttp/error_log/ -o /home/mass/share/student/k128576/jungle_benchmark/distributed/weighttp/log/ -v nodesFile=$nodesFile,requests=100000,schedule=`expr $(date +%s) + 5`,concurrent=200,thread=2 -l nodes=$nodenum:ppn=4 /home/mass/share/student/k128576/jungle_benchmark/distributed/weighttp/weighttp_write_jungle.sh - while [ "x`qstat`" != "x" ] - do - echo "waiting for job to complete" - sleep 5 - done -echo "job complete" - diff -r a972a5814b0b -r d6d3b7180e6d distributed/weighttp/stress_weighttp.sh --- a/distributed/weighttp/stress_weighttp.sh Wed Jan 29 16:22:02 2014 +0900 +++ b/distributed/weighttp/stress_weighttp.sh Thu Jan 30 21:48:09 2014 +0900 @@ -21,8 +21,8 @@ echo ${nodeArray[$num]} } +uri=':8080/showBoardMessage?bname=hello%20world' nodeUrl=`get_node_name < $nodesFile $myNum` -uri=":8080/editMessageUseGet?bname=hello%20world&uuid=0&author=oshiro&msg=write_test&key=" now=$(date +%s) sleepTime=`expr $schedule - $now` sleep $sleepTime diff -r a972a5814b0b -r d6d3b7180e6d distributed/weighttp/weighttp_jungle.sh --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/distributed/weighttp/weighttp_jungle.sh Thu Jan 30 21:48:09 2014 +0900 @@ -0,0 +1,46 @@ +#!/bin/sh + +function usage() { + echo "Please set number of node and mode" + echo "Usage:./loop_qsub.sh nodenum [mode]" + echo "[mode]: read write" + exit 1 +} +if [ $# -lt 2 ]; then + usage +fi +readFlag=1 +if [ $2 == "read" ] +then + readFlag=1 +elif [ $2 == "write" ] +then + readFlag=0 +else + usage +fi + +nodesFile=/home/mass/share/student/k128576/jungle_benchmark/distributed/weighttp/nodes30.txt + +nodenum=$1 +if [ $nodenum -lt 10 ] +then + nodenum=0$nodenum +fi +echo "nodes = $nodenum" +if [ $readFlag -eq 0 ] + echo "write benchmark" + qsub -e /home/mass/share/student/k128576/jungle_benchmark/distributed/weighttp/jungle_log/error_log/ -o /home/mass/share/student/k128576/jungle_benchmark/distributed/weighttp/jungle_log/log/ -v nodesFile=$nodesFile,requests=50000,schedule=`expr $(date +%s) + 10`,concurrent=200,thread=2 -l nodes=$nodenum:ppn=4 /home/mass/share/student/k128576/jungle_benchmark/distributed/weighttp/weighttp_write_jungle.sh +then +else + echo "read benchmark" + qsub -e /home/mass/share/student/k128576/jungle_benchmark/distributed/weighttp/jungle_log/error_log/ -o /home/mass/share/student/k128576/jungle_benchmark/distributed/weighttp/jungle_log/log/ -v nodesFile=$nodesFile,requests=50000,schedule=`expr $(date +%s) + 10`,concurrent=200,thread=2 -l nodes=$nodenum:ppn=4 /home/mass/share/student/k128576/jungle_benchmark/distributed/weighttp/weighttp_read_jungle.sh +fi + +while [ "x`qstat`" != "x" ] +do + echo "waiting for job to complete" + sleep 5 +done +echo "job complete" + diff -r a972a5814b0b -r d6d3b7180e6d distributed/weighttp/weighttp_write_jungle.sh --- a/distributed/weighttp/weighttp_write_jungle.sh Wed Jan 29 16:22:02 2014 +0900 +++ b/distributed/weighttp/weighttp_write_jungle.sh Thu Jan 30 21:48:09 2014 +0900 @@ -4,7 +4,7 @@ #PBS -N write_jungle #PBS -l walltime=00:05:00 -command=/home/mass/share/student/k128576/jungle_benchmark/distributed/weighttp/stress_weighttp.sh +command=/home/mass/share/student/k128576/jungle_benchmark/distributed/weighttp/stress_write_jungle.sh function run() { i=0