diff --git a/tests/tiup-cluster/script/cmd_subtest.sh b/tests/tiup-cluster/script/cmd_subtest.sh index e913fa07b0..54f947591f 100755 --- a/tests/tiup-cluster/script/cmd_subtest.sh +++ b/tests/tiup-cluster/script/cmd_subtest.sh @@ -5,31 +5,42 @@ set -eu function cmd_subtest() { mkdir -p ~/.tiup/bin/ - version=$1 - test_tls=$2 - native_ssh=$3 + local version=$1 + local test_tls=$2 + local native_ssh=$3 + local proxy_ssh=$4 + local node="n" + local client=() + local topo_sep="" + local name="test_cmd_$RANDOM" + local ssh_cmd=(ssh -o "StrictHostKeyChecking=no" -o "PasswordAuthentication=no") + + if [ $proxy_ssh = true ]; then + node="p" + topo_sep="proxy" + client+=("--ssh-proxy-host=bastion") + ssh_cmd+=(-o "ProxyCommand=ssh bastion -W %h:%p") + fi - name="test_cmd_$RANDOM" if [ $test_tls = true ]; then - topo=./topo/full_tls.yaml + topo=./topo/${topo_sep}/full_tls.yaml else - topo=./topo/full.yaml + topo=./topo/${topo_sep}/full.yaml fi - client="" if [ $native_ssh == true ]; then - client="--ssh=system" + client+=("--ssh=system") fi # identify SSH via ssh-agent - eval $(ssh-agent) - ssh-add /root/.ssh/id_rsa + eval $(ssh-agent) &> /dev/null + ssh-add /root/.ssh/id_rsa &> /dev/null mv /root/.ssh/id_rsa{,.bak} - tiup-cluster $client check $topo -i ~/.ssh/id_rsa --enable-mem --enable-cpu --apply + tiup-cluster "${client[@]}" check $topo -i ~/.ssh/id_rsa --enable-mem --enable-cpu --apply mv /root/.ssh/id_rsa{.bak,} - check_result=`tiup-cluster $client --yes check $topo -i ~/.ssh/id_rsa` + check_result=`tiup-cluster "${client[@]}" --yes check $topo -i ~/.ssh/id_rsa` # check the check result echo $check_result | grep "cpu-cores" @@ -39,80 +50,80 @@ function cmd_subtest() { echo $check_result | grep "service" echo $check_result | grep "thp" - for i in {1..5}; do ssh -o "StrictHostKeyChecking=no" -o "PasswordAuthentication=no" n"$i" "grep -q tidb /etc/passwd && (killall -u tidb; userdel -f -r tidb) || true"; done + for i in {1..5}; do "${ssh_cmd[@]}" "${node}${i}" "grep -q tidb /etc/passwd && (killall -u tidb; userdel -f -r tidb) || true"; done # This should fail because there is no such user: tidb - ! tiup-cluster $client --yes deploy $name $version $topo -i ~/.ssh/id_rsa --skip-create-user + ! tiup-cluster "${client[@]}" --yes deploy $name $version $topo -i ~/.ssh/id_rsa --skip-create-user # This is a normal deploy - tiup-cluster $client --yes deploy $name $version $topo -i ~/.ssh/id_rsa + tiup-cluster "${client[@]}" --yes deploy $name $version $topo -i ~/.ssh/id_rsa # Cleanup cluster meta and test --skip-create-user again, this should success rm -rf ~/.tiup/storage/cluster/clusters/$name - tiup-cluster $client --yes deploy $name $version $topo -i ~/.ssh/id_rsa --skip-create-user + tiup-cluster "${client[@]}" --yes deploy $name $version $topo -i ~/.ssh/id_rsa --skip-create-user # check the local config - tiup-cluster $client exec $name -N n1 --command "grep tidb.rules.yml /home/tidb/deploy/prometheus-9090/conf/prometheus.yml" - ! tiup-cluster $client exec $name -N n1 --command "grep node.rules.yml /home/tidb/deploy/prometheus-9090/conf/prometheus.yml" - tiup-cluster $client exec $name -N n1 --command "grep magic-string-for-test /home/tidb/deploy/prometheus-9090/conf/tidb.rules.yml" - tiup-cluster $client exec $name -N n1 --command "grep magic-string-for-test /home/tidb/deploy/grafana-3000/dashboards/tidb.json" - tiup-cluster $client exec $name -N n1 --command "grep magic-string-for-test /home/tidb/deploy/alertmanager-9093/conf/alertmanager.yml" + tiup-cluster "${client[@]}" exec $name -N ${node}1 --command "grep tidb.rules.yml /home/tidb/deploy/prometheus-9090/conf/prometheus.yml" + ! tiup-cluster "${client[@]}" exec $name -N ${node}1 --command "grep node.rules.yml /home/tidb/deploy/prometheus-9090/conf/prometheus.yml" + tiup-cluster "${client[@]}" exec $name -N ${node}1 --command "grep magic-string-for-test /home/tidb/deploy/prometheus-9090/conf/tidb.rules.yml" + tiup-cluster "${client[@]}" exec $name -N ${node}1 --command "grep magic-string-for-test /home/tidb/deploy/grafana-3000/dashboards/tidb.json" + tiup-cluster "${client[@]}" exec $name -N ${node}1 --command "grep magic-string-for-test /home/tidb/deploy/alertmanager-9093/conf/alertmanager.yml" - tiup-cluster $client list | grep "$name" + tiup-cluster "${client[@]}" list | grep "$name" - tiup-cluster $client audit | grep "deploy $name $version" + tiup-cluster "${client[@]}" audit | grep "deploy $name $version" # Get the audit id can check it just runnable id=`tiup-cluster audit | grep "deploy $name $version" | awk '{print $1}'` - tiup-cluster $client audit $id + tiup-cluster "${client[@]}" audit $id - tiup-cluster $client --yes start $name + tiup-cluster "${client[@]}" --yes start $name # Patch a stopped cluster - tiup-cluster $client --yes patch $name ~/.tiup/storage/cluster/packages/tidb-v$version-linux-amd64.tar.gz -R tidb --offline - tiup-cluster $client display $name | grep "tidb (patched)" + tiup-cluster "${client[@]}" --yes patch $name ~/.tiup/storage/cluster/packages/tidb-v$version-linux-amd64.tar.gz -R tidb --offline + tiup-cluster "${client[@]}" display $name | grep "tidb (patched)" - tiup-cluster $client _test $name writable + tiup-cluster "${client[@]}" _test $name writable # check the data dir of tikv # it's ok to omit client type after deploy - tiup-cluster exec $name -N n1 --command "grep /home/tidb/deploy/tikv-20160/data /home/tidb/deploy/tikv-20160/scripts/run_tikv.sh" - tiup-cluster exec $name -N n1 --command "grep advertise-status-addr /home/tidb/deploy/tikv-20160/scripts/run_tikv.sh" - tiup-cluster exec $name -N n3 --command "grep /home/tidb/my_kv_data /home/tidb/deploy/tikv-20160/scripts/run_tikv.sh" + tiup-cluster "${client[@]}" exec $name -N ${node}1 --command "grep /home/tidb/deploy/tikv-20160/data /home/tidb/deploy/tikv-20160/scripts/run_tikv.sh" + tiup-cluster "${client[@]}" exec $name -N ${node}1 --command "grep advertise-status-addr /home/tidb/deploy/tikv-20160/scripts/run_tikv.sh" + tiup-cluster "${client[@]}" exec $name -N ${node}3 --command "grep /home/tidb/my_kv_data /home/tidb/deploy/tikv-20160/scripts/run_tikv.sh" # test checkpoint - tiup-cluster exec $name -N n1 --command "touch /tmp/checkpoint" - tiup-cluster exec $name -N n1 --command "ls /tmp/checkpoint" - tiup-cluster exec $name -N n1 --command "rm -f /tmp/checkpoint" + tiup-cluster "${client[@]}" exec $name -N ${node}1 --command "touch /tmp/checkpoint" + tiup-cluster "${client[@]}" exec $name -N ${node}1 --command "ls /tmp/checkpoint" + tiup-cluster "${client[@]}" exec $name -N ${node}1 --command "rm -f /tmp/checkpoint" id=`tiup-cluster audit | grep "exec $name" | grep "ls /tmp/checkpoint" | awk '{print $1}'` tiup-cluster replay --yes $id - ! tiup-cluster exec $name -N n1 --command "ls /tmp/checkpoint" + ! tiup-cluster "${client[@]}" exec $name -N ${node}1 --command "ls /tmp/checkpoint" # test patch overwrite - tiup-cluster $client --yes patch $name ~/.tiup/storage/cluster/packages/tidb-v$version-linux-amd64.tar.gz -R tidb --overwrite + tiup-cluster "${client[@]}" --yes patch $name ~/.tiup/storage/cluster/packages/tidb-v$version-linux-amd64.tar.gz -R tidb --overwrite # overwrite with the same tarball twice - tiup-cluster $client --yes patch $name ~/.tiup/storage/cluster/packages/tidb-v$version-linux-amd64.tar.gz -R tidb --overwrite + tiup-cluster "${client[@]}" --yes patch $name ~/.tiup/storage/cluster/packages/tidb-v$version-linux-amd64.tar.gz -R tidb --overwrite # test patch with a non-executable entry rm -rf tidb-server touch tidb-server # this is a non-executable regular file tar -czf tidb-non-executable.tar.gz tidb-server - ! tiup-cluster $client --yes patch $name ./tidb-non-executable.tar.gz -R tidb + ! tiup-cluster "${client[@]}" --yes patch $name ./tidb-non-executable.tar.gz -R tidb # test patch with a dir entry rm -rf tidb-server mkdir tidb-server tar -czf tidb-dir-entry.tar.gz tidb-server - ! tiup-cluster $client --yes patch $name ./tidb-dir-entry.tar.gz -R tidb + ! tiup-cluster "${client[@]}" --yes patch $name ./tidb-dir-entry.tar.gz -R tidb - tiup-cluster $client --yes stop $name + tiup-cluster "${client[@]}" --yes stop $name # test start prometheus,grafana won't hang-forever(can't update topology) # let the CI to stop the job if hang forever - ! tiup-cluster $client --yes start $name -R prometheus,grafana + ! tiup-cluster "${client[@]}" --yes start $name -R prometheus,grafana - tiup-cluster $client --yes restart $name + tiup-cluster "${client[@]}" --yes restart $name - tiup-cluster $client _test $name writable + tiup-cluster "${client[@]}" _test $name writable - tiup-cluster $client _test $name data + tiup-cluster "${client[@]}" _test $name data - display_result=`tiup-cluster $client display $name` + display_result=$(tiup-cluster "${client[@]}" display $name) echo "$display_result" | grep "Cluster type" echo "$display_result" | grep "Cluster name" echo "$display_result" | grep "Cluster version" @@ -121,47 +132,47 @@ function cmd_subtest() { echo "$display_result" | grep -v "Since" # display with --uptime should show process uptime - display_result=`tiup-cluster $client display $name --uptime` + display_result=`tiup-cluster "${client[@]}" display $name --uptime` echo "$display_result" | grep "Since" # Test rename - tiup-cluster $client --yes rename $name "tmp-cluster-name" - tiup-cluster $client display "tmp-cluster-name" - tiup-cluster $client --yes rename "tmp-cluster-name" $name + tiup-cluster "${client[@]}" --yes rename $name "tmp-cluster-name" + tiup-cluster "${client[@]}" display "tmp-cluster-name" + tiup-cluster "${client[@]}" --yes rename "tmp-cluster-name" $name # Test enable & disable - tiup-cluster $client exec $name -R tidb --command="systemctl status tidb-4000|grep 'enabled;'" - tiup-cluster $client exec $name -R pd --command="systemctl status pd-2379|grep 'enabled;'" - tiup-cluster $client disable $name -R tidb - tiup-cluster $client exec $name -R tidb --command="systemctl status tidb-4000|grep 'disabled;'" - tiup-cluster $client exec $name -R pd --command="systemctl status pd-2379|grep 'enabled;'" - tiup-cluster $client disable $name - tiup-cluster $client exec $name -R pd --command="systemctl status pd-2379|grep 'disabled;'" - tiup-cluster $client enable $name - tiup-cluster $client exec $name -R tidb --command="systemctl status tidb-4000|grep 'enabled;'" - tiup-cluster $client exec $name -R pd --command="systemctl status pd-2379|grep 'enabled;'" - - tiup-cluster $client --yes clean $name --data --all --ignore-node n1:9090 + tiup-cluster "${client[@]}" exec $name -R tidb --command="systemctl status tidb-4000|grep 'enabled;'" + tiup-cluster "${client[@]}" exec $name -R pd --command="systemctl status pd-2379|grep 'enabled;'" + tiup-cluster "${client[@]}" disable $name -R tidb + tiup-cluster "${client[@]}" exec $name -R tidb --command="systemctl status tidb-4000|grep 'disabled;'" + tiup-cluster "${client[@]}" exec $name -R pd --command="systemctl status pd-2379|grep 'enabled;'" + tiup-cluster "${client[@]}" disable $name + tiup-cluster "${client[@]}" exec $name -R pd --command="systemctl status pd-2379|grep 'disabled;'" + tiup-cluster "${client[@]}" enable $name + tiup-cluster "${client[@]}" exec $name -R tidb --command="systemctl status tidb-4000|grep 'enabled;'" + tiup-cluster "${client[@]}" exec $name -R pd --command="systemctl status pd-2379|grep 'enabled;'" + + tiup-cluster "${client[@]}" --yes clean $name --data --all --ignore-node ${node}1:9090 # Test push and pull echo "test_transfer $name $RANDOM `date`" > test_transfer_1.txt - tiup-cluster $client push $name test_transfer_1.txt "{{ .DeployDir }}/test_transfer.txt" -R grafana - tiup-cluster $client pull $name "{{ .DeployDir }}/test_transfer.txt" test_transfer_2.txt -R grafana + tiup-cluster "${client[@]}" push $name test_transfer_1.txt "{{ .DeployDir }}/test_transfer.txt" -R grafana + tiup-cluster "${client[@]}" pull $name "{{ .DeployDir }}/test_transfer.txt" test_transfer_2.txt -R grafana diff test_transfer_1.txt test_transfer_2.txt rm -f test_transfer_{1,2}.txt echo "checking cleanup data and log" - tiup-cluster $client exec $name -N n1 --command "ls /home/tidb/deploy/prometheus-9090/log/prometheus.log" - ! tiup-cluster $client exec $name -N n1 --command "ls /home/tidb/deploy/tikv-20160/log/tikv.log" + tiup-cluster "${client[@]}" exec $name -N ${node}1 --command "ls /home/tidb/deploy/prometheus-9090/log/prometheus.log" + ! tiup-cluster "${client[@]}" exec $name -N ${node}1 --command "ls /home/tidb/deploy/tikv-20160/log/tikv.log" - tiup-cluster $client --yes start $name + tiup-cluster "${client[@]}" --yes start $name - ! tiup-cluster $client _test $name data + ! tiup-cluster "${client[@]}" _test $name data cp ~/.tiup/storage/cluster/clusters/$name/ssh/id_rsa "/tmp/$name.id_rsa" - tiup-cluster $client --yes destroy $name + tiup-cluster "${client[@]}" --yes destroy $name # after destroy the cluster, the public key should be deleted - ! ssh -o "StrictHostKeyChecking=no" -o "PasswordAuthentication=no" -i "/tmp/$name.id_rsa" tidb@n1 "ls" + ! "${ssh_cmd[@]}" -i "/tmp/$name.id_rsa" tidb@${node}1 "ls" unlink "/tmp/$name.id_rsa" } diff --git a/tests/tiup-cluster/script/scale_core.sh b/tests/tiup-cluster/script/scale_core.sh index 86d53bffe9..ba2eb65354 100755 --- a/tests/tiup-cluster/script/scale_core.sh +++ b/tests/tiup-cluster/script/scale_core.sh @@ -5,33 +5,44 @@ set -eu function scale_core() { mkdir -p ~/.tiup/bin/ - version=$1 - test_tls=$2 - native_ssh=$3 - - client="" - if [ $native_ssh == true ]; then - client="--ssh=system" + local version=$1 + local test_tls=$2 + local native_ssh=$3 + local proxy_ssh=$4 + local node="n" + local client=() + local topo_sep="" + local name="test_scale_core_$RANDOM" + local ssh_cmd=(ssh -o "StrictHostKeyChecking=no" -o "PasswordAuthentication=no") + + if [ $proxy_ssh = true ]; then + node="p" + topo_sep="proxy" + client+=("--ssh-proxy-host=bastion") + ssh_cmd+=(-o "ProxyCommand=ssh bastion -W %h:%p") fi - name="test_scale_core_$RANDOM" if [ $test_tls = true ]; then - topo=./topo/full_tls.yaml + topo=./topo/${topo_sep}/full_tls.yaml else - topo=./topo/full.yaml + topo=./topo/${topo_sep}/full.yaml + fi + + if [ $native_ssh == true ]; then + client+=("--ssh=system") fi - tiup-cluster $client --yes deploy $name $version $topo -i ~/.ssh/id_rsa + tiup-cluster "${client[@]}" --yes deploy $name $version $topo -i ~/.ssh/id_rsa - tiup-cluster $client list | grep "$name" + tiup-cluster "${client[@]}" list | grep "$name" - tiup-cluster $client --yes start $name + tiup-cluster "${client[@]}" --yes start $name - tiup-cluster $client _test $name writable + tiup-cluster "${client[@]}" _test $name writable - tiup-cluster $client display $name + tiup-cluster "${client[@]}" display $name - tiup-cluster $client --yes reload $name --skip-restart + tiup-cluster "${client[@]}" --yes reload $name --skip-restart if [ $test_tls = true ]; then total_sub_one=18 @@ -40,73 +51,73 @@ function scale_core() { fi echo "start scale in tidb" - tiup-cluster $client --yes scale-in $name -N n1:4000 - wait_instance_num_reach $name $total_sub_one $native_ssh + tiup-cluster "${client[@]}" --yes scale-in $name -N ${node}1:4000 + wait_instance_num_reach $name $total_sub_one $native_ssh $proxy_ssh # ensure Prometheus's configuration is updated automatically - ! tiup-cluster $client exec $name -N n1 --command "grep -q n1:10080 /home/tidb/deploy/prometheus-9090/conf/prometheus.yml" + ! tiup-cluster "${client[@]}" exec $name -N ${node}1 --command "grep -q ${node}1:10080 /home/tidb/deploy/prometheus-9090/conf/prometheus.yml" echo "start scale out tidb" - topo=./topo/full_scale_in_tidb.yaml - tiup-cluster $client --yes scale-out $name $topo + topo=./topo/${topo_sep}/full_scale_in_tidb.yaml + tiup-cluster "${client[@]}" --yes scale-out $name $topo # after scale-out, ensure the service is enabled - tiup-cluster $client exec $name -N n1 --command "systemctl status tidb-4000 | grep Loaded |grep 'enabled; vendor'" - tiup-cluster $client exec $name -N n1 --command "grep -q n1:10080 /home/tidb/deploy/prometheus-9090/conf/prometheus.yml" + tiup-cluster "${client[@]}" exec $name -N ${node}1 --command "systemctl status tidb-4000 | grep Loaded |grep 'enabled; vendor'" + tiup-cluster "${client[@]}" exec $name -N ${node}1 --command "grep -q ${node}1:10080 /home/tidb/deploy/prometheus-9090/conf/prometheus.yml" # scale in tikv maybe exists in several minutes or hours, and the GitHub CI is not guaranteed # echo "start scale in tikv" - # tiup-cluster --yes scale-in $name -N n3:20160 - # wait_instance_num_reach $name $total_sub_one $native_ssh + # tiup-cluster --yes scale-in $name -N ${node}3:20160 + # wait_instance_num_reach $name $total_sub_one $native_ssh $proxy_ssh # echo "start scale out tikv" - # topo=./topo/full_scale_in_tikv.yaml + # topo=./topo/${topo_sep}/full_scale_in_tikv.yaml # tiup-cluster --yes scale-out $name $topo echo "start scale in pump" - tiup-cluster $client --yes scale-in $name -N n3:8250 - wait_instance_num_reach $name $total_sub_one $native_ssh + tiup-cluster "${client[@]}" --yes scale-in $name -N ${node}3:8250 + wait_instance_num_reach $name $total_sub_one $native_ssh $proxy_ssh # ensure Prometheus's configuration is updated automatically - ! tiup-cluster $client exec $name -N n1 --command "grep -q n3:8250 /home/tidb/deploy/prometheus-9090/conf/prometheus.yml" + ! tiup-cluster "${client[@]}" exec $name -N ${node}1 --command "grep -q ${node}3:8250 /home/tidb/deploy/prometheus-9090/conf/prometheus.yml" echo "start scale out pump" - topo=./topo/full_scale_in_pump.yaml - tiup-cluster $client --yes scale-out $name $topo + topo=./topo/${topo_sep}/full_scale_in_pump.yaml + tiup-cluster "${client[@]}" --yes scale-out $name $topo # after scale-out, ensure this instance come back - tiup-cluster $client exec $name -N n1 --command "grep -q n3:8250 /home/tidb/deploy/prometheus-9090/conf/prometheus.yml" + tiup-cluster "${client[@]}" exec $name -N ${node}1 --command "grep -q ${node}3:8250 /home/tidb/deploy/prometheus-9090/conf/prometheus.yml" echo "start scale in pd" - tiup-cluster $client --yes scale-in $name -N n3:2379 - wait_instance_num_reach $name $total_sub_one $native_ssh + tiup-cluster "${client[@]}" --yes scale-in $name -N ${node}3:2379 + wait_instance_num_reach $name $total_sub_one $native_ssh $proxy_ssh # validate https://github.com/pingcap/tiup/issues/786 # ensure that this instance is removed from the startup scripts of other components that need to rely on PD - ! tiup-cluster $client exec $name -N n1 --command "grep -q n3:2379 /home/tidb/deploy/tidb-4000/scripts/run_tidb.sh" + ! tiup-cluster "${client[@]}" exec $name -N ${node}1 --command "grep -q ${node}3:2379 /home/tidb/deploy/tidb-4000/scripts/run_tidb.sh" # ensure Prometheus's configuration is updated automatically - ! tiup-cluster $client exec $name -N n1 --command "grep -q n3:2379 /home/tidb/deploy/prometheus-9090/conf/prometheus.yml" + ! tiup-cluster "${client[@]}" exec $name -N ${node}1 --command "grep -q ${node}3:2379 /home/tidb/deploy/prometheus-9090/conf/prometheus.yml" echo "start scale out pd" - topo=./topo/full_scale_in_pd.yaml - tiup-cluster $client --yes scale-out $name $topo + topo=./topo/${topo_sep}/full_scale_in_pd.yaml + tiup-cluster "${client[@]}" --yes scale-out $name $topo # after scale-out, ensure this instance come back - tiup-cluster $client exec $name -N n1 --command "grep -q n3:2379 /home/tidb/deploy/tidb-4000/scripts/run_tidb.sh" - tiup-cluster $client exec $name -N n1 --command "grep -q n3:2379 /home/tidb/deploy/prometheus-9090/conf/prometheus.yml" + tiup-cluster "${client[@]}" exec $name -N ${node}1 --command "grep -q ${node}3:2379 /home/tidb/deploy/tidb-4000/scripts/run_tidb.sh" + tiup-cluster "${client[@]}" exec $name -N ${node}1 --command "grep -q ${node}3:2379 /home/tidb/deploy/prometheus-9090/conf/prometheus.yml" echo "start scale in tidb" - tiup-cluster $client --yes scale-in $name -N n2:4000 - wait_instance_num_reach $name $total_sub_one $native_ssh - ! tiup-cluster $client exec $name -N n2 --command "ls /home/tidb/deploy/monitor-9100/deploy/monitor-9100" - ! tiup-cluster $client exec $name -N n2 --command "ps aux | grep node_exporter | grep -qv grep" - ! tiup-cluster $client exec $name -N n2 --command "ps aux | grep blackbox_exporter | grep -qv grep" + tiup-cluster "${client[@]}" --yes scale-in $name -N ${node}2:4000 + wait_instance_num_reach $name $total_sub_one $native_ssh $proxy_ssh + ! tiup-cluster "${client[@]}" exec $name -N ${node}2 --command "ls /home/tidb/deploy/monitor-9100/deploy/monitor-9100" + ! tiup-cluster "${client[@]}" exec $name -N ${node}2 --command "ps aux | grep node_exporter | grep -qv grep" + ! tiup-cluster "${client[@]}" exec $name -N ${node}2 --command "ps aux | grep blackbox_exporter | grep -qv grep" # after all components on the node were scale-ined, the SSH public is automatically deleted - ! ssh -o "StrictHostKeyChecking=no "-o "PasswordAuthentication=no" -i ~/.tiup/storage/cluster/$name/ssh/id_rsa tidb@n2 "ls" + ! "${ssh_cmd[@]}" -i ~/.tiup/storage/cluster/$name/ssh/id_rsa tidb@${node}2 "ls" echo "start scale out tidb" - topo=./topo/full_scale_in_tidb_2nd.yaml - tiup-cluster $client --yes scale-out $name $topo + topo=./topo/${topo_sep}/full_scale_in_tidb_2nd.yaml + tiup-cluster "${client[@]}" --yes scale-out $name $topo # after scalue-out, ensure node_exporter and blackbox_exporter come back - tiup-cluster $client exec $name -N n2 --command "ls /home/tidb/deploy/monitor-9100/deploy/monitor-9100" - tiup-cluster $client exec $name -N n2 --command "ps aux | grep node_exporter | grep -qv grep" - tiup-cluster $client exec $name -N n2 --command "ps aux | grep blackbox_exporter | grep -qv grep" + tiup-cluster "${client[@]}" exec $name -N ${node}2 --command "ls /home/tidb/deploy/monitor-9100/deploy/monitor-9100" + tiup-cluster "${client[@]}" exec $name -N ${node}2 --command "ps aux | grep node_exporter | grep -qv grep" + tiup-cluster "${client[@]}" exec $name -N ${node}2 --command "ps aux | grep blackbox_exporter | grep -qv grep" - tiup-cluster $client _test $name writable - tiup-cluster $client --yes destroy $name + tiup-cluster "${client[@]}" _test $name writable + tiup-cluster "${client[@]}" --yes destroy $name } diff --git a/tests/tiup-cluster/script/scale_tools.sh b/tests/tiup-cluster/script/scale_tools.sh index 0661d410d0..46e6741060 100755 --- a/tests/tiup-cluster/script/scale_tools.sh +++ b/tests/tiup-cluster/script/scale_tools.sh @@ -5,120 +5,134 @@ set -eu function scale_tools() { mkdir -p ~/.tiup/bin/ - version=$1 - test_tls=$2 - native_ssh=$3 - - client="" - if [ $native_ssh == true ]; then - client="--ssh=system" + local version=$1 + local test_tls=$2 + local native_ssh=$3 + local proxy_ssh=$4 + local node="n" + local client=() + local topo_sep="" + local name="test_scale_tools_$RANDOM" + + if [ $proxy_ssh = true ]; then + node="p" + topo_sep="proxy" + client+=("--ssh-proxy-host=bastion") fi - name="test_scale_tools_$RANDOM" if [ $test_tls = true ]; then - topo=./topo/full_tls.yaml + topo=./topo/${topo_sep}/full_tls.yaml else - topo=./topo/full_without_tiflash.yaml + topo=./topo/${topo_sep}/full_without_tiflash.yaml + fi + + if [ $native_ssh == true ]; then + client+=("--ssh=system") fi - tiup-cluster $client --yes deploy $name $version $topo -i ~/.ssh/id_rsa + tiup-cluster "${client[@]}" --yes deploy $name $version $topo -i ~/.ssh/id_rsa # check the local config - tiup-cluster $client exec $name -N n1 --command "grep magic-string-for-test /home/tidb/deploy/prometheus-9090/conf/tidb.rules.yml" - tiup-cluster $client exec $name -N n1 --command "grep magic-string-for-test /home/tidb/deploy/grafana-3000/dashboards/tidb.json" - tiup-cluster $client exec $name -N n1 --command "grep magic-string-for-test /home/tidb/deploy/alertmanager-9093/conf/alertmanager.yml" - tiup-cluster $client exec $name -N n1 --command "grep alertmanagers /home/tidb/deploy/prometheus-9090/conf/prometheus.yml" + tiup-cluster "${client[@]}" exec $name -N ${node}1 --command "grep magic-string-for-test /home/tidb/deploy/prometheus-9090/conf/tidb.rules.yml" + tiup-cluster "${client[@]}" exec $name -N ${node}1 --command "grep magic-string-for-test /home/tidb/deploy/grafana-3000/dashboards/tidb.json" + tiup-cluster "${client[@]}" exec $name -N ${node}1 --command "grep magic-string-for-test /home/tidb/deploy/alertmanager-9093/conf/alertmanager.yml" + tiup-cluster "${client[@]}" exec $name -N ${node}1 --command "grep alertmanagers /home/tidb/deploy/prometheus-9090/conf/prometheus.yml" for item in pump drainer tidb tikv pd grafana node_exporter blackbox_exporter; do - tiup-cluster $client exec $name -N n1 --command "grep $item /home/tidb/deploy/prometheus-9090/conf/prometheus.yml" + tiup-cluster "${client[@]}" exec $name -N ${node}1 --command "grep $item /home/tidb/deploy/prometheus-9090/conf/prometheus.yml" done - tiup-cluster $client list | grep "$name" + tiup-cluster "${client[@]}" list | grep "$name" - tiup-cluster $client --yes start $name + tiup-cluster "${client[@]}" --yes start $name - tiup-cluster $client _test $name writable + tiup-cluster "${client[@]}" _test $name writable - tiup-cluster $client display $name + tiup-cluster "${client[@]}" display $name if [ $test_tls = true ]; then - total_sub_one=18 - total=19 - total_add_one=20 + local total_sub_one=18 + local total=19 + local total_add_one=20 else - total_sub_one=20 - total=21 - total_add_one=22 + local total_sub_one=20 + local total=21 + local total_add_one=22 fi echo "start scale in pump" - tiup-cluster $client --yes scale-in $name -N n3:8250 - wait_instance_num_reach $name $total_sub_one $native_ssh + tiup-cluster "${client[@]}" --yes scale-in $name -N ${node}3:8250 + wait_instance_num_reach $name $total_sub_one $native_ssh $proxy_ssh echo "start scale out pump" - topo=./topo/full_scale_in_pump.yaml - tiup-cluster $client --yes scale-out $name $topo + topo=./topo/${topo_sep}/full_scale_in_pump.yaml + tiup-cluster "${client[@]}" --yes scale-out $name $topo echo "start scale in cdc" - yes | tiup-cluster $client scale-in $name -N n3:8300 - wait_instance_num_reach $name $total_sub_one $native_ssh + yes | tiup-cluster "${client[@]}" scale-in $name -N ${node}3:8300 + wait_instance_num_reach $name $total_sub_one $native_ssh $proxy_ssh echo "start scale out cdc" - topo=./topo/full_scale_in_cdc.yaml - yes | tiup-cluster $client scale-out $name $topo + topo=./topo/${topo_sep}/full_scale_in_cdc.yaml + yes | tiup-cluster "${client[@]}" scale-out $name $topo if [ $test_tls = false ]; then echo "start scale in tispark" - yes | tiup-cluster $client --yes scale-in $name -N n4:7078 - wait_instance_num_reach $name $total_sub_one $native_ssh + yes | tiup-cluster "${client[@]}" --yes scale-in $name -N ${node}4:7078 + wait_instance_num_reach $name $total_sub_one $native_ssh $proxy_ssh echo "start scale out tispark" - topo=./topo/full_scale_in_tispark.yaml - yes | tiup-cluster $client --yes scale-out $name $topo + topo=./topo/${topo_sep}/full_scale_in_tispark.yaml + yes | tiup-cluster "${client[@]}" --yes scale-out $name $topo fi echo "start scale in grafana" - tiup-cluster $client --yes scale-in $name -N n1:3000 - wait_instance_num_reach $name $total_sub_one $native_ssh + tiup-cluster "${client[@]}" --yes scale-in $name -N ${node}1:3000 + wait_instance_num_reach $name $total_sub_one $native_ssh $proxy_ssh echo "start scale out grafana" - topo=./topo/full_scale_in_grafana.yaml - tiup-cluster $client --yes scale-out $name $topo + topo=./topo/${topo_sep}/full_scale_in_grafana.yaml + tiup-cluster "${client[@]}" --yes scale-out $name $topo echo "start scale out prometheus" - topo=./topo/full_scale_in_prometheus.yaml - tiup-cluster $client --yes scale-out $name $topo - wait_instance_num_reach $name $total_add_one $native_ssh + topo=./topo/${topo_sep}/full_scale_in_prometheus.yaml + tiup-cluster "${client[@]}" --yes scale-out $name $topo + wait_instance_num_reach $name $total_add_one $native_ssh $proxy_ssh echo "start scale in prometheus" - tiup-cluster $client --yes scale-in $name -N n2:9090 - wait_instance_num_reach $name $total $native_ssh + tiup-cluster "${client[@]}" --yes scale-in $name -N ${node}2:9090 + wait_instance_num_reach $name $total $native_ssh $proxy_ssh # make sure grafana dashboards has been set to default (since the full_sale_in_grafana.yaml didn't provide a local dashboards dir) - ! tiup-cluster $client exec $name -N n1 --command "grep magic-string-for-test /home/tidb/deploy/grafana-3000/dashboards/tidb.json" + ! tiup-cluster "${client[@]}" exec $name -N ${node}1 --command "grep magic-string-for-test /home/tidb/deploy/grafana-3000/dashboards/tidb.json" # currently tiflash is not supported in TLS enabled cluster # and only Tiflash support data-dir in multipath if [ $test_tls = false ]; then echo "start scale out tiflash(first time)" - topo=./topo/full_scale_in_tiflash.yaml - tiup-cluster $client --yes scale-out $name $topo - tiup-cluster $client exec $name -N n1 --command "grep tiflash /home/tidb/deploy/prometheus-9090/conf/prometheus.yml" + topo=./topo/${topo_sep}/full_scale_in_tiflash.yaml + tiup-cluster "${client[@]}" --yes scale-out $name $topo + tiup-cluster "${client[@]}" exec $name -N ${node}1 --command "grep tiflash /home/tidb/deploy/prometheus-9090/conf/prometheus.yml" # ensure scale-out will mark pd.enable-placement-rules to true. ref https://github.com/pingcap/tiup/issues/1226 - curl n3:2379/pd/api/v1/config 2>/dev/null | grep '"enable-placement-rules": "true"' + local http_proxy="" + if [ $proxy_ssh = true ]; then + ssh bastion curl ${node}3:2379/pd/api/v1/config 2>/dev/null | grep '"enable-placement-rules": "true"' + else + curl ${node}3:2379/pd/api/v1/config 2>/dev/null | grep '"enable-placement-rules": "true"' + fi # ensure tiflash's data dir exists - tiup-cluster $client exec $name -N n3 --command "ls /home/tidb/deploy/tiflash-9000/data1" - tiup-cluster $client exec $name -N n3 --command "ls /data/tiflash-data" + tiup-cluster "${client[@]}" exec $name -N ${node}3 --command "ls /home/tidb/deploy/tiflash-9000/data1" + tiup-cluster "${client[@]}" exec $name -N ${node}3 --command "ls /data/tiflash-data" echo "start scale in tiflash" - tiup-cluster $client --yes scale-in $name -N n3:9000 - tiup-cluster $client display $name | grep Tombstone + tiup-cluster "${client[@]}" --yes scale-in $name -N ${node}3:9000 + tiup-cluster "${client[@]}" display $name | grep Tombstone echo "start prune tiflash" - yes | tiup-cluster $client prune $name - wait_instance_num_reach $name $total $native_ssh - ! tiup-cluster $client exec $name -N n3 --command "ls /home/tidb/deploy/tiflash-9000/data1" - ! tiup-cluster $client exec $name -N n3 --command "ls /data/tiflash-data" + yes | tiup-cluster "${client[@]}" prune $name + wait_instance_num_reach $name $total $native_ssh $proxy_ssh + ! tiup-cluster "${client[@]}" exec $name -N ${node}3 --command "ls /home/tidb/deploy/tiflash-9000/data1" + ! tiup-cluster "${client[@]}" exec $name -N ${node}3 --command "ls /data/tiflash-data" echo "start scale out tiflash(second time)" - topo=./topo/full_scale_in_tiflash.yaml - tiup-cluster $client --yes scale-out $name $topo + topo=./topo/${topo_sep}/full_scale_in_tiflash.yaml + tiup-cluster "${client[@]}" --yes scale-out $name $topo fi - tiup-cluster $client _test $name writable - tiup-cluster $client --yes destroy $name + tiup-cluster "${client[@]}" _test $name writable + tiup-cluster "${client[@]}" --yes destroy $name # test cluster log dir tiup-cluster notfound-command 2>&1 | grep $HOME/.tiup/logs/tiup-cluster-debug diff --git a/tests/tiup-cluster/script/upgrade.sh b/tests/tiup-cluster/script/upgrade.sh old mode 100755 new mode 100644 index d802d2ca2d..5c3039d14a --- a/tests/tiup-cluster/script/upgrade.sh +++ b/tests/tiup-cluster/script/upgrade.sh @@ -3,50 +3,62 @@ set -eu function upgrade() { - mkdir -p ~/.tiup/bin/ + mkdir -p ~/.tiup/bin && cp -f ./root.json ~/.tiup/bin/ - old_version=$1 - version=$2 - test_tls=$3 + local old_version=$1 + local version=$2 + local test_tls=$3 + local native_ssh=$4 + local proxy_ssh=$5 + local node="n" + local client=() + local topo_sep="" + local name="test_upgrade_$RANDOM" + + if [ $proxy_ssh = true ]; then + node="p" + client+=("--ssh-proxy-host=bastion") + topo_sep="proxy" + fi - name=test_upgrade_$RANDOM if [ $test_tls = true ]; then - topo=./topo/upgrade_tls.yaml + topo=./topo/${topo_sep}/upgrade_tls.yaml else - topo=./topo/upgrade.yaml + topo=./topo/${topo_sep}/upgrade.yaml fi - mkdir -p ~/.tiup/bin && cp -f ./root.json ~/.tiup/bin/ - - yes | tiup-cluster deploy $name $old_version $topo -i ~/.ssh/id_rsa + if [ $native_ssh == true ]; then + client+=("--ssh=system") + fi - yes | tiup-cluster start $name + yes | tiup-cluster "${client[@]}" deploy $name $old_version $topo -i ~/.ssh/id_rsa + yes | tiup-cluster "${client[@]}" start $name # ENV_LABELS_ENV will be replaced only if the rule_dir is not specified. if [ $test_tls = true ]; then - tiup-cluster exec $name -N n1 --command "grep -q ${name} /home/tidb/deploy/prometheus-9090/conf/*.rules.yml" - ! tiup-cluster exec $name -N n1 --command "grep -q ENV_LABELS_ENV /home/tidb/deploy/prometheus-9090/conf/*.rules.yml" + tiup-cluster "${client[@]}" exec $name -N n1 --command "grep -q ${name} /home/tidb/deploy/prometheus-9090/conf/*.rules.yml" + ! tiup-cluster "${client[@]}" exec $name -N n1 --command "grep -q ENV_LABELS_ENV /home/tidb/deploy/prometheus-9090/conf/*.rules.yml" fi - tiup-cluster _test $name writable + tiup-cluster "${client[@]}" _test $name writable - yes | tiup-cluster upgrade $name $version --transfer-timeout 60 + yes | tiup-cluster "${client[@]}" upgrade $name $version --transfer-timeout 60 - tiup-cluster _test $name writable + tiup-cluster "${client[@]}" _test $name writable # test edit-config & reload # change the config of pump and check it after reload # https://stackoverflow.com/questions/5978108/open-vim-from-within-bash-shell-script - EDITOR=ex tiup-cluster edit-config -y $name <