linux - what's the best way to fast copy some big file from serval servers to hunderands servers? -
there big file on serval of servers, want copy servers fast possible. below current script, there faster way this, viral distribution, example: source a, targets b c d. a->b, after b done ,then b->c , a->d (at same time) . how can improve script, sugesstion, !
#!/bin/bash set -u source_pool=(seq -f 10.10.10.%g 10 20) host_ip_list=(seq -f 10.10.10.%g 21 200) ready_flag=/tmp/ready_flag lock_flag=/tmp/lock_flag do_lock(){ ip=$1;shift 1 ssh root@$ip "touch $lock_flag" } do_unlock(){ ip=$1;shift 1 ssh root@$ip "rm -f $lock_flag" } find_source(){ while true;do source_ip in ${source_pool[@]} ping $source_ip -c 1 1>/dev/null 2>/dev/null if [[ $? -ne 0 ]];then continue fi ssh root@$source_ip "ls $lock_flag" 1>/dev/null 2>/dev/null if [[ $? -eq 0 ]];then continue fi ssh root@$source_ip "ls $ready_flag" 1>/dev/null 2>/dev/null if [[ $? -ne 0 ]];then continue fi do_lock $source_ip echo $source_ip return 0 done source_ip in ${host_ip_list[@]} ping $source_ip -c 1 1>/dev/null 2>/dev/null if [[ $? -ne 0 ]];then continue fi ssh root@$source_ip "ls $lock_flag" 1>/dev/null 2>/dev/null if [[ $? -eq 0 ]];then continue fi ssh root@$source_ip "ls $ready_flag" 1>/dev/null 2>/dev/null if [[ $? -ne 0 ]];then continue fi do_lock $source_ip echo $source_ip return 0 done sleep 3 done return 1 } do_scp(){ source_ip=$1;shift 1 target_ip=$1;shift 1 ssh root@$target_ip "scp root@$source_ip:~/some_big_file.tar ." ssh root@$target_ip "touch $ready_flag" do_unlock $source_ip } scp_all(){ target_ip in ${host_ip_list[@]} source_ip=$(find_source) do_scp $source_ip $target_ip & done } do_wait(){ local flag=1 while [[ $flag -eq 1 ]];do flag=0 ip in ${host_ip_list[@]} scp_num=`ssh root@$ip "ps -ef | grep scp | grep -v grep | wc -l" ` if [[ $scp_num -ne 0 ]];then flag=1 break fi done sleep 30 done } scp_all do_wait
Comments
Post a Comment