#!/bin/bash
jobid=$JOB_ID
machinefile=$TMPDIR/machines
head_node=''
password=''
declare -a list_machines
# build uniq list of machines assigned by scheduler
for machine in $(cat $TMPDIR/machines | uniq)
do
list_machines[${#list_machines[@]}]=$machine
done
# first node is head node
master_node=${list_machines[0]}
# head node bootstrap
if [[ "x$(hostname)" == "x$master_node" ]]
then
numcpus=$(grep $master_node $machinefile | wc -l)
echo "Isabella Ray head - $numcpus cores @ $master_node"
head_start_log=$(ray start --num-cpus $numcpus --head | grep "ray start")
head_start_log=${head_start_log#*ray start}
head_node=$(echo $head_start_log | awk '{print $1}' | awk -F'=' '{print $2}')
head_node="${head_node%\'*}'"
head_node="'${head_node#*\'}"
password=$(echo $head_start_log | awk '{print $2}' | awk -F'=' '{print $2}')
password="${password%\'*}'"
password="'${password#*\'}"
fi
# worker nodes bootstrap
for machine in ${list_machines[@]:1}
do
numcpus=$(grep $machine $machinefile | wc -l)
echo "Isabella Ray worker - $numcpus cores @ $machine"
master_arg=$(echo $head_node | sed "s/'//g")
password_arg=$(echo $password | sed "s/'//g")
ssh $machine ray start --num-cpus $numcpus --address=$master_arg --redis-password=$password_arg
done
|