smake

#!/bin/bash

SPARK_DIR="spark-1.1.1"
SPARK_ENV="$SPARK_DIR/conf/spark-env.sh"
SPARK_SLAVES="$SPARK_DIR/conf/slaves"
SPARK_START="$SPARK_DIR/sbin/start-all.sh"
SPARK_STOP="$SPARK_DIR/sbin/stop-all.sh"
SPARK_SUBMIT="$SPARK_DIR/bin/spark-submit"
SPARK_APP="$SPARK_DIR/target/scala-2.10/simple-project_2.10-1.0.jar"
SBT_FILE="simple.sbt"
SRC_DIR="src"
IP_ADDR=$(hostname -i)
TOTAL_CORES=20

#Useless funcs
function splash {
	echo "               _       "
	echo " ____ __  __ _| |_____ "
	echo "(_-< '  \/ _\` | / / -_)"
	echo "/__/_|_|_\__,_|_\_\___|"
	echo
}

function usage {
	echo "Usage:"
	echo -e "\t--compile: compile source code to be executed"
	echo -e "\t--master: configure this pc as master"
	echo -e "\t--clear-workers: Remove all workers from config file"
	echo -e "\t--worker <hostname>: add a worker to the worker list"
	echo -e "\t--start: starts the configured cluster"
	echo -e "\t--cores <n>: specifiy the number of cores to use. Must be called with --run"
	echo -e "\t--run <makefile>: run the compiled source code with the specified makefile"
	echo -e "\t--stop: stops the cluster"
	echo "You can provide several commands on the same smake call. They will be"
	echo "executed in the specified order."
	echo ""
}


#Usefull funcs
function compile {
	echo "Compiling"
	#Prepare files
	cp -r $SBT_FILE $SRC_DIR $SPARK_DIR
	cd $SPARK_DIR
	#Compile
	./sbt/sbt package || ERR=YES
	#Go back
	rm -rf $SBT_FILE $SRC_DIR
	cd ..
	echo -e "Done!\n"
}

function master {
	echo -n "Setting this computer as master.. "
	echo "export SPARK_MASTER_IP=$IP_ADDR" > $SPARK_ENV
	echo "export SPARK_MASTER_PORT=7077" >> $SPARK_ENV
	echo "export MASTER=spark://\${SPARK_MASTER_IP}:\${SPARK_MASTER_PORT}" >> $SPARK_ENV
	echo "export SPARK_WORKER_DIR=/tmp" >> $SPARK_ENV
	echo -e "Done!\n"
}

function worker {
	echo "Adding worker $WORKER_HOST"
	echo -n "Checking worker state.. "
	if ping -c 1 $WORKER_HOST &> /dev/null
	then
		echo "UP!"
		echo -n "Resolving ip.. "
		WORKER_IP=`host $WORKER_HOST | awk '/has address/ { print $4 }'`
		if [[ $WORKER_IP ]]
		then
			echo "OK"
			echo -n "Adding to workers' list.. "
			echo "$WORKER_IP" >> $SPARK_SLAVES
			echo "Done!"
		else
			echo "KO"
			echo "Unable to resolve ip, skipping"
		fi
	else
		echo "DOWN!"
		echo "Skipping $WORKER_HOST because it's down"
	fi
	echo ""
}

function clearworkers {
	echo -n "Clearing workers' list.. "
	echo "#Put here workers' ips" > $SPARK_SLAVES
	echo -e "Done!\n"
}

function start {
	echo "Starting cluster"
	$SPARK_START || ERR=YES
	echo -e "Done!\n"
}

function stop {
	echo "Stopping cluster"
	$SPARK_STOP || ERR=YES
	echo -e "Done!\n"
}

function run {
	echo "Running application"
	$SPARK_SUBMIT --class TestRead --total-executor-cores $TOTAL_CORES $SPARK_APP $MAKEFILE pcserveur.ensimag.fr || ERR=YES
	echo -e "Done!\n"
}

#Start
splash #Display ascii
#Check for args
if [ $# -eq 0 ]
then
    echo "Please provide at least one argument"
	usage
	exit -1
fi
#Check spark folder
if [ ! -d "$SPARK_DIR" ]; then
	echo "Spark directory not found, please run get-spark.sh first."
	exit -1
fi
#Parse args
ERR=
while [ "$1" ]
do
	case "$1" in
	"--compile")
		compile
		;;
	"--master")
		master
		;;
	"--worker")
		shift
		WORKER_HOST=$1
		worker
		;;
	"--clear-workers")
		clearworkers
		;;
	"--start")
		start
		;;
	"--stop")
		stop
		;;
	"--cores")
		shift
		TOTAL_CORES=$1
		;;
	"--run")
		shift
		MAKEFILE=$1
		run
		;;
	*)
		echo -e "\nWARNING: Unknown argument $1, skipping.\n"
		;;
	esac
    shift
done

if [[ "$ERR" ]]; then
	exit 1
fi
exit 0

# End of file