Configure JAVA_HOME and SCALA_HOME from environment

Author: jey

deploy_templates.py

@@ -45,7 +45,9 @@
   "hdfs_data_dirs": os.getenv("MESOS_HDFS_DATA_DIRS"),
   "mapred_local_dirs": os.getenv("MESOS_MAPRED_LOCAL_DIRS"),
   "spark_local_dirs": os.getenv("MESOS_SPARK_LOCAL_DIRS"),
-  "default_spark_mem": "%dm" % spark_mb
+  "default_spark_mem": "%dm" % spark_mb,
+  "scala_home": os.getenv("SCALA_HOME"),
+  "java_home": os.getenv("JAVA_HOME")
 }
 
 template_dir="/root/spark-ec2/templates"

setup.sh

@@ -3,6 +3,9 @@
 # Make sure we are in the spark-ec2 directory
 cd /root/spark-ec2
 
+# Load the environment variables specific to this AMI
+source /root/.bash_profile
+
 # Load the cluster variables set by the deploy script
 source ec2-variables.sh
 
@@ -29,10 +32,18 @@ MASTERS=`cat masters`
 NUM_MASTERS=`cat masters | wc -l`
 OTHER_MASTERS=`cat masters | sed '1d'`
 SLAVES=`cat slaves`
-
-JAVA_HOME=/usr/lib/jvm/java-1.6.0-openjdk.x86_64
 SSH_OPTS="-o StrictHostKeyChecking=no -o ConnectTimeout=5"
 
+if [[ "x$JAVA_HOME" == "x" ]] ; then
+    echo "Expected JAVA_HOME to be set in .bash_profile!"
+    exit 1
+fi
+
+if [[ "x$SCALA_HOME" == "x" ]] ; then
+    echo "Expected SCALA_HOME to be set in .bash_profile!"
+    exit 1
+fi
+
 if [[ `tty` == "not a tty" ]] ; then
     echo "Expecting a tty or pty! (use the ssh -t option)."
     exit 1

templates/root/ephemeral-hdfs/conf/hadoop-env.sh

@@ -6,7 +6,7 @@
 # remote nodes.
 
 # The java implementation to use.  Required.
-export JAVA_HOME=/usr/lib/jvm/java-1.6.0
+export JAVA_HOME={{java_home}}
 
 # Extra Java CLASSPATH elements.  Optional.
 # export HADOOP_CLASSPATH=

templates/root/persistent-hdfs/conf/hadoop-env.sh

@@ -6,7 +6,7 @@
 # remote nodes.
 
 # The java implementation to use.  Required.
-export JAVA_HOME=/usr/lib/jvm/java-1.6.0
+export JAVA_HOME={{java_home}}
 
 # Extra Java CLASSPATH elements.  Optional.
 # export HADOOP_CLASSPATH=

templates/root/spark-ec2/mesos/hadoop-framework-conf/hadoop-env.sh

@@ -6,7 +6,7 @@
 # remote nodes.
 
 # The java implementation to use.  Required.
-export JAVA_HOME=/usr/lib/jvm/java-1.6.0-openjdk.x86_64
+export JAVA_HOME={{java_home}}
 
 # Mesos build directory, useful for finding JARs and the native library.
 export MESOS_BUILD_DIR=/root/mesos/build

templates/root/spark/conf/spark-env.sh

@@ -10,7 +10,7 @@
 #   be in the same format as the JVM's -Xmx option, e.g. 300m or 1g).
 # - SPARK_LIBRARY_PATH, to add extra search paths for native libraries.
 
-export SCALA_HOME=/root/scala-2.9.2
+export SCALA_HOME={{scala_home}}
 export MESOS_NATIVE_LIBRARY=/usr/local/lib/libmesos.so
 
 # Set Spark's memory per machine; note that you can also comment this out