Merge pull request #91 from voukka/SPARK-5246_voukka_resolving_hostname

Spark-5246 resolving hostname

Author: shivaram

.gitignore

@@ -33,3 +33,6 @@ nosetests.xml
 .mr.developer.cfg
 .project
 .pydevproject
+
+# JetBrains products
+.idea/

resolve-hostname.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+# Starting new instance in VPC often results that `hostname` returns something like 'ip-10-1-1-24', which is
+# not resolvable. Which leads to problems like SparkUI failing to bind itself on start up to that hostname as
+# described in https://issues.apache.org/jira/browse/SPARK-5246.
+# This script maps private ip to such hostname via '/etc/hosts'.
+#
+
+# Are we in VPC?
+MAC=`wget -q -O - http://169.254.169.254/latest/meta-data/mac`
+VCP_ID=`wget -q -O - http://169.254.169.254/latest/meta-data/network/interfaces/macs/${MAC}/vpc-id`
+if [ -z "${VCP_ID}" ]; then
+    # echo "nothing to do - instance is not in VPC"
+    exit 0
+fi
+
+SHORT_HOSTNAME=`hostname`
+
+PRIVATE_IP=`wget -q -O - http://169.254.169.254/latest/meta-data/local-ipv4`
+
+# do changes only if short hostname does not resolve
+if [ ! ping -c 1 -q "${SHORT_HOSTNAME}" > /dev/null 2>&1 ]; then
+    echo -e "\n ${PRIVATE_IP} ${SHORT_HOSTNAME}\n" >> /etc/hosts
+
+    # let's make sure that it got fixed
+    if [ ! ping -c 1 -q "${SHORT_HOSTNAME}" > /dev/null 2>&1 ]; then
+        # return some non-zero code to indicate problem
+        echo "Possible bug: unable to fix resolution of local hostname"
+        return 62
+    fi
+
+fi

setup-slave.sh

@@ -14,11 +14,13 @@ source ec2-variables.sh
 
 # Set hostname based on EC2 private DNS name, so that it is set correctly
 # even if the instance is restarted with a different private DNS name
-PRIVATE_DNS=`wget -q -O - http://instance-data.ec2.internal/latest/meta-data/local-hostname`
+PRIVATE_DNS=`wget -q -O - http://169.254.169.254/latest/meta-data/local-hostname`
 hostname $PRIVATE_DNS
 echo $PRIVATE_DNS > /etc/hostname
 HOSTNAME=$PRIVATE_DNS  # Fix the bash built-in hostname variable too
 
+bash /root/spark-ec2/resolve-hostname.sh
+
 echo "Setting up slave on `hostname`..."
 
 # Work around for R3 instances without pre-formatted ext3 disks

setup.sh

@@ -21,8 +21,8 @@ source ec2-variables.sh
 
 # Set hostname based on EC2 private DNS name, so that it is set correctly
 # even if the instance is restarted with a different private DNS name
-PRIVATE_DNS=`wget -q -O - http://instance-data.ec2.internal/latest/meta-data/local-hostname`
-PUBLIC_DNS=`wget -q -O - http://instance-data.ec2.internal/latest/meta-data/hostname`
+PRIVATE_DNS=`wget -q -O - http://169.254.169.254/latest/meta-data/local-hostname`
+PUBLIC_DNS=`wget -q -O - http://169.254.169.254/latest/meta-data/hostname`
 hostname $PRIVATE_DNS
 echo $PRIVATE_DNS > /etc/hostname
 export HOSTNAME=$PRIVATE_DNS  # Fix the bash built-in hostname variable too