diff --git a/src/client/beboot/spindle_bootstrap.c b/src/client/beboot/spindle_bootstrap.c
index d986a0a..97b5575 100644
--- a/src/client/beboot/spindle_bootstrap.c
+++ b/src/client/beboot/spindle_bootstrap.c
@@ -36,6 +36,8 @@ Place, Suite 330, Boston, MA 02111-1307 USA
 
 #include "config.h"
 
+
+
 #if !defined(LIBEXECDIR)
 #error Expected to be built with libdir defined
 #endif
@@ -43,6 +45,9 @@ Place, Suite 330, Boston, MA 02111-1307 USA
 #error Expected to be built with proglib defined
 #endif
 
+extern  char* instantiate_directory( char *pathlist, char *defaultpath, int number );
+extern char *parse_location(char *loc, int number);
+
 char spindle_daemon[] = LIBEXECDIR "/spindle_be";
 char spindle_interceptlib[] = PROGLIBDIR "/libspindleint.so";
 
@@ -50,18 +55,30 @@ int ldcsid;
 unsigned int shm_cachesize;
 
 static int rankinfo[4]={-1,-1,-1,-1};
-static int number;
 static int use_cache;
-static unsigned int cachesize;
-static char *location, *number_s, *orig_location, *symbolic_location;
-static char **cmdline;
+
 static char *executable;
 static char *client_lib;
-static char *opts_s;
 static char **daemon_args;
-static char *cachesize_s;
 
+// Initialized via parse_cmdline()
+static char *symbolic_location;
+static char *cache_path, *fifo_path; // potentially multiple colon-separated paths
+static char *number_s;
+static int number;
+static char *opts_s;
 opt_t opts;
+static char *cachesize_s;
+static unsigned int cachesize;
+static char **cmdline;
+
+// Initialized via main()
+static char *instantiated_cache_path;
+static char *instantiated_fifo_path;
+static char *orig_location;     // The client should be checking
+                                // instantiated_cache_path for files,
+                                // but sometimes this doesn't happen
+                                // and we need to check against orig_location.
 
 char libstr_socket_subaudit[] = PROGLIBDIR "/libspindle_subaudit_socket.so";
 char libstr_pipe_subaudit[] = PROGLIBDIR "/libspindle_subaudit_pipe.so";
@@ -85,13 +102,11 @@ static char *default_subaudit_libstr = libstr_biter_subaudit;
 #endif
 
 extern int spindle_mkdir(char *path);
-extern char *parse_location(char *loc, int number);
-extern char *realize(char *path);
 
 static int establish_connection()
 {
    debug_printf2("Opening connection to server\n");
-   ldcsid = client_open_connection(location, number);
+   ldcsid = client_open_connection(instantiated_fifo_path, number);
    if (ldcsid == -1) 
       return -1;
 
@@ -113,8 +128,9 @@ static void setup_environment()
       connection_str = client_get_connection_string(ldcsid);
 
    setenv("LD_AUDIT", client_lib, 1);
-   setenv("LDCS_LOCATION", location, 1);
    setenv("LDCS_ORIG_LOCATION", orig_location, 1);
+   setenv("LDCS_INSTANTIATED_CACHE_PATH", instantiated_cache_path, 1);
+   setenv("LDCS_INSTANTIATED_FIFO_PATH", instantiated_fifo_path, 1);
    setenv("LDCS_NUMBER", number_s, 1);
    setenv("LDCS_RANKINFO", rankinfo_str, 1);
    if (connection_str)
@@ -161,6 +177,8 @@ static int parse_cmdline(int argc, char *argv[])
    }
 
    symbolic_location = argv[i++];
+   cache_path = argv[i++];
+   fifo_path = argv[i++];
    number_s = argv[i++];
    number = atoi(number_s);
    opts_s = argv[i++];
@@ -173,7 +191,7 @@ static int parse_cmdline(int argc, char *argv[])
    return 0;
 }
 
-static void launch_daemon(char *location)
+static void launch_daemon()
 {
    /*grand-child fork, then execv daemon.  By grand-child forking we ensure that
      the app won't get confused by seeing an unknown process as a child. */
@@ -183,13 +201,9 @@ static void launch_daemon(char *location)
    char unique_file[MAX_PATH_LEN+1];
    char buffer[32];
 
-   result = spindle_mkdir(location);
-   if (result == -1) {
-      debug_printf("Exiting due to spindle_mkdir error\n");
-      exit(-1);
-   }
-   snprintf(unique_file, MAX_PATH_LEN, "%s/spindle_daemon_pid", location);
    unique_file[MAX_PATH_LEN] = '\0';
+   // Use the fifo path to also hold the daemon file.
+   snprintf(unique_file, MAX_PATH_LEN, "%s/spindle_daemon_pid", instantiated_fifo_path);
    fd = open(unique_file, O_CREAT | O_EXCL | O_WRONLY, 0600);
    if (fd == -1) {
       debug_printf("Not starting daemon -- %s already exists\n", unique_file);
@@ -302,13 +316,6 @@ int main(int argc, char *argv[])
    char **j, *spindle_env;
 
    LOGGING_INIT_PREEXEC("Client");
-   debug_printf("Launched Spindle Bootstrapper\n");
-
-   result = parse_cmdline(argc, argv);
-   if (result == -1) {
-      fprintf(stderr, "spindle_boostrap cannot be invoked directly\n");
-      return -1;
-   }
 
    spindle_env = getenv("SPINDLE");
    if (spindle_env) {
@@ -320,16 +327,46 @@ int main(int argc, char *argv[])
       }
    }
 
+
+   debug_printf("Launched Spindle Bootstrapper\n");
+
+   result = parse_cmdline(argc, argv);
+   if (result == -1) {
+      fprintf(stderr, "spindle_boostrap cannot be invoked directly\n");
+      return -1;
+   }
+
    orig_location = parse_location(symbolic_location, number);
    if (!orig_location) {
-      return -1;
+       fprintf( stderr, "parse_location() failed on symbolic_location=%s, number=%d\n",
+               symbolic_location, number );
+       return -1;
+   }
+
+   debug_printf2("Candidate cache paths:  %s:%s\n", cache_path, symbolic_location );
+   instantiated_cache_path  = instantiate_directory( cache_path, symbolic_location, number );
+   if( NULL == cache_path ){
+        debug_printf( "None of the following cache path directory candidates could be instantiated.\n");
+        debug_printf( "%s:%s\n", cache_path, symbolic_location );
+        exit(-1);
+   }else{
+       debug_printf2("Instantiated cache path:  %s\n", instantiated_cache_path);
+   }
+
+   debug_printf2("Candidate fifo paths:   %s:%s\n", fifo_path, symbolic_location );
+   instantiated_fifo_path  = instantiate_directory( fifo_path, symbolic_location, number );
+   if( NULL == fifo_path ){
+        debug_printf( "None of the following fifo path directory candidates could be instantiated.\n");
+        debug_printf( "%s:%s\n", fifo_path, symbolic_location );
+        exit(-1);
+   }else{
+       debug_printf2("Instantiated fifo path:  %s\n", instantiated_fifo_path);
    }
-   location = realize(orig_location);
 
    if (daemon_args) {
-      launch_daemon(location);
+      launch_daemon();
    }
-   
+
    if (opts & OPT_RELOCAOUT) {
       result = establish_connection();
       if (result == -1) {
@@ -346,7 +383,7 @@ int main(int argc, char *argv[])
 #else
       shm_cache_limit = cachesize;
 #endif
-      shmcache_init(location, number, cachesize, shm_cache_limit);
+      shmcache_init(instantiated_cache_path, number, cachesize, shm_cache_limit);
       use_cache = 1;
    }      
    
diff --git a/src/client/client/client.c b/src/client/client/client.c
index c2e9c6f..fcc90be 100644
--- a/src/client/client/client.c
+++ b/src/client/client/client.c
@@ -56,7 +56,6 @@ static char debugging_name[32];
 static char old_cwd[MAX_PATH_LEN+1];
 static int rankinfo[4]={-1,-1,-1,-1};
 
-extern char *parse_location(char *loc, int number);
 extern int is_in_spindle_cache(const char *pathname);
 
 /* compare the pointer top the cookie not the cookie itself, it may be changed during runtime by audit library  */
@@ -67,11 +66,9 @@ static const ElfW(Phdr) *libc_phdrs, *interp_phdrs;
 static int num_libc_phdrs, num_interp_phdrs;
 ElfW(Addr) libc_loadoffset, interp_loadoffset;
 
-/* location has the realize'd path to the local file cache. orig_location is not realized and
- * may contain symlinks
- */
-char *location;
-char *orig_location;
+char *instantiated_cache_path;          // needed by should_intercept.c
+static char *instantiated_fifo_path;
+char *orig_location;                    // needed by should_intercept.c
 int number;
 
 static char *concatStrings(const char *str1, const char *str2) 
@@ -194,7 +191,8 @@ static int init_server_connection()
    if (!use_ldcs)
       return 0;
 
-   location = getenv("LDCS_LOCATION");
+   instantiated_cache_path = getenv("LDCS_INSTANTIATED_CACHE_PATH");
+   instantiated_fifo_path = getenv("LDCS_INSTANTIATED_FIFO_PATH");
    orig_location = getenv("LDCS_ORIG_LOCATION");
    number = atoi(getenv("LDCS_NUMBER"));
    connection = getenv("LDCS_CONNECTION");
@@ -204,10 +202,6 @@ static int init_server_connection()
    opts = atol(opts_s);
    shm_cachesize = atoi(cachesize_s) * 1024;
 
-   if (strchr(location, '$')) {
-      location = parse_location(location, number);
-   }
-
    if (!(opts & OPT_FOLLOWFORK)) {
       debug_printf("Disabling environment variables because we're not following forks\n");
       unsetenv("LD_AUDIT");
@@ -226,14 +220,14 @@ static int init_server_connection()
 #else
       shm_cache_limit = shm_cachesize;
 #endif
-      shmcache_init(location, number, shm_cachesize, shm_cache_limit);
+      shmcache_init(instantiated_cache_path, number, shm_cachesize, shm_cache_limit);
    }
 
    if (connection) {
       /* boostrapper established the connection for us.  Reuse it. */
       debug_printf("Recreating existing connection to server\n");
-      debug_printf3("location = %s, number = %d, connection = %s, rankinfo = %s\n",
-                    location, number, connection, rankinfo_s);
+      debug_printf3("instantiated_fifo_path = %s, number = %d, connection = %s, rankinfo = %s\n",
+                    instantiated_fifo_path, number, connection, rankinfo_s);
       ldcsid  = client_register_connection(connection);
       if (ldcsid == -1)
          return -1;
@@ -243,13 +237,13 @@ static int init_server_connection()
    }
    else {
       /* Establish a new connection */
-      debug_printf("open connection to ldcs %s %d\n", location, number);
-      ldcsid = client_open_connection(location, number);
+      debug_printf("open connection to ldcs %s %d\n", instantiated_fifo_path, number);
+      ldcsid = client_open_connection(instantiated_fifo_path, number);
       if (ldcsid == -1)
          return -1;
 
       send_pid(ldcsid);
-      send_location(ldcsid, location);
+      send_location(ldcsid, instantiated_fifo_path);
       send_rankinfo_query(ldcsid, rankinfo+0, rankinfo+1, rankinfo+2, rankinfo+3);
 #if defined(LIBNUMA)      
       if (opts & OPT_NUMA)
@@ -397,7 +391,7 @@ char *client_library_load(const char *name)
 
    char *orig_file_name = (char *) name;
    if (is_in_spindle_cache(name)) {
-      debug_printf2("Library %s is in spindle cache (%s). Translating request\n", name, location);
+      debug_printf2("Library %s is in spindle cache (%s). Translating request\n", name, instantiated_cache_path);
       memset(fixed_name, 0, MAX_PATH_LEN+1);
       send_orig_path_request(ldcsid, orig_file_name, fixed_name);
       orig_file_name = fixed_name;      
diff --git a/src/client/client/intercept_exec.c b/src/client/client/intercept_exec.c
index 54ceb94..7ae1d2d 100644
--- a/src/client/client/intercept_exec.c
+++ b/src/client/client/intercept_exec.c
@@ -136,8 +136,8 @@ static char **removeEnvironmentStrs(char **envp)
          continue;
       if (strIsPrefix("LD", envp[i])) {
          if (strIsPrefix("LD_AUDIT=", envp[i]) ||
-             strIsPrefix("LDCS_LOCATION=", envp[i]) ||
-             strIsPrefix("LDCS_ORIG_LOCATION=", envp[i]) ||
+             strIsPrefix("LDCS_INSTANTIATED_CACHE_PATH=", envp[i]) ||
+             strIsPrefix("LDCS_INSTANTIATED_FIFO_PATH=", envp[i]) ||
              strIsPrefix("LDCS_CONNECTION=", envp[i]) ||
              strIsPrefix("LDCS_RANKINFO=", envp[i]) ||
              strIsPrefix("LDCS_OPTIONS=", envp[i]) ||
@@ -171,8 +171,8 @@ static char **updateEnvironment(char **envp, int *num_modified, int propogate_sp
          unsetf = orig_unsetenv ? orig_unsetenv : unsetenv;
          unsetf("SPINDLE");
          unsetf("LD_AUDIT");
-         unsetf("LDCS_LOCATION");
-         unsetf("LDCS_ORIG_LOCATION");
+         unsetf("LDCS_INSTANTIATED_CACHE_PATH");
+         unsetf("LDCS_INSTANTIATED_FIFO_PATH");
          unsetf("LDCS_CONNECTION");
          unsetf("LDCS_RANKINFO");
          unsetf("LDCS_OPTIONS");
diff --git a/src/client/client/intercept_readlink.c b/src/client/client/intercept_readlink.c
index 50958e1..73d5bbf 100644
--- a/src/client/client/intercept_readlink.c
+++ b/src/client/client/intercept_readlink.c
@@ -28,7 +28,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA
 ssize_t (*orig_readlink)(const char *path, char *buf, size_t bufsiz);
 int (*orig_readlinkat)(int dirfd, const char *pathname, char *buf, size_t bufsiz);
 
-extern char *location;
+extern char *instantiated_cache_path;
 extern int number;
 
 static ssize_t readlink_worker(const char *path, char *buf, size_t bufsiz,
@@ -38,12 +38,12 @@ static ssize_t readlink_worker(const char *path, char *buf, size_t bufsiz,
    char spindle_id[32];
    int location_len;
 
-   location_len = strlen(location);   
+   location_len = strlen(instantiated_cache_path);   
    snprintf(spindle_id, sizeof(spindle_id), "spindle.%d", number);
 
    if (!strstr(newbuf, spindle_id) ||
-       strncmp(location, newbuf, location_len) != 0) {
-      debug_printf3("readlink not intercepting, %s not prefixed by %s\n", newbuf, location);
+       strncmp(instantiated_cache_path, newbuf, location_len) != 0) {
+      debug_printf3("readlink not intercepting, %s not prefixed by %s\n", newbuf, instantiated_cache_path);
       int len = strlen(newbuf);
       if (len > bufsiz)
          len = bufsiz;
diff --git a/src/client/client/should_intercept.c b/src/client/client/should_intercept.c
index 438db60..5d61d24 100644
--- a/src/client/client/should_intercept.c
+++ b/src/client/client/should_intercept.c
@@ -29,7 +29,7 @@
 
 extern int relocate_spindleapi();
 
-extern char *location;
+extern char *instantiated_cache_path;
 extern char *orig_location;
 static void get_location_tmpdir(char **tmpdir, int *tmpdir_size)
 {
@@ -43,12 +43,12 @@ static void get_location_tmpdir(char **tmpdir, int *tmpdir_size)
       return;
    }
 
-   if (location == NULL) {
+   if (instantiated_cache_path == NULL) {
       *tmpdir = NULL;
       *tmpdir_size = 0;
    }
 
-   strncpy(location_root_cached, location, sizeof(location_root_cached)-1); 
+   strncpy(location_root_cached, instantiated_cache_path, sizeof(location_root_cached)-1); 
    
    last_slash = strrchr(location_root_cached, '/');
    while (last_slash && strncmp(last_slash, "/spindle.", 9) != 0)
@@ -73,12 +73,12 @@ int is_in_spindle_cache(const char *pathname)
    static int location_size = 0;
    static int orig_location_size = 0;
    if (!location_size) {
-      location_size = strlen(location);
+      location_size = strlen(instantiated_cache_path);
    }
    if (!orig_location_size) {
       orig_location_size = strlen(orig_location);
    }
-   return ((strncmp(pathname, location, location_size) == 0) ||
+   return ((strncmp(pathname, instantiated_cache_path, location_size) == 0) ||
            (strncmp(pathname, orig_location, orig_location_size) == 0));
 }
 
@@ -88,7 +88,7 @@ static int is_local_file(const char *pathname)
    int loctmpdir_size;
    if (is_in_spindle_cache(pathname)) {
       debug_printf3("Decided that %s is part of spindle cache %s. Sending to spindle\n",
-                    pathname, location);
+                    pathname, instantiated_cache_path);
       return 0;
    }
    get_location_tmpdir(&loctmpdir, &loctmpdir_size);
diff --git a/src/fe/startup/config_mgr.cc b/src/fe/startup/config_mgr.cc
index ce73c17..1d1134e 100644
--- a/src/fe/startup/config_mgr.cc
+++ b/src/fe/startup/config_mgr.cc
@@ -56,6 +56,18 @@ using namespace std;
 #define SPINDLE_LOC_STR "$TMPDIR"
 #endif
 
+#if defined(SPINDLE_CACHE_PATH)
+#define SPINDLE_CACHEPATH_STR SPINDLE_CACHE_PATH
+#else
+#define SPINDLE_CACHEPATH_STR "$TMPDIR"
+#endif
+
+#if defined(SPINDLE_FIFO_PATH)
+#define SPINDLE_FIFOPATH_STR SPINDLE_FIFO_PATH
+#else
+#define SPINDLE_FIFOPATH_STR "$TMPDIR"
+#endif
+
 #if defined(TESTRM)
 #  define DEFAULT_LAUNCHER_STR TESTRM
 #else
@@ -233,7 +245,11 @@ const list<SpindleOption> Options = {
    { confStrip, "strip", shortStrip, groupMisc, cvBool, {}, "true", 
      "Strip debug and symbol information from binaries before distributing them." },
    { confLocation, "location", shortLocation, groupMisc, cvString, {}, SPINDLE_LOC_STR,
-     "Back-end directory for storing relocated files.  Should be a non-shared location such as a ramdisk." },
+     "Back-end default directory for storing cache, fifos, and daemon files.  Should be a non-shared location such as a ramdisk." },
+   { confCachePath, "cache-path", shortCachePath, groupMisc, cvString, {}, SPINDLE_CACHEPATH_STR,
+     "Colon-separated list of local paths to be used for relocated file caching.  Will use the first path in the list that has rwx permissions." },
+   { confFifoPath, "fifo-path", shortFifoPath, groupMisc, cvString, {}, SPINDLE_FIFOPATH_STR,
+     "Colon-separated list of local paths to be used for fifo files.  Will use the first path in the list that has rwx permissions." },
    { confNoclean, "noclean", shortNoClean, groupMisc, cvBool, {}, "false",
      "Don't remove local file cache after execution." },
    { confDisableLogging, "disable-logging", shortDisableLogging, groupMisc, cvBool, {}, DISABLE_LOGGING_STR,
@@ -671,11 +687,15 @@ bool ConfigMap::toSpindleArgs(spindle_args_t &args, bool alloc_strs) const
          case confNumPorts:
             args.num_ports = numresult;
             break;
-         case confLocation: {
-            string loc = strresult + "/spindle.$NUMBER";
-            args.location = strdup(loc.c_str());
+         case confLocation:
+            args.location = getstr(strresult, alloc_strs);
+            break;
+         case confCachePath:
+            args.cache_path = getstr(strresult, alloc_strs);
+            break;
+         case confFifoPath:
+            args.fifo_path = getstr(strresult, alloc_strs);
             break;
-         }
          case confCachePrefix:
          case confPythonPrefix:
             args.pythonprefix = getstr(strresult, alloc_strs);
diff --git a/src/fe/startup/config_mgr.h b/src/fe/startup/config_mgr.h
index 2de05e4..02be92d 100644
--- a/src/fe/startup/config_mgr.h
+++ b/src/fe/startup/config_mgr.h
@@ -30,6 +30,8 @@ enum SpindleConfigID {
    confPort,
    confNumPorts,
    confLocation,
+   confCachePath,
+   confFifoPath,
    confCachePrefix,
    confPythonPrefix,
    confStrip,
@@ -117,7 +119,9 @@ enum CmdlineShortOptions {
    shortLauncher = 292,
    shortNetwork = 293,
    shortHostbinEnable = 294,
-   shortSpindleLevel = 295
+   shortSpindleLevel = 295,
+   shortCachePath = 296,
+   shortFifoPath = 297,
 };
 
 enum CmdlineGroups {
diff --git a/src/fe/startup/spindle_fe.cc b/src/fe/startup/spindle_fe.cc
index e81a4a1..1027fd5 100644
--- a/src/fe/startup/spindle_fe.cc
+++ b/src/fe/startup/spindle_fe.cc
@@ -67,6 +67,8 @@ static int pack_data(spindle_args_t *args, void* &buffer, unsigned &buffer_size)
    buffer_size += sizeof(opt_t);
    buffer_size += sizeof(unique_id_t);
    buffer_size += args->location ? strlen(args->location) + 1 : 1;
+   buffer_size += args->cache_path ? strlen(args->cache_path) + 1 : 1;
+   buffer_size += args->fifo_path ? strlen(args->fifo_path) + 1 : 1;
    buffer_size += args->pythonprefix ? strlen(args->pythonprefix) + 1 : 1;
    buffer_size += args->preloadfile ? strlen(args->preloadfile) + 1 : 1;
    buffer_size += args->numa_files ? strlen(args->numa_files) + 1 : 1;
@@ -84,6 +86,8 @@ static int pack_data(spindle_args_t *args, void* &buffer, unsigned &buffer_size)
    pack_param(args->startup_type, buf, pos);
    pack_param(args->shm_cache_size, buf, pos);
    pack_param(args->location, buf, pos);
+   pack_param(args->cache_path, buf, pos);
+   pack_param(args->fifo_path, buf, pos);
    pack_param(args->pythonprefix, buf, pos);
    pack_param(args->preloadfile, buf, pos);
    pack_param(args->bundle_timeout_ms, buf, pos);
@@ -206,7 +210,7 @@ int getApplicationArgsFE(spindle_args_t *params, int *spindle_argc, char ***spin
    snprintf(uniqueid_s, sizeof(uniqueid_s), "%lu", params->unique_id);
    snprintf(daemonargc_s, sizeof(daemonargc_s), "6");
    
-   #define MAX_ARGS 16
+   #define MAX_ARGS 32
    *spindle_argv = (char **) malloc(sizeof(char*) * MAX_ARGS);
    (*spindle_argv)[n++] = strdup(spindle_bootstrap);
    if (params->opts & OPT_SELFLAUNCH) {
@@ -220,6 +224,8 @@ int getApplicationArgsFE(spindle_args_t *params, int *spindle_argc, char ***spin
       (*spindle_argv)[n++] = strdup(uniqueid_s);
    }
    (*spindle_argv)[n++] = strdup(params->location);
+   (*spindle_argv)[n++] = strdup(params->cache_path);
+   (*spindle_argv)[n++] = strdup(params->fifo_path);
    (*spindle_argv)[n++] = strdup(number_s);
    (*spindle_argv)[n++] = strdup(opt_s);
    (*spindle_argv)[n++] = strdup(cachesize_s);
@@ -384,9 +390,11 @@ int spindleInitFE(const char **hosts, spindle_args_t *params)
    /* Start FE server */
    debug_printf("spindle_args_t { number = %u; port = %u; num_ports = %u; opts = %lu; unique_id = %lu; "
                 "use_launcher = %u; startup_type = %u; shm_cache_size = %u; location = %s; "
+                "cache_location = %s; fifo_location = %s; "
                 "pythonprefix = %s; preloadfile = %s; bundle_timeout_ms = %u; bundle_cachesize_kb = %u }\n",
                 params->number, params->port, params->num_ports, params->opts, params->unique_id,
                 params->use_launcher, params->startup_type, params->shm_cache_size, params->location,
+                params->cache_path, params->fifo_path,
                 params->pythonprefix, params->preloadfile, params->bundle_timeout_ms,
                 params->bundle_cachesize_kb);
    printSpindleFlags(params->opts);
diff --git a/src/flux/flux-spindle.c b/src/flux/flux-spindle.c
index 64cde39..8333ae6 100644
--- a/src/flux/flux-spindle.c
+++ b/src/flux/flux-spindle.c
@@ -234,6 +234,7 @@ static int sp_getopts (flux_shell_t *shell, struct spindle_ctx *ctx)
     const char *followfork = NULL, *preload = NULL, *level = NULL;
     const char *pyprefix = NULL;
     char *numafiles = NULL;
+    const char *location = NULL, *cache_path = NULL, *fifo_path = NULL;
 
     if (flux_shell_getopt_unpack (shell, "spindle", "o", &opts) < 0)
         return -1;
@@ -255,7 +256,10 @@ static int sp_getopts (flux_shell_t *shell, struct spindle_ctx *ctx)
      *  supplied by the user, but not unpacked (This handles typos, etc).
      */
     if (json_unpack_ex (opts, &error, JSON_STRICT,
-                        "{s?i s?i s?i s?i s?s s?s s?s s?s s?s s?s s?i s?s s?s s?s}",
+                        "{s?s s?s s?s s?s s?i s?i s?i s?i s?s s?s s?s s?s s?s s?s s?i s?s s?s s?s}",
+                        "location", &location,
+                        "cache-path", &cache_path,
+                        "fifo-path", &fifo_path,
                         "noclean", &noclean,
                         "nostrip", &nostrip,
                         "push", &push,
@@ -298,6 +302,12 @@ static int sp_getopts (flux_shell_t *shell, struct spindle_ctx *ctx)
        had_error |= parse_yesno(&ctx->params.opts, OPT_RELOCPY, relocpython);
     if (preload)
        ctx->params.preloadfile = (char *) preload;
+    if (location)
+        ctx->params.location = (char *) location;
+    if (cache_path)
+        ctx->params.cache_path = ( char * ) cache_path;
+    if(fifo_path)
+        ctx->params.fifo_path = ( char * ) fifo_path;
     if (numa) {
        ctx->params.opts |= OPT_NUMA;
     }
diff --git a/src/include/spindle_launch.h b/src/include/spindle_launch.h
index 8c8fd61..de09f42 100644
--- a/src/include/spindle_launch.h
+++ b/src/include/spindle_launch.h
@@ -120,9 +120,13 @@ typedef struct {
    /* Size of client shared memory cache */
    unsigned int shm_cache_size;
 
-   /* The local-disk location where Spindle will store its cache */
+   /* Default local-disk path for the Spindle cache, fifos, and daemons. */
    char *location;
 
+   /* Colon-speparated lists of local-disk paths for caches and fifos*/
+   char *cache_path;
+   char *fifo_path;
+
    /* Colon-seperated list of directories where Python is installed */
    char *pythonprefix;
 
diff --git a/src/server/auditserver/ldcs_audit_server_handlers.c b/src/server/auditserver/ldcs_audit_server_handlers.c
index 5151a9b..53a9c98 100644
--- a/src/server/auditserver/ldcs_audit_server_handlers.c
+++ b/src/server/auditserver/ldcs_audit_server_handlers.c
@@ -2727,7 +2727,7 @@ int exit_note_cb(int fd, int serverid, void *data)
       eresult = -1;
    }
 
-   result = handleExitNote(fd, procdata->location);
+   result = handleExitNote(fd, procdata->instantiated_fifo_path);
    if (result == -1) {
       debug_printf("handleExitNote failed\n");
       eresult = -1;
diff --git a/src/server/auditserver/ldcs_audit_server_md_msocket.c b/src/server/auditserver/ldcs_audit_server_md_msocket.c
index c9d616c..73daa12 100644
--- a/src/server/auditserver/ldcs_audit_server_md_msocket.c
+++ b/src/server/auditserver/ldcs_audit_server_md_msocket.c
@@ -62,7 +62,6 @@ int ldcs_audit_server_md_init ( ldcs_process_data_t *ldcs_process_data ) {
   int rc=0;
 
   char* ldcs_nportsstr=getenv("LDCS_NPORTS");
-  char* ldcs_locmodstr=getenv("LDCS_LOCATION_MOD");
 
   int usedport;
   int serverfd, serverid, i;
@@ -78,7 +77,7 @@ int ldcs_audit_server_md_init ( ldcs_process_data_t *ldcs_process_data ) {
   }
   
   /* open listening port for bootstrapping startup  */
-  serverid=ldcs_audit_server_md_msocket_create_server(ldcs_process_data->location, 
+  serverid=ldcs_audit_server_md_msocket_create_server(ldcs_process_data->instantiated_fifo_path, 
 						      ldcs_msocket_data.default_portlist, ldcs_msocket_data.default_num_ports, 
 						      &usedport);
   if (serverid<0) {
@@ -128,21 +127,6 @@ int ldcs_audit_server_md_init ( ldcs_process_data_t *ldcs_process_data ) {
 
   ldcs_listen_unregister_fd(serverfd);
 
-  
-  if(ldcs_locmodstr) {
-    int ldcs_locmod=atoi(ldcs_locmodstr);
-    if(ldcs_locmod>0) {
-      char buffer[MAX_PATH_LEN];
-      debug_printf3("multiple server per node add modifier to location mod=%d\n",ldcs_locmod);
-      if(strlen(ldcs_process_data->location)+10<MAX_PATH_LEN) {
-	sprintf(buffer,"%s-%02d",ldcs_process_data->location,ldcs_process_data->md_rank%ldcs_locmod);
-	debug_printf3("change location to %s (locmod=%d)\n",buffer,ldcs_locmod);
-	free(ldcs_process_data->location);
-	ldcs_process_data->location=strdup(buffer);
-      } else _error("location path too long");
-    }
-  } 
-  
   return(rc);
 }
 
diff --git a/src/server/auditserver/ldcs_audit_server_process.c b/src/server/auditserver/ldcs_audit_server_process.c
index ad28d94..c0c4ebf 100644
--- a/src/server/auditserver/ldcs_audit_server_process.c
+++ b/src/server/auditserver/ldcs_audit_server_process.c
@@ -43,6 +43,8 @@ Place, Suite 330, Boston, MA 02111-1307 USA
 #include <gperftools/profiler.h>
 #endif
 
+extern  char* instantiate_directory( char *pathlist, char *defaultpath, int number );
+
 ldcs_process_data_t ldcs_process_data;
 unsigned int opts;
 
@@ -140,7 +142,6 @@ int ldcs_audit_server_process(spindle_args_t *args)
    startprofile(args);
 
    debug_printf3("Initializing server data structures\n");
-   ldcs_process_data.location = args->location;
    ldcs_process_data.number = args->number;
    ldcs_process_data.pythonprefix = args->pythonprefix;
    ldcs_process_data.numa_substrs = args->numa_files;
@@ -159,7 +160,28 @@ int ldcs_audit_server_process(spindle_args_t *args)
    ldcs_process_data.completed_ldso_requests = new_requestor_list();
    ldcs_process_data.handling_bundle = 0;
    ldcs_process_data.exit_note_done = 0;
-   
+
+
+   debug_printf2("Candidate cache paths:  %s:%s\n", args->cache_path, args->location );
+   ldcs_process_data.instantiated_cache_path  = instantiate_directory( args->cache_path, args->location, args->number );
+   if( NULL == ldcs_process_data.instantiated_cache_path ){
+        debug_printf( "None of the following cache path directory candidates could be instantiated.\n");
+        debug_printf( "%s:%s\n", args->cache_path, args->location );
+        exit(-1);
+   }else{
+       debug_printf2("Instantiated cache path:  %s\n", ldcs_process_data.instantiated_cache_path);
+   }
+
+   debug_printf2("Candidate fifo paths:  %s:%s\n", args->fifo_path, args->location );
+   ldcs_process_data.instantiated_fifo_path  = instantiate_directory( args->fifo_path, args->location, args->number );
+   if( NULL == ldcs_process_data.instantiated_fifo_path ){
+        debug_printf( "None of the following fifo path directory candidates could be instantiated.\n");
+        debug_printf( "%s:%s\n", args->fifo_path, args->location );
+        exit(-1);
+   }else{
+       debug_printf2("Instantiated fifo path:  %s\n", ldcs_process_data.instantiated_fifo_path);
+   }
+
    if (ldcs_process_data.opts & OPT_PULL) {
       debug_printf("Using PULL model\n");
       ldcs_process_data.dist_model = LDCS_PULL;
@@ -182,14 +204,14 @@ int ldcs_audit_server_process(spindle_args_t *args)
    }
    ldcs_process_data.server_stat.hostname=ldcs_process_data.hostname;
 
-   debug_printf3("Initializing file cache location %s\n", ldcs_process_data.location);
-   ldcs_audit_server_filemngt_init(ldcs_process_data.location);
+   debug_printf3("Initializing file cache location %s\n", ldcs_process_data.instantiated_cache_path);
+   ldcs_audit_server_filemngt_init(ldcs_process_data.instantiated_cache_path);
    if (ldcs_process_data.opts & OPT_PROCCLEAN)
-      init_cleanup_proc(ldcs_process_data.location);
+      init_cleanup_proc(ldcs_process_data.instantiated_cache_path);
 
    debug_printf3("Initializing connections for clients at %s and %u\n",
-                 ldcs_process_data.location, ldcs_process_data.number);
-   serverid = ldcs_create_server(ldcs_process_data.location, ldcs_process_data.number);
+                 ldcs_process_data.instantiated_fifo_path, ldcs_process_data.number);
+   serverid = ldcs_create_server(ldcs_process_data.instantiated_fifo_path, ldcs_process_data.number);
    if (serverid == -1) {
       err_printf("Unable to setup area for client connections\n");
       return -1;
@@ -205,7 +227,7 @@ int ldcs_audit_server_process(spindle_args_t *args)
       ldcs_listen_register_fd(fd, serverid, &_ldcs_server_CB, (void *) &ldcs_process_data);
 
    if (args->opts & OPT_BEEXIT) {
-      fd = createExitNote(args->location);
+      fd = createExitNote(args->fifo_path);
       if (fd != -1) {
          ldcs_listen_register_fd(fd, serverid, exit_note_cb, (void *) &ldcs_process_data);
       }
@@ -235,7 +257,7 @@ int ldcs_audit_server_run()
 
    _ldcs_server_stat_print(&ldcs_process_data.server_stat);
   
-   debug_printf("destroy server (%s,%d)\n", ldcs_process_data.location, ldcs_process_data.number);
+   debug_printf("destroy server (%s,%d)\n", ldcs_process_data.instantiated_cache_path, ldcs_process_data.number);
    ldcs_destroy_server(ldcs_process_data.serverid);
   
    /* destroy md support (multi-daemon) */
diff --git a/src/server/auditserver/ldcs_audit_server_process.h b/src/server/auditserver/ldcs_audit_server_process.h
index 477d42b..bec4898 100644
--- a/src/server/auditserver/ldcs_audit_server_process.h
+++ b/src/server/auditserver/ldcs_audit_server_process.h
@@ -123,7 +123,8 @@ struct ldcs_process_data_struct
   int exit_readys_recvd;
   ldcs_dist_model_t dist_model;
   ldcs_client_t* client_table;
-  char *location;
+  char *instantiated_cache_path;
+  char *instantiated_fifo_path;
   char *hostname;
   char *pythonprefix;
   char *numa_substrs;
diff --git a/src/server/startup/spindle_be.cc b/src/server/startup/spindle_be.cc
index 07f9c49..c412524 100644
--- a/src/server/startup/spindle_be.cc
+++ b/src/server/startup/spindle_be.cc
@@ -59,6 +59,8 @@ static int unpack_data(spindle_args_t *args, void *buffer, int buffer_size)
    unpack_param(args->startup_type, buf, pos);
    unpack_param(args->shm_cache_size, buf, pos);
    unpack_param(args->location, buf, pos);
+   unpack_param(args->cache_path, buf, pos);
+   unpack_param(args->fifo_path, buf, pos);
    unpack_param(args->pythonprefix, buf, pos);
    unpack_param(args->preloadfile, buf, pos);
    unpack_param(args->bundle_timeout_ms, buf, pos);
@@ -139,17 +141,6 @@ int spindleRunBE(unsigned int port, unsigned int num_ports, unique_id_t unique_i
    assert(args.unique_id == unique_id);
    assert(args.port == port);
    
-   
-   /* Expand environment variables in location. */
-   char *new_location = parse_location(args.location, args.number);
-   if (!new_location) {
-      err_printf("Failed to convert location %s\n", args.location);
-      return -1;
-   }
-   debug_printf("Translated location from %s to %s\n", args.location, new_location);
-   free(args.location);
-   args.location = new_location;
-
    result = ldcs_audit_server_process(&args);
    if (result == -1) {
       err_printf("Error in ldcs_audit_server_process\n");
diff --git a/src/utils/parseloc.c b/src/utils/parseloc.c
index de85a52..02b16b9 100644
--- a/src/utils/parseloc.c
+++ b/src/utils/parseloc.c
@@ -45,6 +45,8 @@ extern char *custom_getenv();
 #endif
 #define IS_ENVVAR_CHAR(X) ((X >= 'a' && X <= 'z') || (X >= 'A' && X <= 'Z') || (X == '_'))
 
+extern int spindle_mkdir(char *path);
+
 /* Expand the user specified location with environment variable values */
 char *parse_location(char *loc, int number)
 {
@@ -189,3 +191,48 @@ char *realize(char *path)
    debug_printf2("Realized %s to %s\n", path, newpath);
    return strdup(newpath);
 }
+
+char* instantiate_directory( char *pathlist, char *defaultpath, int number ){
+    /* Test each colon-separated path in pathlist until we find one that passes
+     * parse_location(), realize(), and spindle_mkdir() successfully.  Returns
+     * with a path of a created/confirmed directory, or NULL.*/
+
+    char *full_pathlist = NULL;
+    char *instantiated_path = NULL;
+    if( NULL == pathlist && NULL == defaultpath ){
+        return NULL;
+    }else if ( NULL == pathlist ){
+        full_pathlist = defaultpath;
+    }else if ( NULL == defaultpath ){
+        full_pathlist = pathlist;
+    }else{
+        full_pathlist = calloc( strlen( pathlist ) + strlen( defaultpath ) + 2, sizeof( char ) );
+        sprintf( full_pathlist, "%s:%s", pathlist, defaultpath );
+    }
+
+    debug_printf2("full_pathlist = %s\n", full_pathlist);
+
+    int success = -1;
+    char *saveptr;
+    char *candidate_path = strtok_r( full_pathlist, ":", &saveptr );
+    while( candidate_path ){
+        debug_printf2("Working on candidate_path = %s\n", candidate_path);
+        char *parsed_candidate = parse_location( candidate_path, number );
+        debug_printf2("After parse_location(), parsed_candidate = %s\n", parsed_candidate);
+        if( parsed_candidate ){
+            char *realized_candidate = realize( parsed_candidate );
+            debug_printf2("After realize(), realized_candidate = %s\n", realized_candidate);
+            success = spindle_mkdir( realized_candidate );
+            if( 0 == success ){
+                instantiated_path = strdup( realized_candidate );
+                break;
+            }
+        }
+        candidate_path = strtok_r( NULL, ":", &saveptr );
+    }
+
+    free( full_pathlist );
+    return instantiated_path;
+}
+
+