Add capability to specify directories for cache, daemons, and fifo.

src/client/beboot/spindle_bootstrap.c

@@ -36,32 +36,49 @@ Place, Suite 330, Boston, MA 02111-1307 USA
 
 #include "config.h"
 
+
+
 #if !defined(LIBEXECDIR)
 #error Expected to be built with libdir defined
 #endif
 #if !defined(PROGLIBDIR)
 #error Expected to be built with proglib defined
 #endif
 
+extern  char* instantiate_directory( char *pathlist, char *defaultpath, int number );
+extern char *parse_location(char *loc, int number);
+
 char spindle_daemon[] = LIBEXECDIR "/spindle_be";
 char spindle_interceptlib[] = PROGLIBDIR "/libspindleint.so";
 
 int ldcsid;
 unsigned int shm_cachesize;
 
 static int rankinfo[4]={-1,-1,-1,-1};
-static int number;
 static int use_cache;
-static unsigned int cachesize;
-static char *location, *number_s, *orig_location, *symbolic_location;
-static char **cmdline;
+
 static char *executable;
 static char *client_lib;
-static char *opts_s;
 static char **daemon_args;
-static char *cachesize_s;
 
+// Initialized via parse_cmdline()
+static char *symbolic_location;
+static char *cache_path, *fifo_path; // potentially multiple colon-separated paths
+static char *number_s;
+static int number;
+static char *opts_s;
 opt_t opts;
+static char *cachesize_s;
+static unsigned int cachesize;
+static char **cmdline;
+
+// Initialized via main()
+static char *instantiated_cache_path;
+static char *instantiated_fifo_path;
+static char *orig_location;     // The client should be checking
+                                // instantiated_cache_path for files,
+                                // but sometimes this doesn't happen
+                                // and we need to check against orig_location.
 
 char libstr_socket_subaudit[] = PROGLIBDIR "/libspindle_subaudit_socket.so";
 char libstr_pipe_subaudit[] = PROGLIBDIR "/libspindle_subaudit_pipe.so";
@@ -85,13 +102,11 @@ static char *default_subaudit_libstr = libstr_biter_subaudit;
 #endif
 
 extern int spindle_mkdir(char *path);
-extern char *parse_location(char *loc, int number);
-extern char *realize(char *path);
 
 static int establish_connection()
 {
    debug_printf2("Opening connection to server\n");
-   ldcsid = client_open_connection(location, number);
+   ldcsid = client_open_connection(instantiated_fifo_path, number);
    if (ldcsid == -1) 
       return -1;
 
@@ -113,8 +128,9 @@ static void setup_environment()
       connection_str = client_get_connection_string(ldcsid);
 
    setenv("LD_AUDIT", client_lib, 1);
-   setenv("LDCS_LOCATION", location, 1);
    setenv("LDCS_ORIG_LOCATION", orig_location, 1);
+   setenv("LDCS_INSTANTIATED_CACHE_PATH", instantiated_cache_path, 1);
+   setenv("LDCS_INSTANTIATED_FIFO_PATH", instantiated_fifo_path, 1);
    setenv("LDCS_NUMBER", number_s, 1);
    setenv("LDCS_RANKINFO", rankinfo_str, 1);
    if (connection_str)
@@ -161,6 +177,8 @@ static int parse_cmdline(int argc, char *argv[])
    }
 
    symbolic_location = argv[i++];
+   cache_path = argv[i++];
+   fifo_path = argv[i++];
    number_s = argv[i++];
    number = atoi(number_s);
    opts_s = argv[i++];
@@ -173,7 +191,7 @@ static int parse_cmdline(int argc, char *argv[])
    return 0;
 }
 
-static void launch_daemon(char *location)
+static void launch_daemon()
 {
    /*grand-child fork, then execv daemon.  By grand-child forking we ensure that
      the app won't get confused by seeing an unknown process as a child. */
@@ -183,13 +201,9 @@ static void launch_daemon(char *location)
    char unique_file[MAX_PATH_LEN+1];
    char buffer[32];
 
-   result = spindle_mkdir(location);
-   if (result == -1) {
-      debug_printf("Exiting due to spindle_mkdir error\n");
-      exit(-1);
-   }
-   snprintf(unique_file, MAX_PATH_LEN, "%s/spindle_daemon_pid", location);
    unique_file[MAX_PATH_LEN] = '\0';
+   // Use the fifo path to also hold the daemon file.
+   snprintf(unique_file, MAX_PATH_LEN, "%s/spindle_daemon_pid", instantiated_fifo_path);
    fd = open(unique_file, O_CREAT | O_EXCL | O_WRONLY, 0600);
    if (fd == -1) {
       debug_printf("Not starting daemon -- %s already exists\n", unique_file);
@@ -302,13 +316,6 @@ int main(int argc, char *argv[])
    char **j, *spindle_env;
 
    LOGGING_INIT_PREEXEC("Client");
-   debug_printf("Launched Spindle Bootstrapper\n");
-
-   result = parse_cmdline(argc, argv);
-   if (result == -1) {
-      fprintf(stderr, "spindle_boostrap cannot be invoked directly\n");
-      return -1;
-   }
 
    spindle_env = getenv("SPINDLE");
    if (spindle_env) {
@@ -320,16 +327,46 @@ int main(int argc, char *argv[])
       }
    }
 
+
+   debug_printf("Launched Spindle Bootstrapper\n");
+
+   result = parse_cmdline(argc, argv);
+   if (result == -1) {
+      fprintf(stderr, "spindle_boostrap cannot be invoked directly\n");
+      return -1;
+   }
+
    orig_location = parse_location(symbolic_location, number);
    if (!orig_location) {
-      return -1;
+       fprintf( stderr, "parse_location() failed on symbolic_location=%s, number=%d\n",
+               symbolic_location, number );
+       return -1;
+   }
+
+   debug_printf2("Candidate cache paths:  %s:%s\n", cache_path, symbolic_location );
+   instantiated_cache_path  = instantiate_directory( cache_path, symbolic_location, number );
+   if( NULL == cache_path ){
+        debug_printf( "None of the following cache path directory candidates could be instantiated.\n");
+        debug_printf( "%s:%s\n", cache_path, symbolic_location );
+        exit(-1);
+   }else{
+       debug_printf2("Instantiated cache path:  %s\n", instantiated_cache_path);
+   }
+
+   debug_printf2("Candidate fifo paths:   %s:%s\n", fifo_path, symbolic_location );
+   instantiated_fifo_path  = instantiate_directory( fifo_path, symbolic_location, number );
+   if( NULL == fifo_path ){
+        debug_printf( "None of the following fifo path directory candidates could be instantiated.\n");
+        debug_printf( "%s:%s\n", fifo_path, symbolic_location );
+        exit(-1);
+   }else{
+       debug_printf2("Instantiated fifo path:  %s\n", instantiated_fifo_path);
    }
-   location = realize(orig_location);
 
    if (daemon_args) {
-      launch_daemon(location);
+      launch_daemon();
    }
-   
+
    if (opts & OPT_RELOCAOUT) {
       result = establish_connection();
       if (result == -1) {
@@ -346,7 +383,7 @@ int main(int argc, char *argv[])
 #else
       shm_cache_limit = cachesize;
 #endif
-      shmcache_init(location, number, cachesize, shm_cache_limit);
+      shmcache_init(instantiated_cache_path, number, cachesize, shm_cache_limit);
       use_cache = 1;
    }      
 

src/client/client/client.c

@@ -56,7 +56,6 @@ static char debugging_name[32];
 static char old_cwd[MAX_PATH_LEN+1];
 static int rankinfo[4]={-1,-1,-1,-1};
 
-extern char *parse_location(char *loc, int number);
 extern int is_in_spindle_cache(const char *pathname);
 
 /* compare the pointer top the cookie not the cookie itself, it may be changed during runtime by audit library  */
@@ -67,11 +66,9 @@ static const ElfW(Phdr) *libc_phdrs, *interp_phdrs;
 static int num_libc_phdrs, num_interp_phdrs;
 ElfW(Addr) libc_loadoffset, interp_loadoffset;
 
-/* location has the realize'd path to the local file cache. orig_location is not realized and
- * may contain symlinks
- */
-char *location;
-char *orig_location;
+char *instantiated_cache_path;          // needed by should_intercept.c
+static char *instantiated_fifo_path;
+char *orig_location;                    // needed by should_intercept.c
 int number;
 
 static char *concatStrings(const char *str1, const char *str2) 
@@ -194,7 +191,8 @@ static int init_server_connection()
    if (!use_ldcs)
       return 0;
 
-   location = getenv("LDCS_LOCATION");
+   instantiated_cache_path = getenv("LDCS_INSTANTIATED_CACHE_PATH");
+   instantiated_fifo_path = getenv("LDCS_INSTANTIATED_FIFO_PATH");
    orig_location = getenv("LDCS_ORIG_LOCATION");
    number = atoi(getenv("LDCS_NUMBER"));
    connection = getenv("LDCS_CONNECTION");
@@ -204,10 +202,6 @@ static int init_server_connection()
    opts = atol(opts_s);
    shm_cachesize = atoi(cachesize_s) * 1024;
 
-   if (strchr(location, '$')) {
-      location = parse_location(location, number);
-   }
-
    if (!(opts & OPT_FOLLOWFORK)) {
       debug_printf("Disabling environment variables because we're not following forks\n");
       unsetenv("LD_AUDIT");
@@ -226,14 +220,14 @@ static int init_server_connection()
 #else
       shm_cache_limit = shm_cachesize;
 #endif
-      shmcache_init(location, number, shm_cachesize, shm_cache_limit);
+      shmcache_init(instantiated_cache_path, number, shm_cachesize, shm_cache_limit);
    }
 
    if (connection) {
       /* boostrapper established the connection for us.  Reuse it. */
       debug_printf("Recreating existing connection to server\n");
-      debug_printf3("location = %s, number = %d, connection = %s, rankinfo = %s\n",
-                    location, number, connection, rankinfo_s);
+      debug_printf3("instantiated_fifo_path = %s, number = %d, connection = %s, rankinfo = %s\n",
+                    instantiated_fifo_path, number, connection, rankinfo_s);
       ldcsid  = client_register_connection(connection);
       if (ldcsid == -1)
          return -1;
@@ -243,13 +237,13 @@ static int init_server_connection()
    }
    else {
       /* Establish a new connection */
-      debug_printf("open connection to ldcs %s %d\n", location, number);
-      ldcsid = client_open_connection(location, number);
+      debug_printf("open connection to ldcs %s %d\n", instantiated_fifo_path, number);
+      ldcsid = client_open_connection(instantiated_fifo_path, number);
       if (ldcsid == -1)
          return -1;
 
       send_pid(ldcsid);
-      send_location(ldcsid, location);
+      send_location(ldcsid, instantiated_fifo_path);
       send_rankinfo_query(ldcsid, rankinfo+0, rankinfo+1, rankinfo+2, rankinfo+3);
 #if defined(LIBNUMA)      
       if (opts & OPT_NUMA)
@@ -397,7 +391,7 @@ char *client_library_load(const char *name)
 
    char *orig_file_name = (char *) name;
    if (is_in_spindle_cache(name)) {
-      debug_printf2("Library %s is in spindle cache (%s). Translating request\n", name, location);
+      debug_printf2("Library %s is in spindle cache (%s). Translating request\n", name, instantiated_cache_path);
       memset(fixed_name, 0, MAX_PATH_LEN+1);
       send_orig_path_request(ldcsid, orig_file_name, fixed_name);
       orig_file_name = fixed_name;      

src/client/client/intercept_exec.c

@@ -136,8 +136,8 @@ static char **removeEnvironmentStrs(char **envp)
          continue;
       if (strIsPrefix("LD", envp[i])) {
          if (strIsPrefix("LD_AUDIT=", envp[i]) ||
-             strIsPrefix("LDCS_LOCATION=", envp[i]) ||
-             strIsPrefix("LDCS_ORIG_LOCATION=", envp[i]) ||
+             strIsPrefix("LDCS_INSTANTIATED_CACHE_PATH=", envp[i]) ||
+             strIsPrefix("LDCS_INSTANTIATED_FIFO_PATH=", envp[i]) ||
              strIsPrefix("LDCS_CONNECTION=", envp[i]) ||
              strIsPrefix("LDCS_RANKINFO=", envp[i]) ||
              strIsPrefix("LDCS_OPTIONS=", envp[i]) ||
@@ -171,8 +171,8 @@ static char **updateEnvironment(char **envp, int *num_modified, int propogate_sp
          unsetf = orig_unsetenv ? orig_unsetenv : unsetenv;
          unsetf("SPINDLE");
          unsetf("LD_AUDIT");
-         unsetf("LDCS_LOCATION");
-         unsetf("LDCS_ORIG_LOCATION");
+         unsetf("LDCS_INSTANTIATED_CACHE_PATH");
+         unsetf("LDCS_INSTANTIATED_FIFO_PATH");
          unsetf("LDCS_CONNECTION");
          unsetf("LDCS_RANKINFO");
          unsetf("LDCS_OPTIONS");

src/client/client/intercept_readlink.c

@@ -28,7 +28,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA
 ssize_t (*orig_readlink)(const char *path, char *buf, size_t bufsiz);
 int (*orig_readlinkat)(int dirfd, const char *pathname, char *buf, size_t bufsiz);
 
-extern char *location;
+extern char *instantiated_cache_path;
 extern int number;
 
 static ssize_t readlink_worker(const char *path, char *buf, size_t bufsiz,
@@ -38,12 +38,12 @@ static ssize_t readlink_worker(const char *path, char *buf, size_t bufsiz,
    char spindle_id[32];
    int location_len;
 
-   location_len = strlen(location);   
+   location_len = strlen(instantiated_cache_path);   
    snprintf(spindle_id, sizeof(spindle_id), "spindle.%d", number);
 
    if (!strstr(newbuf, spindle_id) ||
-       strncmp(location, newbuf, location_len) != 0) {
-      debug_printf3("readlink not intercepting, %s not prefixed by %s\n", newbuf, location);
+       strncmp(instantiated_cache_path, newbuf, location_len) != 0) {
+      debug_printf3("readlink not intercepting, %s not prefixed by %s\n", newbuf, instantiated_cache_path);
       int len = strlen(newbuf);
       if (len > bufsiz)
          len = bufsiz;

src/client/client/should_intercept.c

@@ -29,7 +29,7 @@
 
 extern int relocate_spindleapi();
 
-extern char *location;
+extern char *instantiated_cache_path;
 extern char *orig_location;
 static void get_location_tmpdir(char **tmpdir, int *tmpdir_size)
 {
@@ -43,12 +43,12 @@ static void get_location_tmpdir(char **tmpdir, int *tmpdir_size)
       return;
    }
 
-   if (location == NULL) {
+   if (instantiated_cache_path == NULL) {
       *tmpdir = NULL;
       *tmpdir_size = 0;
    }
 
-   strncpy(location_root_cached, location, sizeof(location_root_cached)-1); 
+   strncpy(location_root_cached, instantiated_cache_path, sizeof(location_root_cached)-1); 
 
    last_slash = strrchr(location_root_cached, '/');
    while (last_slash && strncmp(last_slash, "/spindle.", 9) != 0)
@@ -73,12 +73,12 @@ int is_in_spindle_cache(const char *pathname)
    static int location_size = 0;
    static int orig_location_size = 0;
    if (!location_size) {
-      location_size = strlen(location);
+      location_size = strlen(instantiated_cache_path);
    }
    if (!orig_location_size) {
       orig_location_size = strlen(orig_location);
    }
-   return ((strncmp(pathname, location, location_size) == 0) ||
+   return ((strncmp(pathname, instantiated_cache_path, location_size) == 0) ||
            (strncmp(pathname, orig_location, orig_location_size) == 0));
 }
 
@@ -88,7 +88,7 @@ static int is_local_file(const char *pathname)
    int loctmpdir_size;
    if (is_in_spindle_cache(pathname)) {
       debug_printf3("Decided that %s is part of spindle cache %s. Sending to spindle\n",
-                    pathname, location);
+                    pathname, instantiated_cache_path);
       return 0;
    }
    get_location_tmpdir(&loctmpdir, &loctmpdir_size);