comm_data.c

Go to the documentation of this file.

/* $Id: comm_data.c,v 1.82 2008/12/11 02:06:24 tbrady Exp $ */
/* $UTK_Copyright: $ */

#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <string.h>
#include <stdarg.h>
#include <errno.h>

#ifdef HAVE_CONFIG_H
#include "config.h"
#endif /* HAVE_CONFIG_H */

#include "portability.h"
#include "utility.h"
#include "comm_basics.h"
#include "comm_encode.h"
#include "comm_data.h"
#include "grpc.h"
#include "gs_seq_dsi.h"
#include "gs_seq_data_handle.h"


/* used for tracking call signatures by GridSolve request sequencing */
static char *client_callsig = NULL;
static char *server_callsig = NULL;


static int gs_send_data_transfer_request(
char *data_handle, gs_argument_t *argptr, int my_dsig) {
    LFS_DSI_OBJECT *obj;
    char *server_name, *path, *file_name, *full_name;
    int server_sock, tag, port;
    int len, sender_major, sender_dsig;
    struct hostent *hp;
    char cid[CID_LEN];
    ipaddr_t ipaddr;

    
    if (!data_handle || !argptr) {
        fprintf(stderr, "bad data handle or argument pointer \
                in gs_send_data_transfer_request\n");
        return -1;
    }
    
    /* decode the data handle */
    if ((obj = gs_seq_decode_lfs_dsi_object(data_handle)) == NULL) {
        fprintf(stderr, "error decoding data handle\n");
        return -1;
    }

    if (!DATA_RECV_FILE_PATH) {
        gs_seq_set_lfs_dsi_data_storage_path();
    }

    /* construct the full path name */
    path = DATA_RECV_FILE_PATH;
    file_name = obj->file_name;

    full_name = (char *) malloc(sizeof(char) 
    * (strlen(path) + strlen(file_name) + 1));
    if (!full_name) {
        perror("malloc");
        return -1;
    }
    
    strcpy(full_name, path);
    full_name[strlen(path)] = '\0';
    
    strcat(full_name, file_name);
    full_name[strlen(path) + strlen(file_name)] = '\0';

    /* get the name of the server to send data transfer request */
    server_name = obj->source;

    memset(cid, 0xFF, CID_LEN);

    /* default port */
    port = GRIDSOLVE_SERVER_PORT_DEFAULT;
    
    /* get server host by its name */
    if ((hp = gethostbyname(server_name)) == NULL) {
        errno = errno_socket();
        fprintf(stderr, 
        "could not gethostbyname for %s (errno %d) \n", server_name, errno);
        return INVALID_SOCKET;
    }
    
    memcpy((void *) &ipaddr, hp->h_addr_list[0], sizeof(ipaddr));
  
    /* all ones will match any component ID */
    memset(cid, 0xFF, CID_LEN);

    /* connect to the server */
    server_sock = gs_connect_to_host(cid, ipaddr, port, 0, 0);

    if(server_sock < 0) {
        ERRPRINTF("unsuccessful (connecting server)\n");
        return -1;
    }

    /* send tag and GridSolve version */
    if((gs_send_tag(server_sock, GS_PROT_DATA_TRANSFER) < 0) ||
       (gs_send_string(server_sock, VERSION) < 0)) {
        ERRPRINTF("unsuccessful (sending tag to Server)\n");
        return -1;
    }
    
    /* receive the replied tag */
    if(gs_recv_tag(server_sock, &tag) < 0) {
        ERRPRINTF("error communicating with server.\n");
        return -1;
    }

    if(tag != GS_PROT_OK) {
        if(tag == GS_PROT_VERSION_MISMATCH)
            ERRPRINTF("error: server is an incompatible version\n");
        else
            ERRPRINTF("error: server refused\n");
         return -1;
    }
        
    /* send the data handle */
    if (gs_send_string(server_sock, data_handle) < 0) {
        ERRPRINTF("unsuccessful (sending data handle)\n");
        return -1;
    }

    /*
    if (gs_recv_arg_into_file(server_sock, full_name) < 0) {
        ERRPRINTF("error receiving data into file\n");
        return -1;
    }

    if (gs_seq_restore_data_from_file(full_name, argptr, my_dsig) < 0) {
        ERRPRINTF("error reading data from file\n");
        return -1;
    }

    if (remove(full_name) < 0) {
        ERRPRINTF("error removing file: %s\n", full_name);
        return -1;
    }
    */
    
    if (gs_recv_int(server_sock, &len) < 0) {
        fprintf(stderr, "error receiving file length\n");
        return -1;
    }
    
    if (gs_recv_tag(server_sock, &sender_major) < 0) {
        fprintf(stderr, "error receiving sender major\n");
        return -1;
    }

    if (gs_recv_int(server_sock, &sender_dsig) < 0) {
        fprintf(stderr, "error receiving sender data signature\n");
        return -1;
    }

    if (gs_recv_arg(server_sock, argptr, sender_major, 
        sender_dsig, my_dsig, GS_SERVER_SIDE) < 0) {
        fprintf(stderr, "error receiving argument\n");
        return -1;
    }
    
    return 0;
}


int 
gs_send_tag(SOCKET sock, int tag)
{
  unsigned char bytetag;
  
  DBGPRINTF("sock %d tag %d\n", sock, tag);
  /* Check that the tag is in the correct range */
  if (tag<0 || tag>127) {
    ERRPRINTF("Tag value %d cannot be held in one byte!!!\n", tag);
    return -1;
  }
  bytetag = tag;
  if (gs_twrite(sock, &bytetag, sizeof(bytetag)) <= 0) {
    ERRPRINTF("Could not write tag %d to the network\n", tag);
    return -1;
  }
  return 0;
}

int 
gs_recv_tag(SOCKET sock, int *tag) 
{
  char bytetag = -1;

  if (gs_tread(sock, &bytetag, sizeof(bytetag)) == -1) {
    ERRPRINTF("Error in reading tag \n");
    return(-1);
  }
  *tag = bytetag;
  return 0;
}

int 
gs_send_int(SOCKET sock, int tosend) 
{
  uint32_t uint1 = (uint32_t)tosend;
  uint32_t uint2 = htonl(uint1);

  if (tosend < 0) {
    ERRPRINTF("The integer must be unsigned %d\n", tosend);
    return -1;
  }

  if (gs_twrite(sock, &uint2, sizeof(uint32_t)) <= 0) {
    ERRPRINTF("Could not send int %d\n", tosend);
    return -1;
  }  

  return 0;
}

int 
gs_recv_int(SOCKET sock, int *torecv)
{
  uint32_t uint2;

  if (gs_tread(sock, (char *)&uint2, sizeof(uint32_t)) == -1) {
    ERRPRINTF("Error receiving int \n");
    return(-1);
  }

  *torecv = (int)ntohl(uint2);

  return 0;
}

int 
gs_send_string(SOCKET sock, char *tosend) 
{
  int len = -1;

  ASSERT_EXPR((tosend != NULL), return(-1));
  len = strlen(tosend);

#ifdef GS_DEBUG
  {
    char *tmps = strdup(tosend);
    if (strlen(tmps) > 50) tmps[50]='\0';
    DBGPRINTF("sending string of len %d to sock %d: '%s...'\n", len, sock, tmps);
    FREE(tmps);
 }
#else 
  DBGPRINTF("sending string of len %d to sock %d \n", len, sock);
#endif

  if(gs_send_int(sock, len) < 0) {
    ERRPRINTF("Error sending string length\n");
    return -1;
  }

  if(gs_twrite(sock, tosend, len*sizeof(char)) < 0) {
    ERRPRINTF("Error sending string\n");
    return -1;
  }

  return 0;
}

int 
gs_recv_string(SOCKET sock, char **torecv)
{
  int len = -1;

  if(gs_recv_int(sock, &len) < 0) {
    ERRPRINTF("Error receiving string length\n");
    return -1;
  }

  if((*torecv = (char *)CALLOC(len+1, sizeof(char))) == NULL) {
    ERRPRINTF("Error allocating memory for string\n");
    return -1;
  }

  if(gs_tread(sock, *torecv, len*sizeof(char)) < 0) {
    free(*torecv);
    ERRPRINTF("Error reading string\n");
    return -1;
  }

#ifdef GS_DEBUG
  {
    char *tmps = strdup(*torecv);
    if (strlen(tmps) > 50) tmps[51]='\0';
    DBGPRINTF("recvd string of len %d: '%s...'\n", len, tmps);
    FREE(tmps);
 }
#else 
  DBGPRINTF("recvd string of len %d\n", len); fflush(NULL);
#endif

  return 0;
}

int 
gs_get_element_size(enum datatype data_type, int dsig)
{
  switch(data_type) {
    case GS_INT:
      return DSIG_INT_SIZE(dsig);
    case GS_FLOAT:
      return DSIG_FLOAT_SIZE(dsig);
    case GS_DOUBLE:
      return DSIG_DOUBLE_SIZE(dsig);
    case GS_SCOMPLEX:
      return 2 * DSIG_FLOAT_SIZE(dsig);
    case GS_DCOMPLEX:
      return 2 * DSIG_DOUBLE_SIZE(dsig);
    case GS_CHAR:
      return sizeof(char);
    default:     
      ERRPRINTF("gs_get_element_size: bad data type\n");
      return 1;
  }
}

int
gs_recv_arg_into_file(SOCKET sock, char *fname)
{
  char fxbuf[FILE_XFER_BUFSZ];
  int n, nleft, len, fd;

  fd = open(fname, O_WRONLY | O_CREAT | O_TRUNC, 0666);
  if(fd < 0) {
    ERRPRINTF("File '%s' could not be opened\n", fname);
    return -1;
  }

  /* should probably change this at some point to recv_long to
   * support large file sizes.
   */
  if(gs_recv_int(sock, &len) < 0) {
    ERRPRINTF("error in receiving file len\n");
    close(fd);
    return -1;
  }

  nleft = len;
  while(nleft > 0) {
    n = gs_tread(sock, fxbuf, MIN(FILE_XFER_BUFSZ, nleft));

    if(n < 0) {
      if(errno_socket() == EINTR)
        continue;
      else {
        ERRPRINTF("error in reading argument data from sock\n");
        close(fd);
        return -1;
      }
    } else if (n == 0)
      break;

    if(write(fd, fxbuf, n) != n) {
      ERRPRINTF("error in writing argument data to file\n");
      close(fd);
      return -1;
    }

    nleft -= n;
  }

  close(fd);

  if(nleft == 0)
    return 0;
  else
    return -1;
}

int
gs_recv_file_scalar(SOCKET sock, gs_argument_t *arg, gs_side_t side)
{
  if(side == GS_SERVER_SIDE) {
    if(gs_recv_arg_into_file(sock, arg->name) < 0) {
      ERRPRINTF("error receiving arg data into file\n");
      return -1;
    }

    arg->data = strdup(arg->name);

    if(!arg->data) {
      ERRPRINTF("error allocating string for arg.\n");
      return -1;
    }
  }
  else if(side == GS_CLIENT_SIDE) {
    if(gs_recv_arg_into_file(sock, arg->data) < 0) {
      ERRPRINTF("error receiving arg data into file\n");
      return -1;
    }
  }
  else {
    ERRPRINTF("Bad 'side' argument.\n");
    return -1;
  }

  return 0;
}

int
gs_recv_file_vector(SOCKET sock, gs_argument_t *arg, gs_side_t side)
{
  char **data;
  int i;

  if(!arg) return -1;

  if(side == GS_SERVER_SIDE) {
    arg->data = malloc(arg->rows * sizeof(char *));
    if(!arg->data) {
      ERRPRINTF("error allocating memory for filenames.\n");
      return -1;
    }

    data = (char **)arg->data;

    for(i = 0; i < arg->rows; i++) {
      data[i] = dstring_sprintf("%s_%d", arg->name, i);

      if(!data[i]) {
        ERRPRINTF("error allocating string for arg.\n");
        return -1;
      }

      if(gs_recv_arg_into_file(sock, data[i]) < 0) {
        ERRPRINTF("error receiving arg data into file\n");
        return -1;
      }
    }
  }
  else if(side == GS_CLIENT_SIDE) {
    data = (char **)arg->data;

    for(i = 0; i < arg->rows; i++) {
      if(gs_recv_arg_into_file(sock, data[i]) < 0) {
        ERRPRINTF("error receiving arg data into file\n");
        return -1;
      }
    }
  }
  else {
    ERRPRINTF("Bad 'side' argument.\n");
    return -1;
  }

  return 0;
}

int 
gs_recv_arg(SOCKET sock, gs_argument_t *arg, int sender_major, 
  int sender_dsig, int my_dsig, gs_side_t side)
{
  int sender_elsize, data_size;
  char *data;

  if(!arg || !arg->prob)
    return -1;

  if(arg->objecttype == GS_FILE)
    return gs_recv_file_scalar(sock, arg, side);
  else if(arg->objecttype == GS_PACKEDFILE)
    return gs_recv_file_vector(sock, arg, side);

  /* size expressions must have been evaluated first before calling this! */
  sender_elsize = gs_get_element_size(arg->datatype, sender_dsig);
  data_size = arg->rows * arg->cols * sender_elsize;

  data = arg->data;

  DBGPRINTF("receiving arg %s, type=%s, rows=%d, cols=%d, bytes=%d\n",
    arg->name, gs_c_datatype[arg->datatype], arg->rows, arg->cols, data_size);

  if(arg->dsi) {
    if(side == GS_SERVER_SIDE) {
      if(gs_recv_dsi_input_arg(sock, arg, data_size) < 0) {
        ERRPRINTF("Error receiving DSI input arg\n");
        return -1;
      }

      data = arg->data;
    }
    else {
      if(gs_recv_dsi_output_arg(sock, arg, data_size) < 0) {
        ERRPRINTF("Error receiving DSI output arg\n");
        return -1;
      }

      /* currently output from DSI is not supported, so don't
       * fall through to the transpose & byte swapping stuff below.
       */

      return 0;
    }
  }
  else {
    /* if this is a variable-length output arg, the user has passed in
     * a double pointer.  first receive the length, malloc space for the
     * argument, and set the data pointer to the dereferenced data 
     * pointer (which is a pointer to the actual data).
     */
    if(arg->inout == GS_VAROUT) {
      char **dptr;
      int srv_data_size;
    
      if(!arg->data) {
        arg->data = (char **) malloc(sizeof(char *));
        if(!arg->data) {
          ERRPRINTF("malloc\n");
          return -1;
        }
      }

      dptr = arg->data;

      if(gs_recv_int(sock, &srv_data_size) < 0) {
        ERRPRINTF("Error reading arg size\n");
        return -1;
      }

      data_size = srv_data_size;

      *dptr = (char *)malloc(data_size);
      data = *dptr;

      if(!data) {
        ERRPRINTF("error allocating space for arg\n");
        return -1;
      }
    }

    if(!arg->data) {
      arg->data = (char *)malloc(data_size);

      if(!arg->data) {
        ERRPRINTF("error allocating space for argument ");
        ERRPRINTF("%s (rows %s (%d) cols %s (%d))(%d bytes)\n", 
          arg->name, arg->rowexp, arg->rows, arg->colexp, arg->cols, data_size);
        return -1;
      }

      data = arg->data;
    }

    if(gs_tread(sock, data, data_size) == -1) {
      ERRPRINTF("error in receiving argument ");
      ERRPRINTF("%s (rows %s (%d) cols %s (%d))(%d bytes)\n", 
        arg->name, arg->rowexp, arg->rows, arg->colexp, arg->cols, data_size);
      return -1;
    }
  }

  if((arg->objecttype == GS_MATRIX) && (sender_major != arg->prob->major))
    gs_transpose_matrix(arg, data, side);
  if(sender_dsig != my_dsig)
    if(gs_convert_arg(arg, data, sender_dsig, my_dsig) < 0)
      return -1;

  return 0;
}

int
gs_send_arg_from_file(SOCKET sock, char *filename)
{
  struct stat st;
  char fxbuf[FILE_XFER_BUFSZ];
  int n, fd;

  if(stat(filename, &st) == -1) {
    ERRPRINTF("Could not stat file '%s'\n", filename);
    return -1;
  }

  fd = open(filename, O_RDONLY, 0666);
  if(fd < 0) {
    ERRPRINTF("File '%s' could not be opened\n", filename);
    return -1;
  }

  /* should probably change this at some point to send_long to
   * support large file sizes.
   */
  if(gs_send_int(sock, (int)st.st_size) < 0) {
    ERRPRINTF("error in sending file len\n");
    close(fd);
    return -1;
  }

  while((n=read(fd, fxbuf, FILE_XFER_BUFSZ)) > 0) {
    if(gs_twrite(sock, fxbuf, n) != n) {
      ERRPRINTF("error in sending argument data\n");
      close(fd);
      return -1;
    }
  }

  close(fd);

  if(n < 0) {
    ERRPRINTF("Error reading from file\n");
    return -1;
  }

  return 0;
}

int 
gs_send_dsi_input_arg(SOCKET sock, gs_argument_t *arg, int my_dsig)
{
  int data_size;
  char *data_encoding;

  if(!arg || !arg->data)
    return -1;

  data_size = gs_get_element_size(arg->datatype, my_dsig) *
       arg->rows * arg->cols;

  DBGPRINTF("sending DSI arg %s, type=%s, rows=%d, cols=%d, bytes=%d\n",
    arg->name, gs_c_datatype[arg->datatype], arg->rows, arg->cols, data_size);

  if(gs_encode_dsi_object(&data_encoding, arg->data) < 0) {
    ERRPRINTF("Failed to encode DSI object\n");
    return -1;
  }

  if(gs_send_string(sock, data_encoding) < 0) {
    ERRPRINTF("failed to send encoded DSI object\n");
    free(data_encoding);
    return -1;
  }

  free(data_encoding);

  return 0;
}

int
gs_send_dsi_output_arg(SOCKET sock, gs_argument_t *arg, int my_dsig)
{
  int data_size;
  char *data_encoding;

  if(!arg || !arg->data)
    return -1;

  data_size = gs_get_element_size(arg->datatype, my_dsig) *
       arg->rows * arg->cols;

  DBGPRINTF("sending DSI arg %s, type=%s, rows=%d, cols=%d, bytes=%d\n",
    arg->name, gs_c_datatype[arg->datatype], arg->rows, arg->cols, data_size);

  if(gs_encode_dsi_object(&data_encoding, arg->dsi_obj) < 0) {
    ERRPRINTF("Failed to encode DSI object\n");
    return -1;
  }

  if(gs_send_string(sock, data_encoding) < 0) {
    ERRPRINTF("failed to send encoded DSI object\n");
    free(data_encoding);
    return -1;
  }

  free(data_encoding);

  return 0;
}

int
gs_recv_dsi_input_arg(SOCKET sock, gs_argument_t *arg, int data_size)
{
  DSI_OBJECT *obj;
  int bytes_read;
  char *encoding;
  void *data = NULL;

  data = (void *)malloc(data_size);

  if(!data) {
    ERRPRINTF("failed to allocate data\n");
    return -1;
  }

  obj = (DSI_OBJECT *)calloc(1, sizeof(DSI_OBJECT));

  if(!obj) {
    ERRPRINTF("malloc failed\n");
    return -1;
  }

  if(gs_recv_string(sock, &encoding) < 0) {
    ERRPRINTF("error in receiving DSI object encoding\n");
    return -1;
  }

  if(gs_decode_dsi_object(encoding, obj) < 0) {
    ERRPRINTF("error decoding DSI object\n");
    free(encoding);
    free(data);
    free(obj);
    return -1;
  }

  arg->dsi_obj = obj;

  if(grpc_dsi_read_vector(obj, data, arg->rows * arg->cols,
        arg->datatype, &bytes_read) != GRPC_NO_ERROR)
  {
    ERRPRINTF("error reading DSI object\n");
    free(encoding);
    free(data);
    free(obj);
    return -1;
  }

  if(!data) {
    ERRPRINTF("Empty data after reading DSI object\n");
    free(encoding);
    free(data);
    free(obj);
    return -1;
  }

  arg->data = (char *)data;

  free(encoding);

  return 0;
}

int
gs_recv_dsi_output_arg(SOCKET sock, gs_argument_t *arg, int data_size)
{
  char *encoding;

  if(gs_recv_string(sock, &encoding) < 0) {
    ERRPRINTF("error in receiving DSI object encoding\n");
    return -1;
  }

  /* currently the returned DSI object is ignored since we
   * expect it to be the same, but later if we want to
   * support having the server create the DSI object and
   * return it to the client, we can add code here.
   */

  free(encoding);

  return 0;
}

int 
gs_send_arg(SOCKET sock, gs_argument_t *arg, int my_dsig)
{
  int data_size;
  char *data;

  if(!arg || !arg->data)
    return -1;

  data_size = gs_get_element_size(arg->datatype, my_dsig) *
       arg->rows * arg->cols;

  data = arg->data;
  
  DBGPRINTF("sending arg %s, type=%s, rows=%d, cols=%d, bytes=%d\n",
    arg->name, gs_c_datatype[arg->datatype], arg->rows, arg->cols, data_size);

  if(arg->objecttype == GS_FILE) {
    if(gs_send_arg_from_file(sock, data) < 0) {
      ERRPRINTF("error arg data from file '%s'\n", data);
      return -1;
    }

    return 0;
  }
  else if(arg->objecttype == GS_PACKEDFILE) {
    char **filenames;
    int i;

    filenames = (char **)arg->data;

    for(i=0; i < arg->rows; i++)
      if(gs_send_arg_from_file(sock, filenames[i]) < 0) {
        ERRPRINTF("error arg data from file '%s'\n", filenames[i]);
        return -1;
      }

    return 0;
  }

  if(arg->inout == GS_VAROUT) {
    if(gs_send_int(sock, data_size) < 0) {
      ERRPRINTF("error in sending arg len\n");
      return -1;
    }

    if(arg->objecttype != GS_SCALAR)
      data = *((char **)(arg->data));
  }

  if(gs_twrite(sock, data, data_size) != data_size) {
    ERRPRINTF("error in sending argument data\n");
    return -1;
  }

  return 0;
}

int 
gs_convert_arg(gs_argument_t *arg, char *data, int sender_dsig, int my_dsig)
{
  int i;

  if(!arg || !arg->data)
    return -1;

  switch(arg->datatype) {
    case GS_INT:
      if(DSIG_INT_SIZE(sender_dsig) == DSIG_INT_SIZE(my_dsig)) {
        for(i=0; i< arg->rows * arg->cols; i++)
          gs_int_swap_func[DSIG_INT_ORDER(sender_dsig)][DSIG_INT_ORDER(my_dsig)](
              ((int *)data)+i);
      }
      else {
        ERRPRINTF("Error: different int sizes not supported yet\n");
        return -1;
      }
      break;
    case GS_FLOAT:
      if(DSIG_FLOAT_SIZE(sender_dsig) == DSIG_FLOAT_SIZE(my_dsig)) {
        for(i=0; i< arg->rows * arg->cols; i++)
          gs_float_swap_func[DSIG_FLOAT_ORDER(sender_dsig)][DSIG_FLOAT_ORDER(my_dsig)](((float *)data)+i);
      }
      else {
        ERRPRINTF("Error: different float sizes not supported yet\n");
        return -1;
      }
      break;
    case GS_DOUBLE:
      if(DSIG_DOUBLE_SIZE(sender_dsig) == DSIG_DOUBLE_SIZE(my_dsig)) {
        for(i=0; i< arg->rows * arg->cols; i++)
          gs_double_swap_func[DSIG_DOUBLE_ORDER(sender_dsig)][DSIG_DOUBLE_ORDER(my_dsig)](((double *)data)+i);
      }
      else {
        ERRPRINTF("Error: different double sizes not supported yet\n");
        return -1;
      }
      break;

    case GS_SCOMPLEX:
      if(DSIG_FLOAT_SIZE(sender_dsig) == DSIG_FLOAT_SIZE(my_dsig)) {
        gs_scomplex *iptr;

        for(i=0; i< arg->rows * arg->cols; i++) {
          iptr = ((gs_scomplex *)data) + i;

          gs_float_swap_func[DSIG_FLOAT_ORDER(sender_dsig)][DSIG_FLOAT_ORDER(my_dsig)](&(iptr->r));
          gs_float_swap_func[DSIG_FLOAT_ORDER(sender_dsig)][DSIG_FLOAT_ORDER(my_dsig)](&(iptr->i));
        }

      }
      else {
        ERRPRINTF("Error: different scomplex sizes not supported yet\n");
        return -1;
      }
      break;
    case GS_DCOMPLEX:
      if(DSIG_DOUBLE_SIZE(sender_dsig) == DSIG_DOUBLE_SIZE(my_dsig)) {
        gs_dcomplex *iptr;

        for(i=0; i< arg->rows * arg->cols; i++) {
          iptr = ((gs_dcomplex *)data) + i;

          gs_double_swap_func[DSIG_DOUBLE_ORDER(sender_dsig)][DSIG_DOUBLE_ORDER(my_dsig)](&(iptr->r));
          gs_double_swap_func[DSIG_DOUBLE_ORDER(sender_dsig)][DSIG_DOUBLE_ORDER(my_dsig)](&(iptr->i));
        }

      }
      else {
        ERRPRINTF("Error: different dcomplex sizes not supported yet\n");
        return -1;
      }
      break;
    case GS_CHAR:
      /* no conversion necessary for char */
      break;
    case GS_BAD_DTYPE:
      ERRPRINTF("Error: argument has invalid data type.\n");
      return -1;
      break;
  }

  return 0;
}

int
gs_set_dsi_flags(gs_problem_t *prob, char *callsig)
{
  gs_argument_t *argptr;
  int i;

  if(!prob) return -1;

  if(prob->callsig)
    free(prob->callsig);

  prob->callsig = callsig;

  if(!strcmp(callsig, GS_NO_CALL_SIG)) {
    for(argptr=prob->arglist; argptr != NULL; argptr=argptr->next) {
      argptr->dsi = 0;
      argptr->dsi_obj = NULL;
    }
    return 0;
  }

  i = 0;
  for(argptr=prob->arglist; argptr != NULL; argptr=argptr->next, i++) {
    if(prob->callsig) {
      if(prob->callsig[i] == 'D')
        argptr->dsi = 1;
    }
    else
      argptr->dsi = 0;

    argptr->dsi_obj = NULL;
  }
  
  return 0;
}


int gs_set_pass_back_flags(gs_problem_t *prob, char *callsig) {
    gs_argument_t *argptr;
    int i;

    if (!prob) return -1;

/* 
    if (prob->callsig)
        free(prob->callsig);

    prob->callsig = callsig;

    if (!prob->callsig) {
        fprintf(stderr, "bad call signature\n");
        return -1;
    }

    if (!strcmp(callsig, GS_NO_CALL_SIG)) {
        for (argptr = prob->arglist; argptr != NULL; argptr = argptr->next) {
            argptr->pass_back = 1;
            argptr->data_handle = 0;
        }

        return 0;
    }
*/
    
    i = 0;
    for (argptr = prob->arglist; argptr != NULL; argptr = argptr->next, i++) {
        if (prob->callsig[i] == 'R') {
            argptr->pass_back = 0;
            argptr->data_handle = 0;
        }
        else if (prob->callsig[i] == 'S') {
            argptr->pass_back = 0;
            argptr->data_handle = 1;
        }
        else if (prob->callsig[i] == 'T') {
            argptr->pass_back = 1;
            argptr->data_handle = 1;
        }
        else {
            argptr->pass_back = 1;
            argptr->data_handle = 0;
        }
    }

    return 0;
}


int
gs_set_call_signature(gs_problem_t *prob_desc)
{
  gs_argument_t *argptr;
  char *callsig;
  int i, count;

  if(!prob_desc) return -1;

  /* count args */
  count = 0;
  for(argptr=prob_desc->arglist; argptr != NULL; argptr=argptr->next)
    count++;

  callsig = (char *)malloc(count + 1);
  if(!callsig)
    return -1;

  i = 0;
  for(argptr=prob_desc->arglist; argptr != NULL; argptr=argptr->next, i++) {
    argptr->dsi = 0;
    
    if(argptr->data) {
      if(gs_is_dsi_object(argptr->data)) {
        callsig[i] = 'D';
        argptr->dsi = 1;

        if((argptr->objecttype == GS_FILE) ||
           (argptr->objecttype == GS_PACKEDFILE) ||
           (argptr->inout == GS_INOUT) ||
           (argptr->inout == GS_OUT) ||
           (argptr->inout == GS_VAROUT) ||
           (argptr->inout == GS_WORKSPACE))
        {
          ERRPRINTF("DSI only supported for INPUT variables ");
          ERRPRINTF("(offending variable = %s)\n", argptr->name);
          free(callsig);
          return -1;
        }
      }
      
      /* indicating whether an argument should be passed 
         back to the client or not, and whether it is 
         replaced with data handle */
      /* modified for GridSolve request sequencing */
      if (!argptr->pass_back && !argptr->data_handle) {
          callsig[i] = 'R';
      }
      else if (!argptr->pass_back && argptr->data_handle) {
          callsig[i] = 'S';
      }
      else if (argptr->pass_back && argptr->data_handle) {
          callsig[i] = 'T';
      }
      else
        callsig[i] = 'P';
    }
    else
      callsig[i] = 'N';

    argptr->dsi_obj = NULL;
  }

  callsig[i] = 0;

  if(prob_desc->callsig)
    free(prob_desc->callsig);

  prob_desc->callsig = callsig;

  client_callsig = callsig;
  
  return 0;
}

int 
gs_send_input_scalar_args(gs_va_list *args, void **argstack, SOCKET sock, 
   gs_problem_t *problem, int my_dsig, int lang, int major)
{
  gs_argument_t *argptr;

  if(!problem)
    return -1;

  /* if args and argstack are both NULL, then skip computing the arg
   * sizes since they would have been computed on the previous failed
   * submission.
   */
  if(args || argstack)
    if(gs_sender_compute_arg_sizes(args, argstack, problem, lang, major) < 0)
      return -1;

  if(gs_send_tag(sock, major) < 0) {
    ERRPRINTF("failed to send major %d\n", problem->major);
    return -1;
  }

  /* determine call signature if not already set. */

  if(!strcmp(problem->callsig, GS_NO_CALL_SIG))
    if(gs_set_call_signature(problem) < 0)
      return -1;

  if(gs_send_string(sock, problem->callsig) < 0) {
    ERRPRINTF("failed to send call signature '%s'\n", problem->callsig);
    return -1;
  }

  /* first we send all the scalar arguments so that the receiver
   * can compute the size expressions for the vector/matrix args.
   */

  for(argptr = problem->arglist; argptr != NULL; argptr = argptr->next)
  {
    if(((argptr->inout == GS_IN) || (argptr->inout == GS_INOUT)) &&
        (argptr->objecttype == GS_SCALAR))
    {
      if(gs_send_arg(sock, argptr, my_dsig) < 0) {
        ERRPRINTF("error in sending input scalar argument\n");
        return -1;
      }
    }
  }

  return 0;
}

  
int 
gs_send_input_nonscalar_args(SOCKET sock, gs_problem_t *problem, int my_dsig)
{
  gs_argument_t *argptr;
  char *data_handle;
  int i;


  i = 0;
  for(argptr = problem->arglist; argptr != NULL; argptr = argptr->next, i++)
  {
    argptr->index = i;
    if(((argptr->inout == GS_IN) || (argptr->inout == GS_INOUT)) &&
        (argptr->objecttype != GS_SCALAR))
    {
      if(argptr->dsi) {
        if(gs_send_dsi_input_arg(sock, argptr, my_dsig) < 0) {
          ERRPRINTF("error in sending input DSI argument\n");
          return -1;
        }
      }
      /* if the data to be sent is a data handle */
      /* modified for GridSolve request sequencing */
      else if (argptr->data_handle) {  
          data_handle = find_data_handle(
          problem->name, problem->seq_id, argptr->index, argptr->data);
          if (!data_handle) {
              return -1;
          }
          
          /* send the data handle to the server - string */
          if (gs_send_string(sock, data_handle) < 0) {
            fprintf(stderr, "error in sending data handle - client side\n");
            return -1;
          }
      }
      /* the normal argument case */
      else {
        if(gs_send_arg(sock, argptr, my_dsig) < 0) {
          fprintf(stderr, "error in sending input argument\n");
          return -1;
        }
      }
    }
  }
  
  return 0;
}

int 
gs_send_input_args(gs_va_list *args, void **argstack, SOCKET sock, 
   gs_problem_t *problem, int my_dsig, int lang, int major)
{

  if(!problem)
    return -1;

  /* First send the scalar arguments */
  if(gs_send_input_scalar_args(args, argstack, sock, problem,
                my_dsig, lang, major) < 0)  
    return -1;

  /* now send all the non-scalar arguments */

  if(gs_send_input_nonscalar_args(sock, problem, my_dsig) < 0)
    return -1;

  return 0;
}

int
gs_wait_for_output(SOCKET sock)
{
  int nready;

  nready = proxy_readable_timeout(sock, -1);

  return (nready > 0) ? 0 : -1;
}

int 
gs_recv_output_args(SOCKET sock, gs_problem_t *problem, int sender_dsig, 
       int my_dsig)
{
  gs_argument_t *argptr;
  char *data_handle;
  int sender_major;
  int i;
  
  if(!problem)
    return -1;

  if(gs_wait_for_output(sock) < 0) {
    ERRPRINTF("error waiting for output to be ready\n");
    return -1;
  }

  if(gs_recv_tag(sock, &sender_major) < 0) {
    ERRPRINTF("error in receiving major\n");
    return -1;
  }

  /* reconstruct the pass_back flags on the client 
     side for nonblocking sequencing calls */
  /* modified for GridSolve request sequencing */
  gs_set_pass_back_flags(problem, client_callsig);

  i = 0;
  for(argptr = problem->arglist; argptr != NULL; argptr = argptr->next, i++)
  {
    argptr->index = i;
    if((argptr->inout == GS_INOUT) || (argptr->inout == GS_OUT) ||
       (argptr->inout == GS_VAROUT)) 
    {

      /* if the argument won't be passed back, 
         expect to receive the data handle instead */
      /* modified for GridSolve request sequencing */
      if (!argptr->pass_back) {
          /* receive data handle from the server - string */
          if (gs_recv_string(sock, &data_handle) < 0) {
              ERRPRINTF("error in receiving data handle - client side\n");
              return -1;
          }

          /* save the data handle so that the depending job can get it */
          if (insert_data_handle(problem->name, problem->seq_id,
              argptr->index, argptr->data, data_handle) < 0) {
              return -1;
          }
      }
      else {
        if(gs_recv_arg(sock, argptr, sender_major, sender_dsig, my_dsig,
                GS_CLIENT_SIDE) < 0) 
        {
            ERRPRINTF("error in receiving output arguments\n");
            return -1;
        }
      }
    }
  }

  return 0;
}

int 
gs_send_output_args(SOCKET sock, gs_problem_t *problem, int my_dsig)
{
  gs_argument_t *argptr;
  char *full_name, *path, *file_name, *data_handle;
  LFS_DSI_OBJECT *obj;
  int i;
  
  if(!problem)
    return -1;

  if(gs_send_tag(sock, problem->major) < 0) {
    ERRPRINTF("failed to send major\n");
    return -1;
  }

  if(gs_receiver_compute_arg_sizes(problem, GS_OUT) < 0)
  {
    ERRPRINTF("could not compute argument sizes.\n");
    return -1;
  } 

  /* reconstruct the server-side pass-back flags */
  /* modified for GridSolve request sequencing */
  gs_set_pass_back_flags(problem, server_callsig);

  i = 0;
  for(argptr = problem->arglist; argptr != NULL; argptr = argptr->next, i++)
  {
    /* the index of the argument in argument list */
    argptr->index = i;
    if((argptr->inout == GS_INOUT) || (argptr->inout == GS_OUT) ||
       (argptr->inout == GS_VAROUT))
    {
      if(argptr->dsi) {
        if(gs_send_dsi_output_arg(sock, argptr, my_dsig) < 0) {
          ERRPRINTF("error in sending output DSI argument\n");
          return -1;
        }
      }

      /* if the argument to be sent back is a data handle */
      /* modified for GridSolve request sequencing */
      if (!argptr->pass_back) {
        struct sockaddr_in addr;
        socklen_t addr_length;
        
        /* get the ip address of this host */
        if (getsockname(sock, (struct sockaddr *) &addr, &addr_length) < 0) {
            fprintf(stderr, "error getting local host name\n");
            return -1;
        }
        argptr->hostname = inet_ntoa(addr.sin_addr);

        /* generate a unique file name */
        file_name = gs_seq_create_unique_file_name(argptr);

        if (!file_name) {
            fprintf(stderr, "error generating unique data file name\n");
            return -1;
        }

        if (!DATA_SEND_FILE_PATH) {
            gs_seq_set_lfs_dsi_data_storage_path();
        }

        path = DATA_SEND_FILE_PATH;
        
        /* construct the full path name */
        full_name = (char *) malloc(sizeof(char)
        * (strlen(path) + strlen(file_name) + 1));
        if (!full_name) {
            perror("malloc");
            return -1;
        }
        
        strcpy(full_name, path);
        full_name[strlen(path)] = '\0';
        
        strcat(full_name, file_name);
        full_name[strlen(path) + strlen(file_name)] = '\0';

        /* save the data to the file */
        if (gs_seq_save_data_into_file(
            full_name, argptr, problem->major, my_dsig) < 0) {
            fprintf(stderr, "error saving data to file\n");
            return -1;
        }


        /* create a data handle for the file */
        obj = gs_seq_create_data_handle(
        argptr, path, file_name, problem->major);
        if (!obj) {
            fprintf(stderr, "error creating data handle\n");
            return -1;
        }
        
        /* encode the data handle */
        data_handle = gs_seq_encode_lfs_dsi_object(obj);

        /* make sure the data handle is ok */
        if (!data_handle) {
            fprintf(stderr, "bad data handle - server side\n");
            return -1;
        }

        /* send the data handle back to the client - string */
        if (gs_send_string(sock, data_handle) < 0) {
            fprintf(stderr, "error in sending data handle - server side\n");
            return -1;
        }
      }
      else {
        if(gs_send_arg(sock, argptr, my_dsig) < 0) {
          fprintf(stderr, "error in sending output arguments\n");
          return -1;
        }
      }
    }
  }

  return 0;
}

int 
gs_recv_input_scalar_args(SOCKET sock, gs_problem_t *problem, int sender_dsig, 
      int my_dsig, int *sender_major_p)
{
  gs_argument_t *argptr;
  char *sender_callsig;
  int sender_major;

  if(!problem)
    return -1;

  if(gs_recv_tag(sock, &sender_major) < 0) {
    ERRPRINTF("error in receiving major\n");
    return -1;
  }
  *sender_major_p = sender_major;

  if(gs_recv_string(sock, &sender_callsig) < 0) {
    ERRPRINTF("error in receiving sender's call signature\n");
    return -1;
  }

  gs_set_dsi_flags(problem, sender_callsig);
  
  gs_set_pass_back_flags(problem, sender_callsig); 
  server_callsig = strdup(sender_callsig);
  
  /* first we receive all the scalar arguments so that the we
   * can compute the size expressions for the vector/matrix args.
   */

  for(argptr = problem->arglist; argptr != NULL; argptr = argptr->next)
  {
    if(((argptr->inout == GS_IN) || (argptr->inout == GS_INOUT)) &&
        (argptr->objecttype == GS_SCALAR))
    {

      argptr->rows = argptr->cols = 1;
      argptr->data = NULL;

      if(gs_recv_arg(sock, argptr, sender_major, sender_dsig, my_dsig, 
            GS_SERVER_SIDE) < 0) 
      {
        ERRPRINTF("error in receiving input scalar argument\n");
        return -1;
      }
    }
  }

  return 0;
}

int 
gs_recv_input_args(SOCKET sock, gs_problem_t *problem, int sender_dsig,
      int my_dsig)
{
  gs_argument_t *argptr;
  int sender_major;
  char *data_handle;

  if(!problem)
    return -1;

  /* Receive scalar input args */
  if (gs_recv_input_scalar_args(sock, problem, sender_dsig,
         my_dsig, &sender_major) < 0)
  {
    ERRPRINTF("Error in receiving input scalar args\n");
    return -1;
  }

  /* once the scalar arguments have been received, compute the sizes
   * of the other args so we'll know how many bytes to receive.
   */
  if(gs_receiver_compute_arg_sizes(problem, GS_IN) < 0)
  {
    ERRPRINTF("error computing argument sizes.\n");
    return -1;
  }

  /* now receive all the non-scalar arguments */

  for(argptr = problem->arglist; argptr != NULL; argptr = argptr->next)
  {
    if(((argptr->inout == GS_IN) || (argptr->inout == GS_INOUT)) &&
        (argptr->objecttype != GS_SCALAR))
    {
      /*argptr->data = NULL;*/
      /* if the data to be received is a data handle */
      /* modified for GRidSolve request sequencing */
      if (argptr->data_handle) {
        /* receive data handle from the client - string */
        if (gs_recv_string(sock, &data_handle) < 0) {
            ERRPRINTF("error receiving data handle - server side\n");
            return -1;
        }

        if (!data_handle) {
            ERRPRINTF("bad data handle\n");
            return -1;
        }

        /* requrst a data transfer from the server that stores
           the actual data represented by the data handle */
        if (gs_send_data_transfer_request(data_handle, argptr, my_dsig) < 0) {
            return -1;
        }
      }
      else {
        if(gs_recv_arg(sock, argptr, sender_major, sender_dsig, my_dsig,
            GS_SERVER_SIDE) < 0)
        {
          ERRPRINTF("error receiving input non-scalar arguments\n");
          return -1;
        }
      }
    }

    if((argptr->objecttype == GS_FILE) && (argptr->inout == GS_OUT)) {
      argptr->data = strdup(argptr->name);

      /* since this is an output-only file, then set the filename to be
       * the arg name here.
       */

      if(!argptr->data) {
        ERRPRINTF("error allocating space for output arg\n");
        return -1;
      }
    }
    else if((argptr->objecttype == GS_PACKEDFILE) && (argptr->inout == GS_OUT)) {
      char **data;
      int i;

      argptr->data = malloc(argptr->rows * sizeof(char *));
      if(!argptr->data) {
        ERRPRINTF("error allocating memory for filenames.\n");
        return -1;
      }

      data = (char **)argptr->data;

      for(i = 0; i < argptr->rows; i++) {
        data[i] = dstring_sprintf("%s_%d", argptr->name, i);
        if(!data[i]) {
          ERRPRINTF("error allocating memory for filename.\n");
          return -1;
        }
      }
    }
    else if((argptr->inout == GS_OUT) || (argptr->inout == GS_WORKSPACE)) {
      int elsize;

      /* since this is an output-only or workspace arg, allocate memory for 
       * it now since nothing would have been sent from the client.
       */

      elsize = gs_get_element_size(argptr->datatype, my_dsig);

      argptr->data = (char *)CALLOC(argptr->rows * argptr->cols, elsize);

      if(!argptr->data) {
        ERRPRINTF("error allocating space for output arg\n");
        return -1;
      }
    }
    else if(argptr->inout == GS_VAROUT)
    {
      char **foo = (char **)malloc(sizeof(char*));
      argptr->data = foo;
    }
  }

  return 0;
}

int 
gs_dump_args(FILE *out, gs_problem_t *problem) 
{
  gs_argument_t *args;
  int i=0;

  if(!out || !problem)
    return -1;

  fprintf(out, "Arguments:\n");

  for(args = problem->arglist; args != NULL; args = args->next)
  {
    fprintf(out, "arg %d: name='%s', rowexp='%s', ",
       i, args->name, args->rowexp);
    fprintf(out, "colexp='%s', rows=%d, cols=%d, major=%c\n",
       args->colexp, args->rows, args->cols, args->prob->major);

    if(!args->data) continue;

    switch(args->datatype) {
      case GS_INT:
        for(i=0;i<args->rows*args->cols;i++)
          fprintf(out,"%d\n", ((int*)args->data)[i]);
        break;
      case GS_FLOAT:
        for(i=0;i<args->rows*args->cols;i++)
          fprintf(out,"%f\n", ((float*)args->data)[i]);
        break;
      case GS_DOUBLE:
        for(i=0;i<args->rows*args->cols;i++)
          fprintf(out,"%g\n", ((double*)args->data)[i]);
        break;
      case GS_SCOMPLEX:
        for(i=0;i<args->rows*args->cols;i++)
          fprintf(out,"%g + %gi\n", ((gs_scomplex*)args->data)[i].r,
            ((gs_scomplex*)args->data)[i].i);
        break;
      case GS_DCOMPLEX:
        for(i=0;i<args->rows*args->cols;i++)
          fprintf(out,"%g + %gi\n", ((gs_dcomplex*)args->data)[i].r,
            ((gs_dcomplex*)args->data)[i].i);
        break;
      case GS_CHAR:
        for(i=0;i<args->rows*args->cols;i++)
          fprintf(out,"%c\n", ((char*)args->data)[i]);
        break;
      case GS_BAD_DTYPE:
        fprintf(out,"[INVALID DATA TYPE]\n");
        break;
    }
  }

  return 0;
}

int
gs_construct_scalar_hashtable(icl_hash_t **new_hash, gs_problem_t *prob_desc, 
  enum inout cond)
{
  gs_argument_t *argptr;
  icl_hash_t *symtab;

  if(!prob_desc)
    return -1;

  symtab = icl_hash_create(11, NULL);

  if(!symtab)
    return -1;

  for(argptr = prob_desc->arglist; argptr != NULL; argptr=argptr->next)
  {
    if(((argptr->inout == GS_IN) || (argptr->inout == cond) || 
        (argptr->inout == GS_INOUT)) && (argptr->objecttype == GS_SCALAR))
    {
      if(!argptr->data) {
        icl_hash_destroy(symtab, NULL, NULL);
        return -1;
      }

      switch(argptr->datatype) {
        case GS_INT:
          argptr->expr_val = (double) *((int *)(argptr->data));
          break;
        case GS_CHAR:
          argptr->expr_val = (double) *((char *)(argptr->data));
          break;
        case GS_FLOAT:
          argptr->expr_val = (double) *((float *)(argptr->data));
          break;
        case GS_DOUBLE:
          argptr->expr_val = (double) *((double *)(argptr->data));
          break;
        case GS_SCOMPLEX:
          argptr->expr_val = (double) ((gs_scomplex *)(argptr->data))->r;
          break;
        case GS_DCOMPLEX:
          argptr->expr_val = (double) ((gs_dcomplex *)(argptr->data))->r;
          break;
        default:
          ERRPRINTF("Bad data type\n");
          icl_hash_destroy(symtab, NULL, NULL);
          return -1;
      }
    }
    else
      argptr->expr_val = 0.0;

    icl_hash_insert(symtab, argptr->name, &(argptr->expr_val));
  }

  *new_hash = symtab;

  return 0;
}

int
gs_receiver_compute_arg_sizes(gs_problem_t *prob_desc, enum inout cond)
{
  icl_hash_t *symtab;

  if(!prob_desc)
    return -1;

  if(gs_construct_scalar_hashtable(&symtab, prob_desc, cond) < 0) {
    ERRPRINTF("error setting up hash table.\n");
    return -1;
  }

  if(gs_setup_workspace(prob_desc, symtab) < 0) {
    ERRPRINTF("error setting up workspace for matrix transpose.\n");
    return -1;
  }

  icl_hash_destroy(symtab, NULL, NULL);

  return 0;
}

int
gs_sender_compute_arg_sizes(gs_va_list *ap, void **argstack, gs_problem_t *prob_desc, 
  int language, int major)
{
  gs_argument_t *argptr;
  icl_hash_t *symtab;
  int i, j;

  if (prob_desc->size_computed) return 0;

  if(!prob_desc)
    return -1;

  symtab = icl_hash_create(11, NULL);

  if(!symtab)
    return -1;

  prob_desc->major = major;

  i = -1;
  j = 0;
  for(argptr=prob_desc->arglist; argptr != NULL; argptr=argptr->next)
  {
    argptr->prob = prob_desc;

    /* if this is workspace, there will be no real argument passed in the 
     * calling sequence, so do nothing here.
     */
    if(argptr->inout == GS_WORKSPACE) {
      argptr->data = NULL;
      continue;
    }

    /* this is a real arg, so increment the index and continue */
    i++;

    if((argptr->objecttype == GS_SCALAR) && (argptr->inout == GS_IN))
    {
      switch(argptr->datatype) {
        case GS_INT:
          if(ap) {
            if(language == GS_CALL_FROM_C) {
              argptr->scalar_val.int_val = (int)va_arg(ap->args, int);
            } else
              argptr->scalar_val.int_val = *((int*)va_arg(ap->args, int*));
          }
          else {
            argptr->scalar_val.int_val = *((int *)argstack[i]);
          }

          argptr->data = &(argptr->scalar_val.int_val);
          argptr->expr_val = (double) argptr->scalar_val.int_val;
          break;
        case GS_CHAR:
          if(ap) {
            /* note promotion to int in va_arg() for pass by value scalar */
            if(language == GS_CALL_FROM_C)
              argptr->scalar_val.char_val = (char)va_arg(ap->args, int);
            else
              argptr->scalar_val.char_val = *((char*)va_arg(ap->args, char*));
          }
          else {
            argptr->scalar_val.char_val = *((char *)argstack[i]);
          }

          argptr->data = &(argptr->scalar_val.char_val);
          argptr->expr_val = (double) argptr->scalar_val.char_val;
          break;
        case GS_FLOAT:
          if(ap) {
            /* note promotion to double in va_arg() for pass by value scalar */
            if(language == GS_CALL_FROM_C)
              argptr->scalar_val.float_val = (float)va_arg(ap->args, double);
            else
              argptr->scalar_val.float_val = *((float*)va_arg(ap->args, float*));
          }
          else {
            argptr->scalar_val.float_val = *((float *)argstack[i]);
          }

          argptr->data = &(argptr->scalar_val.float_val);
          argptr->expr_val = (double) argptr->scalar_val.float_val;
          break;
        case GS_DOUBLE:
          if(ap) {
            if(language == GS_CALL_FROM_C)
              argptr->scalar_val.double_val = (double)va_arg(ap->args, double);
            else
              argptr->scalar_val.double_val = *((double*)va_arg(ap->args, double*));
          }
          else {
            argptr->scalar_val.double_val = *((double *)argstack[i]);
          }

          argptr->data = &(argptr->scalar_val.double_val);
          argptr->expr_val = argptr->scalar_val.double_val;
          break;
        case GS_SCOMPLEX:
          if(ap) {
            if(language == GS_CALL_FROM_C)
              argptr->scalar_val.scomplex_val = va_arg(ap->args, gs_scomplex);
            else
              argptr->scalar_val.scomplex_val = *((gs_scomplex*)va_arg(ap->args, gs_scomplex*));
          }
          else {
            argptr->scalar_val.scomplex_val = *((gs_scomplex *)argstack[i]);
          }

          argptr->data = &(argptr->scalar_val.scomplex_val);
          argptr->expr_val = argptr->scalar_val.scomplex_val.r;
          break;
        case GS_DCOMPLEX:
          if(ap) {
            if(language == GS_CALL_FROM_C)
              argptr->scalar_val.dcomplex_val = va_arg(ap->args, gs_dcomplex);
            else
              argptr->scalar_val.dcomplex_val = *((gs_dcomplex*)va_arg(ap->args, gs_dcomplex*));
          }
          else {
            argptr->scalar_val.dcomplex_val = *((gs_dcomplex *)argstack[i]);
          }

          argptr->data = &(argptr->scalar_val.dcomplex_val);
          argptr->expr_val = argptr->scalar_val.dcomplex_val.r;
          break;
        default:
          ERRPRINTF("Bad data type\n");
          return -1;
      }

      icl_hash_insert(symtab, argptr->name, &(argptr->expr_val));
    }
    else {
      if(argptr->datatype == GS_INT) {
        if(ap)
          argptr->data = (int*)va_arg(ap->args, int*);
        else
          argptr->data = (int*)argstack[i];
      }
      else if(argptr->datatype == GS_CHAR) {
        if(ap)
          argptr->data = (char*)va_arg(ap->args, char*);
        else
          argptr->data = (char*)argstack[i];
      }
      else if(argptr->datatype == GS_FLOAT) {
        if(ap)
          argptr->data = (float*)va_arg(ap->args, float*);
        else
          argptr->data = (float*)argstack[i];
      }
      else if(argptr->datatype == GS_DOUBLE) {
        if(ap)
          argptr->data = (double*)va_arg(ap->args, double*);
        else
          argptr->data = (double*)argstack[i];
      }
      else if(argptr->datatype == GS_SCOMPLEX) {
        if(ap)
          argptr->data = (gs_scomplex*)va_arg(ap->args, gs_scomplex*);
        else
          argptr->data = (gs_scomplex*)argstack[i];
      }
      else if(argptr->datatype == GS_DCOMPLEX) {
        if(ap)
          argptr->data = (gs_dcomplex*)va_arg(ap->args, gs_dcomplex*);
        else
          argptr->data = (gs_dcomplex*)argstack[i];
      }

      /* if this is an output only scalar, insert a 0 size in the
       * symbol table so we will know not to allocate memory for any
       * objects using this as its size.
       */
      if((argptr->objecttype == GS_SCALAR) && (argptr->inout == GS_OUT)) {
        argptr->expr_val = 0.0;
        icl_hash_insert(symtab, argptr->name, &(argptr->expr_val));
      }
    }
  }

  if(ap)
    va_end(ap->args);

  if(gs_setup_workspace(prob_desc, symtab) < 0) {
    ERRPRINTF("problem setting up workspace for matrix transpose.\n");
    return -1;
  }

  icl_hash_destroy(symtab, NULL, NULL);

  return 0;
}

int
gs_transpose_matrix(gs_argument_t *arg, char *data, gs_side_t side)
{
  int *move,mn,iwrk,iok,rows,cols;

  if(!arg)
    return -1;

  mn = arg->rows * arg->cols;

  if(!arg->prob->work) {
    iwrk = (arg->rows + arg->cols)/2;
    move = (int*)malloc(iwrk*sizeof(int));
  }
  else {
    move = arg->prob->work;
    iwrk = arg->prob->worksize;
  }

  if(!move)
    return -1;

  if(side == GS_CLIENT_SIDE) {
    rows = arg->rows;
    cols = arg->cols;
  }
  else {
    rows = arg->cols;
    cols = arg->rows;
  }

  iok = -1;

  switch(arg->datatype) {
    case GS_INT:
      itrans_(data,&rows,&cols,&mn,move,&iwrk,&iok);
      break;
    case GS_DOUBLE:
      dtrans_(data,&rows,&cols,&mn,move,&iwrk,&iok);
      break;
    case GS_DCOMPLEX:
      dctrans_(data,&rows,&cols,&mn,move,&iwrk,&iok);
      break;
    case GS_SCOMPLEX:
      sctrans_(data,&rows,&cols,&mn,move,&iwrk,&iok);
      break;
    case GS_FLOAT:
      strans_(data,&rows,&cols,&mn,move,&iwrk,&iok);
      break;
    case GS_CHAR:
      ctrans_(data,&rows,&cols,&mn,move,&iwrk,&iok);
      break;
    default:
      ERRPRINTF("gs_transpose_matrix: bad data type\n");
  }

  if(!arg->prob->work)
    free(move);

  return iok ? -1 : 0;
}

int
gs_save_input_args_to_file(char *name, gs_problem_t *problem, int my_dsig,
  int lang, int major)
{  
  gs_argument_t *argptr;
  gs_problem_t *pcopy;
  int fd;

  fd = open(name, O_WRONLY | O_CREAT | O_TRUNC, 0666);
  if(fd < 0) {
    ERRPRINTF("File '%s' could not be opened\n", name);
    return -1;
  }

  pcopy = (gs_problem_t *) malloc(sizeof(gs_problem_t));
  if(!pcopy) {
    close(fd);
    return -1;
  }

  if(gs_dup_problem(pcopy, problem) < 0) {
    ERRPRINTF("Failed to duplicate problem\n");
    close(fd);
    free(pcopy);
    return -1;
  }

  /* set the dsi flags to 0 here since this is the server side and
   * we would have already downloaded the args from dsi.
   */

  for(argptr=pcopy->arglist; argptr != NULL; argptr=argptr->next)
    argptr->dsi = 0;

  if(gs_send_input_args(NULL, NULL, fd, pcopy, my_dsig, lang, major) < 0) {
    ERRPRINTF("Failed to save input args to file\n");
    gs_free_problem(pcopy);
    close(fd);
    return -1;
  }

  gs_free_problem(pcopy);
  close(fd);
  return 0;
}

int
gs_restore_input_args_from_file(char *name, gs_problem_t *problem, int sender_dsig,
      int my_dsig)
{
  int fd;
  char *c;

  fd = open(name, O_RDONLY, 0666);
  if(fd < 0) {
    ERRPRINTF("File '%s' could not be opened\n", name);
    return -1;
  }

  /* remove all DSI flags from callsig */
  c = problem->callsig;
  while(*c) {
    if(*c == 'D')
      *c = 'P';
    c++;
  }

  if(gs_recv_input_args(fd, problem, sender_dsig, my_dsig) < 0) {
    ERRPRINTF("Error reading input args from file\n");
    return -1;
  }

  return 0;
}


int 
gs_save_output_args_to_file(int fd, gs_problem_t *problem, int my_dsig)
{
  gs_argument_t *argptr;

  if(!problem)
    return -1;

  if(gs_send_tag(fd, problem->major) < 0) {
    ERRPRINTF("failed to send major\n");
    return -1;
  }

  /* first save the scalars so we'll know the sizes of the other
   * args when we process the file later.
   */

  for(argptr = problem->arglist; argptr != NULL; argptr = argptr->next)
  {
    if(((argptr->inout == GS_IN) || (argptr->inout == GS_INOUT) ||
        (argptr->inout == GS_OUT)) && (argptr->objecttype == GS_SCALAR))
    {
      if(gs_send_arg(fd, argptr, my_dsig) < 0) {
        ERRPRINTF("error in writing input argument\n");
        return -1;
      }
    }
  }

  if(gs_receiver_compute_arg_sizes(problem, GS_OUT) < 0)
  {
    ERRPRINTF("could not compute argument sizes.\n");
    return -1;
  }

  for(argptr = problem->arglist; argptr != NULL; argptr = argptr->next)
  {
    if((argptr->inout == GS_INOUT) || (argptr->inout == GS_OUT) ||
       (argptr->inout == GS_VAROUT))
    {
      if(argptr->objecttype == GS_FILE) {
        if(gs_send_string(fd, argptr->data) < 0) {
          ERRPRINTF("error in writing file name\n");
          return -1;
        }
      }
      else if(argptr->objecttype == GS_PACKEDFILE) {
        int i;

        if(gs_send_int(fd, argptr->rows) < 0) {
          ERRPRINTF("Error sending string length\n");
          return -1;
        }

        for(i = 0; i < argptr->rows; i++) {
          if(gs_send_string(fd, ((char **)(argptr->data))[i]) < 0) {
            ERRPRINTF("error in writing file name\n");
            return -1;
          }
        }
      }
      else {
        if(gs_send_arg(fd, argptr, my_dsig) < 0) {
          ERRPRINTF("error in writing input argument\n");
          return -1;
        }
      }
    }
  }

  return 0;
}

int 
gs_restore_output_args_from_file(int fd, gs_problem_t *problem, 
  int my_dsig)
{   
  gs_argument_t *argptr;
  int sender_major;

  if(!problem)
    return -1;

  if(gs_recv_tag(fd, &sender_major) < 0) {
    ERRPRINTF("error in receiving major\n");
    return -1;
  }
  
  for(argptr = problem->arglist; argptr != NULL; argptr = argptr->next)
  {
    if(((argptr->inout == GS_IN) || (argptr->inout == GS_INOUT) ||
        (argptr->inout == GS_OUT)) && (argptr->objecttype == GS_SCALAR))
    {
      argptr->rows = argptr->cols = 1;
      argptr->data = NULL;

      if(gs_recv_arg(fd, argptr, sender_major, my_dsig, my_dsig,
            GS_CLIENT_SIDE) < 0)
      {
        ERRPRINTF("error in reading input argument\n");
        return -1;
      }
    }
  }

  if(gs_receiver_compute_arg_sizes(problem, GS_OUT) < 0)
  {
    ERRPRINTF("could not compute argument sizes.\n");
    return -1;
  }

  for(argptr = problem->arglist; argptr != NULL; argptr = argptr->next)
  { 
    if((argptr->inout == GS_INOUT) || (argptr->inout == GS_OUT) ||
       (argptr->inout == GS_VAROUT)) 
    {
      if(argptr->objecttype == GS_FILE) {
        char *fname;

        if(gs_recv_string(fd, &fname) < 0) {
          ERRPRINTF("error in reading file name\n");
          return -1;
        }

        argptr->data = fname;
      }
      else if(argptr->objecttype == GS_PACKEDFILE) {
        char *fname;
        int i, n;

        if(gs_recv_int(fd, &n) < 0) {
          ERRPRINTF("error in reading number of packed files\n");
          return -1;
        }

        argptr->data = malloc(n * sizeof(char *));

        for(i = 0; i < n; i++) {
          if(gs_recv_string(fd, &fname) < 0) {
            ERRPRINTF("error in reading file name\n");
            return -1;
          }

          ((char **)(argptr->data))[i] = fname;
        }
      }
      else {
        if(gs_recv_arg(fd, argptr, sender_major, my_dsig, my_dsig,
            GS_CLIENT_SIDE) < 0)
        {
          ERRPRINTF("error in reading input argument\n");
          return -1;
        }
      }
    }
  }

  return 0;
}

int 
gs_setup_workspace(gs_problem_t *prob_desc, icl_hash_t *symtab)
{
  int worksz, row_max = 0, col_max = 0;
  gs_argument_t *argptr;

  if(!prob_desc)
    return -1;

  for(argptr = prob_desc->arglist; argptr != NULL; argptr=argptr->next)
  {

    /* Evaluate the row/col values */
    if((gs_expr_i(argptr->rowexp, &(argptr->rows), symtab) < 0) ||
       (gs_expr_i(argptr->colexp, &(argptr->cols), symtab) < 0))
    {
      ERRPRINTF("Error parsing dimension expression\n");
      return -1;
    }

    /* If sparse matrix, evaluate nnz, then save and overwrite row/col with that info */
    if(argptr->objecttype == GS_SPARSEMATRIX) {
      if((gs_expr_i(argptr->sparse_attr.nnzexp, &(argptr->sparse_attr.nnz), symtab) < 0)) {
        ERRPRINTF("Error parsing NNZ expression\n");
        return -1;
      }
      /* TODO Save the actual row/col values before overwriting with nnz */
      argptr->sparse_attr.rows_val_saved = argptr->rows;
      argptr->sparse_attr.cols_val_saved = argptr->cols;
      argptr->rows = argptr->sparse_attr.nnz;
      argptr->cols = 1;
    }
    
    if(argptr->rows > row_max)
      row_max = argptr->rows;
    if(argptr->cols > col_max)
      col_max = argptr->cols;
  }
  
  worksz = (row_max+col_max)/2;

  /* check if there was a previous request using this handle and 
   * malloc new memory or reuse the workspace if possible.
   */

  if(!prob_desc->work) {
    prob_desc->work = (int *)malloc(worksz *sizeof(int));
    prob_desc->worksize = worksz;
  }
  else if(prob_desc->worksize < worksz) {
    prob_desc->work = (int *)realloc(prob_desc->work, worksz *sizeof(int));
    prob_desc->worksize = worksz;
  }

  if(!prob_desc->work)
    prob_desc->worksize = 0;

  return 0;
}






#ifdef GS_SMART_GRIDSOLVE


int gs_smart_send_tg(SOCKET sock, gs_smart_tg * task_graph){
 
 gs_smart_tg_task_node * task_node;
 gs_smart_tg_remote_task_node * tg_rem_task_node;
 char * task_node_str=NULL;
 char * problem_str=NULL;
 char * server_str=NULL;
 int i, j;
 if(!task_graph) return -1;

 if(gs_send_int(sock, task_graph->nb_nodes) < 0) {
    DBGPRINTF("failed to send number of task nodes\n");
    return -1;
  }

 if(gs_send_int(sock, task_graph->nb_rem_nodes) < 0) {
    DBGPRINTF("failed to send number of task nodes\n");
    return -1;
  }

 
  for(i=0; i< task_graph->nb_nodes;i++){
     if(!task_graph->task_nodes[i]) return -1;

     task_node=task_graph->task_nodes[i];

     if(gs_send_int(sock,task_node->node_type) < 0) {
       ERRPRINTF("SMART: Failed to send task local\n");
       return -1;
     }
     if(gs_send_int(sock,task_node->id) < 0) {
       ERRPRINTF("SMART: Failed to send task local\n");
       return -1;
     }

    if(task_node->node_type==GS_SMART_TG_REM_TASK_NODE){

       if(!task_node->tg_rem_task_node) return -1;

       tg_rem_task_node=task_node->tg_rem_task_node;
       if(gs_smart_encode_tg_rem_task_node(&task_node_str, task_node)<0){ 
         ERRPRINTF("SMART: Error encoding remote task node\n");
         if(task_node_str) free(task_node_str);
         return -1;
       }
       if(!task_node_str) return -1;

       if(gs_send_string(sock, task_node_str) < 0) {
         ERRPRINTF("Unsuccessful (Sending Rem Task Node to Client)\n");
         if(task_node_str) free(task_node_str);
         return -1;
       }
       if(task_node_str) free(task_node_str);

       if(!tg_rem_task_node->problem) return -1;

       if(gs_encode_problem(&problem_str, tg_rem_task_node->problem)<0){
         ERRPRINTF("SMART: Error encoding problem\n");
         if(problem_str) free(problem_str);
         return -1;
       }
       if(!problem_str) return -1;

       if(gs_send_string(sock, problem_str) < 0) {
         ERRPRINTF("Unsuccessful (Sending Rem Problem STring to Client)\n");
         free(problem_str);
         return -1;
       }

       if(problem_str) free(problem_str);

       for(j=0;j<tg_rem_task_node->nb_servers;j++){
         if(gs_encode_server(&server_str, tg_rem_task_node->avail_servers[j])<0){
           ERRPRINTF("SMART: Error  encoding server\n");
           return -1;
         }
         if(gs_send_string(sock, server_str) < 0) {
           ERRPRINTF("Unsuccessful (Sending Server String to Client)\n");
           free(server_str);
           return -1;
         }
         if(server_str) free(server_str);
         if(gs_send_string(sock, tg_rem_task_node->avail_servers[j]->my_ping_str) < 0) {
           ERRPRINTF("Unsuccessful (Sending Server String to Client)\n");
           return -1;
         }
       }
    }
    else if(task_node->node_type==GS_SMART_TG_CLIENT_NODE){
       if(gs_smart_encode_tg_client_node(&task_node_str, task_node)<0){
         ERRPRINTF("SMART: Error encoding client task node\n");
         return -1;
       }
       if(!task_node_str) return -1;

       if(gs_send_string(sock, task_node_str) < 0) {
         ERRPRINTF("Unsuccessful (Sending Rem Task Node to Client)\n");
         free(task_node_str);
         return -1;
       }

     }

  }
  return 0;

}


int gs_smart_recv_tg(SOCKET sock, gs_smart_tg * task_graph){
  int i;
  gs_smart_tg_task_node * task_node;
 gs_smart_tg_remote_task_node * tg_rem_task_node;
  char * task_node_str=NULL;
  char * problem_str=NULL;
  int nb_servers;
  int j;
  char * server_str=NULL;
  if(!task_graph) return -1;

  if(gs_recv_int(sock, &task_graph->nb_nodes) < 0){
    return -1;
  }

  if(gs_recv_int(sock, &task_graph->nb_rem_nodes) < 0){
    return -1;
  }

  task_graph->task_nodes = (gs_smart_tg_task_node **)
                    calloc(task_graph->nb_nodes, sizeof(gs_smart_tg_task_node *));

  if(!task_graph->task_nodes) return -1;
  
  for(i=0; i< task_graph->nb_nodes;i++){
    task_graph->task_nodes[i]= (gs_smart_tg_task_node *)
                                 calloc(1, sizeof(gs_smart_tg_task_node ));

    if(!task_graph->task_nodes[i]) return -1;

    task_node=task_graph->task_nodes[i];
    if(gs_recv_int(sock, &task_node->node_type) < 0){
      return -1;
    }

    if(gs_recv_int(sock, &task_node->id) < 0){
      return -1;
    }
    
    if(task_node->node_type==GS_SMART_TG_REM_TASK_NODE){
      task_node->tg_rem_task_node=(gs_smart_tg_remote_task_node *)
                                   calloc(1, sizeof(gs_smart_tg_remote_task_node));

      if(!task_node->tg_rem_task_node) return -1;
      
      tg_rem_task_node=task_node->tg_rem_task_node;

      if(gs_recv_string(sock, &(task_node_str)) < 0) {
        ERRPRINTF("SMART: gs_recv_string task_node_str\n");
        return -1;
      }
      if(gs_smart_decode_tg_rem_task_node(task_node_str, task_graph, task_node)<0){
        free(task_node_str);
        ERRPRINTF("SMART: gs_decode_rem_task_node error\n");
        return -1;
      }

      
      if(task_node_str) free(task_node_str);
      if(gs_recv_string(sock, &(problem_str)) < 0) {
        DBGPRINTF("SMART: gs_recv_string problem_str\n");
        return -1;
      }
      tg_rem_task_node->problem = (gs_problem_t *) 
                                              CALLOC(1, sizeof(gs_problem_t));

      if(!tg_rem_task_node->problem) return -1;
      
      if(gs_decode_problem(problem_str, tg_rem_task_node->problem)<0){
        free(problem_str);
        DBGPRINTF("gs_decode_problem error\n");
        return -1;
      }
      free(problem_str);
      nb_servers=tg_rem_task_node->nb_servers;
      tg_rem_task_node->avail_servers=(gs_server_t **) 
                                     CALLOC(nb_servers, sizeof(gs_server_t *));

      for(j=0;j<nb_servers;j++){
        if(gs_recv_string(sock, &(server_str)) < 0) {
          ERRPRINTF("SMART Error: Unsuccessful (Receiving Server String)\n");
          free(server_str);
          return -1;
        }

        tg_rem_task_node->avail_servers[j]= (gs_server_t *) 
                                             CALLOC(1, sizeof(gs_server_t));

        if(gs_decode_server(server_str, tg_rem_task_node->avail_servers[j])){
          ERRPRINTF("SMART Error: Unsuccessful (Decoding Server)\n");
          free(server_str);
          return -1;
        }
        free(server_str);
        if(gs_recv_string(sock, &(tg_rem_task_node->avail_servers[j]->my_ping_str)) < 0) {
          ERRPRINTF("SMART : Unsuccessful (Receiving Server String)\n");
          return -1;
        }
      }
    }
    else if(task_graph->task_nodes[i]->node_type==GS_SMART_TG_CLIENT_NODE){
      task_graph->task_nodes[i]->tg_client_node=(gs_smart_tg_client_node *)
                                                    calloc(1, sizeof(gs_smart_tg_client_node));

     if(!task_graph->task_nodes[i]->tg_client_node){
        fprintf(stderr, "Smart Error : Malloc client task node %d\n", i);
        return -1;
      }

      if(gs_recv_string(sock, &(task_node_str)) < 0) {
        DBGPRINTF("gs_recv_string task_node_str\n");
        return -1;
      }
      if(gs_smart_decode_tg_client_node(task_node_str, task_graph, task_graph->task_nodes[i])<0){
        free(task_node_str);
        ERRPRINTF("SMART: gs_decode_loc_task_node error\n");
        return -1;
      }
      if(task_node_str) free(task_node_str);

    }

  }
  return 0;
}










int
gs_smart_send_all_servers(SOCKET sock, gs_server_t **server_list, int count)
{
  int i;
  if(gs_send_int(sock, count) < 0) {
    DBGPRINTF("failed to send number of servers\n");
    return -1;
  }

  for(i = 0; i < count; i++) {
    char *srv = NULL;
    DBGPRINTF("Encoding server: %s.\n", server_list[i]->hostname);
    if(gs_encode_server(&srv, server_list[i]) < 0){
      FREE(srv);
      DBGPRINTF("Failed to send server list \n");
      return -1;
    }
    if(gs_send_string(sock, srv) < 0) {
      FREE(srv);
      DBGPRINTF("Failed to send server list \n");
      return -1;
    }


    if(server_list[i]->smart==1){
      if(gs_send_string(sock,  server_list[i]->my_ping_str) < 0) {
        FREE(srv);
        DBGPRINTF("Failed to send server list \n");
        return -1;
      }
    }
    FREE(srv);
  }
/*
  for(i = 0; i < count; i++)
    gs_server_free(server_list[i]);
  FREE(server_list);
*/
  return 0;
}






int gs_smart_recv_all_servers(SOCKET sock, gs_server_t *** _server_list, 
                                                      int *_num_servers){
  char *msg=NULL;
  int i,num_servers;
  if(gs_recv_int(sock, &num_servers) < 0){
    ERRPRINTF("SMART: Error receiving number of servers");
    return -1;
  } 
  if(num_servers <= 0) {
    ERRPRINTF("SMART: Error no servers are currently registered\n");
    return -1;
  }
  gs_server_t ** server_list;
  server_list = (gs_server_t **) calloc(num_servers, sizeof(gs_server_t *));
  if(!server_list){
    ERRPRINTF("SMART: Error allocating memory for server list\n");
    return -1;
  }
  int smart_id=0;;
  for(i=0;i<num_servers;i++) {
    server_list[i] = (gs_server_t *) CALLOC(1,sizeof(gs_server_t));
    if(gs_recv_string(sock, &msg) < 0) {
      ERRPRINTF("SMART: Error receiving encoded server string\n");
      if(msg )free(msg);
      return -1;
    }

    if(gs_decode_server(msg, server_list[i]) < 0) {
      ERRPRINTF("SMART: Error decoding server\n");
      if(msg) free(msg);
      return -1;
    }
    if(msg) free(msg);
    if(server_list[i]->smart==1){
      server_list[i]->smart_id=smart_id;
      smart_id++;
      if(gs_recv_string(sock, &server_list[i]->my_ping_str) < 0) {
        ERRPRINTF("SMART : Error recving ping string from agent\n");
        return -1;
      }
    }

  }
  *_num_servers=num_servers;
  *_server_list=server_list;
  return 0;
}




/*
 * This function sends the mapping information of each handle/problem.
 * 
 * The server requires this information to identify where to receive
 * its inputs from and where to send its outputs to. 
 * This information outlines if the inputs are located in a file
 * or in memory or the outputs should be sent remotely to another 
 * server and stored in memory or in file. It may also outline
 * that the output should be cached in a file or memory. It also outlines
 * if a stored argument should be deleted from memory or file.
 *
 *
 * @param sock -- (in) the socket on which the problem's map_info  will be sent.
 * @param problem -- (in) the mapping information of this problem will be sent
 *
 * @returns 0 on success, -1 on failure.
 *
 */



int gs_smart_send_map_info(SOCKET sock, gs_problem_t * problem){

  gs_argument_t * argptr;
  char * enc_arg_map_info;
  int i;
  if(!problem || !problem->arglist) return -1;
  for(argptr=problem->arglist;argptr!=NULL;argptr=argptr->next) {
    if(argptr->objecttype!=GS_SCALAR) {
      if(!argptr){
        ERRPRINTF("SMART: Error, Arg NULL\n");
      }
      if(gs_smart_encode_arg_map_info(&enc_arg_map_info,argptr)<0){
        ERRPRINTF("SMART: failed to encode smart arg info\n");
        return -1;
      }
      
      if(gs_send_string(sock, enc_arg_map_info) < 0) {
        ERRPRINTF("SMART: failed to send encoded smart arg info\n");
        return -1;
      }
      
      if((argptr->input_arg_stored) || (argptr->input_arg_received_remotely)){
        if(gs_send_string(sock, argptr->input_arg_file) < 0) {
          ERRPRINTF("SMART: failed to send encoded smart arg info\n");
          return -1;
        }
      }

      if(argptr->output_arg_sent_remotely){
        if(gs_send_string(sock, argptr->output_arg_file) < 0) {
          ERRPRINTF("SMART: failed to send encoded smart arg info\n");
          return -1;
        }
        for(i=0;i<argptr->output_nb_dest_servers;i++){
          if(gs_send_string(sock, argptr->enc_output_dest_servers[i]) < 0) {
            ERRPRINTF("SMART: failed to send encoded smart arg info\n");
            return -1;
          }
 
        }

      }

    }
  }


  return 0;
}



/*
 * This function receives the mapping information of each handle/problem.
 * 
 * The server requires this information to identify where to receive
 * its inputs from and where to send its outputs to. 
 * This information outlines if the inputs are located in a file
 * or in memory or the outputs should be sent remotely to another 
 * server and stored in memory or in file. It may also outline
 * that the output should be cached in a file or memory. It also outlines
 * if a stored argument should be deleted from memory or file.
 *
 *
 * @param sock -- (in) the socket on which the problem's map_info  will be sent.
 * @param problem -- (out) the mapping information of this problem will be received
 *
 * @returns 0 on success, -1 on failure.
 *
 */





int gs_smart_recv_map_info(SOCKET sock, gs_problem_t * problem){
  char * enc_arg_map_info;
  gs_argument_t * argptr;
  int i;
  if(!problem || !problem->arglist) return -1;
  for(argptr=problem->arglist;argptr!=NULL;argptr=argptr->next) {
    if(argptr->objecttype!=GS_SCALAR){
      if(!argptr){
        ERRPRINTF("SMART: Error, Arg NULL\n");
      }

      if(gs_recv_string(sock, &enc_arg_map_info)<0){    
        ERRPRINTF("SMART: Error receiving encoded smart arg info.\n");
        return -1;
      }

      if(gs_smart_decode_arg_map_info(enc_arg_map_info, argptr)<0){
        ERRPRINTF("SMART: Error decoding smart arg info.\n");
        return -1;
      }
  
      if((argptr->input_arg_stored) || (argptr->input_arg_received_remotely)){
        if(gs_recv_string(sock, &argptr->input_arg_file) < 0) {
          ERRPRINTF("SMART: failed to recv encoded smart arg info\n");
          return -1;
        }
      }

      argptr->enc_output_dest_servers=(char **)calloc(argptr->output_nb_dest_servers, sizeof(char *));

      if(argptr->output_arg_sent_remotely){
        if(gs_recv_string(sock, &argptr->output_arg_file) < 0) {
          ERRPRINTF("SMART: failed to recv encoded smart arg info\n");
          return -1;
        }
        for(i=0;i<argptr->output_nb_dest_servers;i++){
          if(gs_recv_string(sock, &argptr->enc_output_dest_servers[i]) < 0) {
            ERRPRINTF("SMART: failed to recv encoded smart arg info\n");
            return -1;
          }
 
        }

      }
    }
  }
  return 0;
}







/*
 * This function only sends the input args that are not already
 * located on the server (store either in memory on file).
 * This function is used for blocking tasks.
 * 
 *
 * @param args -- variable arg list.  If using the argument stack
 *   calling sequence, then set this parameter to NULL.
 * @param argstack -- array of pointers to the arguments.  If using
 *   the variable arg list, then set this parameter to NULL.
 * @param sock -- the socket on which the argument will be sent.
 * @param problem -- the problem descriptor for the remote procedure
 *   being called.
 * @param my_dsig -- this machine's data signature.
 * @param lang -- language being used on the client side (either
 *   GS_CALL_FROM_C or GS_CALL_FROM_FORTRAN).
 * @param major -- specifies how this client stores matrices.
 *   'r' represents row-major and 'c' represents column-major.
 *
 * @returns 0 on success, -1 on failure.
 *
 */




int
gs_smart_send_input_args(gs_va_list *args, void **argstack, SOCKET sock,
   gs_problem_t *problem, int my_dsig, int lang, int major)
{

  if(!problem)
    return -1;

  
  if(gs_send_input_scalar_args(args, argstack, sock, problem,
                                my_dsig, lang, major) < 0)
    return -1;



  if(gs_smart_send_input_nonscalar_args(sock, problem, my_dsig) < 0)
    return -1;

  return 0;
}


int
gs_smart_recv_input_args(SOCKET sock, gs_server_t * src_server, gs_problem_t *problem, int sender_dsig,
      int my_dsig){

  gs_argument_t *argptr;
  int sender_major;
  int arg_count=0;

  if(!problem)
    return -1;

  if (gs_recv_input_scalar_args(sock, problem, sender_dsig,     
         my_dsig, &sender_major) < 0)                       
  {
    ERRPRINTF("Error in receiving input scalar args\n");
    return -1;
  }

  if(gs_receiver_compute_arg_sizes(problem, GS_IN) < 0)
  {
    ERRPRINTF("error computing argument sizes.\n");
    return -1;
  }

 for(argptr = problem->arglist; argptr != NULL; argptr = argptr->next)
  {
    if(((argptr->inout == GS_IN) || (argptr->inout == GS_INOUT)) &&
        (argptr->objecttype != GS_SCALAR))
    {
      argptr->data = NULL;
      if(!argptr->input_arg_received_remotely){
        if(gs_recv_arg(sock, argptr, sender_major, sender_dsig, my_dsig, 
            GS_SERVER_SIDE) < 0){
          ERRPRINTF("error receiving input non-scalar arguments\n");
          return -1;
        }
      }
      else{
        int arg_in_mem;
        LOGPRINTF("SMART : Problem %s receiving arg %s from server memory\n", problem->name, argptr->name);
        if(gs_smart_recv_arg_from_s_mem(src_server, argptr, sender_major,  my_dsig,
            GS_SERVER_SIDE, &arg_in_mem)<0){
          ERRPRINTF("SMART : Error receiving arg from server memory\n");
          return -1;
        }
        if(arg_in_mem==0){
           LOGPRINTF("SMART : Problem %s receiving arg %s from file %s\n", problem->name, argptr->name, argptr->input_arg_file);
          if(gs_smart_read_arg_from_file(src_server, argptr->input_arg_file, argptr, sender_major, my_dsig)<0){
            ERRPRINTF("SMART : Error reading arg from file\n");
            return -1;
          }
        }


      }
 

      if(argptr->input_arg_stored==1){
        int s_mem_full=0;
        argptr->output_arg_file=strdup(argptr->input_arg_file);
        LOGPRINTF("SMART : Problem %s storing input arg %s %s to local server memory\n", problem->name, argptr->name, argptr->output_arg_file); 
        if(gs_smart_send_arg_remotely_to_s_mem(src_server, argptr,  problem->major, 
                                                          my_dsig, &s_mem_full) < 0) {
          ERRPRINTF("error in sending output arguments\n");
          return -1;
        }
        if(s_mem_full==1){
          LOGPRINTF("SMART : Problem %s storing input arg %s to local file %s as server memory full\n", problem->name, argptr->name, argptr->output_arg_file); 
          if(gs_smart_write_arg_to_file(argptr->output_arg_file, argptr)<0){
            ERRPRINTF("SMART : Error writing arg to file\n");
            return -1;
          }
        }
      }
      arg_count++;
    }

    if((argptr->objecttype == GS_FILE) && (argptr->inout == GS_OUT)) {
      argptr->data = strdup(argptr->name);


      if(!argptr->data) {
        ERRPRINTF("error allocating space for output arg\n");
        return -1;
      }
    }
    else if((argptr->objecttype == GS_PACKEDFILE) && (argptr->inout == GS_OUT)) {
      char **data;
      int i;

      argptr->data = malloc(argptr->rows * sizeof(char *));
      if(!argptr->data) {
        ERRPRINTF("error allocating memory for filenames.\n");
        return -1;
      }

      data = (char **)argptr->data;

      for(i = 0; i < argptr->rows; i++) {
        data[i] = dstring_sprintf("%s_%d", argptr->name, i);
        if(!data[i]) {
          ERRPRINTF("error allocating memory for filename.\n");
          return -1;
        }
      }
    }
    else if((argptr->inout == GS_OUT) || (argptr->inout == GS_WORKSPACE)) {
      int elsize;

      elsize = gs_get_element_size(argptr->datatype, my_dsig);

      argptr->data = (char *)CALLOC(argptr->rows * argptr->cols, elsize);



      if(!argptr->data) {
        ERRPRINTF("error allocating space for output arg\n");
        return -1;
      }
    }
    else if((argptr->objecttype == GS_PACKEDFILE) && (argptr->inout == GS_OUT)) {
      char **data;
      int i;

      argptr->data = malloc(argptr->rows * sizeof(char *));
      if(!argptr->data) {
        ERRPRINTF("error allocating memory for filenames.\n");
        return -1;
      }

      data = (char **)argptr->data;

      for(i = 0; i < argptr->rows; i++) {
        data[i] = dstring_sprintf("%s_%d", argptr->name, i);
        if(!data[i]) {
          ERRPRINTF("error allocating memory for filename.\n");
          return -1;
        }
      }
    }
    else if((argptr->inout == GS_OUT) || (argptr->inout == GS_WORKSPACE)) {
      int elsize;

      elsize = gs_get_element_size(argptr->datatype, my_dsig);

      argptr->data = (char *)CALLOC(argptr->rows * argptr->cols, elsize);

      if(!argptr->data) {
        ERRPRINTF("error allocating space for output arg\n");
        return -1;
      }
    }
    else if(argptr->inout == GS_VAROUT)
    {
      char **foo = (char **)malloc(sizeof(char*));
      argptr->data = foo;
    }
  }
  return 0;
}


/*
 * This function only sends the nonscalar input args that 
 * are not already located on the server (store either in memory on file).
 * This function is used for non-blocking tasks. 
 *
 * @param sock -- the socket on which the argument will be sent.
 * @param problem -- the problem descriptor for the remote procedure
 *   being called.
 * @param my_dsig -- this machine's data signature.
 *
 * @returns 0 on success, -1 on failure.
 *
 */







int 
gs_smart_send_input_nonscalar_args(SOCKET sock, gs_problem_t *problem, int my_dsig)
{
  gs_argument_t *argptr;
  int arg_count=0;
  for(argptr = problem->arglist; argptr != NULL; argptr = argptr->next)
  {
    if(((argptr->inout == GS_IN) || (argptr->inout == GS_INOUT)) &&
        (argptr->objecttype != GS_SCALAR))
    {
      if(argptr->dsi) {
        if(gs_send_dsi_input_arg(sock, argptr, my_dsig) < 0) {
          ERRPRINTF("error in sending input DSI argument\n");
          return -1;
        }
      }
      else {
        if(!argptr->input_arg_received_remotely){
           if(argptr->input_arg_stored){
             printf("SMART : Sending input arg %s from client to server and stored locally on server\n", argptr->name);
           }
           else{
             printf("SMART : Sending input arg %s from client to server\n", argptr->name);
           }
           if(gs_send_arg(sock, argptr, my_dsig) < 0) {
            ERRPRINTF("error in sending input argument\n");
            return -1;
           }
        }
        else{
          printf("SMART : Input arg %s will be received remotely\n", argptr->name);

        }
      }
    }
    arg_count++;
  }

  return 0;
}






int gs_smart_send_input_args_remotely(SOCKET sock, gs_problem_t *problem, int my_dsig){
  gs_argument_t *argptr;
  int  s_mem_full;
  if(!problem)
    return -1;
  if(gs_send_tag(sock, problem->major) < 0) {
    ERRPRINTF("failed to send major\n");
    return -1;
  }

  if(gs_receiver_compute_arg_sizes(problem, GS_OUT) < 0)
  {
    ERRPRINTF("could not compute argument sizes.\n");
    return -1;
  }

  int arg_count=0;
  for(argptr = problem->arglist; argptr != NULL; argptr = argptr->next)
  {
    if((argptr->inout == GS_INOUT) || (argptr->inout == GS_IN) ||
       (argptr->inout == GS_VAROUT))
    {
      if(argptr->dsi) {
        if(gs_send_dsi_input_arg(sock, argptr, my_dsig) < 0) {
          ERRPRINTF("error in sending input DSI argument\n");
          return -1;
        }
      }
      else {
        if(argptr->input_arg_sent_remotely==1){
         gs_server_t ** dest_srvrs;
         int nb_dest_servers;
         nb_dest_servers=argptr->input_nb_dest_servers;
         dest_srvrs=(gs_server_t **) CALLOC(nb_dest_servers,sizeof(gs_server_t *));
        
         argptr->output_arg_file=strdup(argptr->input_arg_file);

         int i; 
         for(i=0;i<nb_dest_servers;i++){
           dest_srvrs[i]=(gs_server_t *) CALLOC(1,sizeof(gs_server_t));  
           if(gs_decode_server(argptr->enc_input_dest_servers[i], dest_srvrs[i]) < 0) {
             ERRPRINTF("Error decoding server string\n");
             FREE(argptr->enc_input_dest_servers[i]);
             return -1;
           }
            printf("SMART : Problem %s sending input arg %s remotely to server %s to store in server memory\n", problem->name, argptr->name, dest_srvrs[i]->hostname);
           if(gs_smart_send_arg_remotely_to_s_mem(dest_srvrs[i], argptr,  problem->major, my_dsig, &s_mem_full) < 0) {
             ERRPRINTF("SMART : Error in sending input arguments to server memory\n");
             return -1;
            }
            
            if(s_mem_full==1){
              printf("SMART : Problem %s sending arg %s remotely to server %s store in file %s\n", problem->name, argptr->name, dest_srvrs[i]->hostname, argptr->output_arg_file);

              if(gs_smart_send_arg_remotely_to_file(dest_srvrs[i], argptr,  problem->major, 
                                                  my_dsig) < 0) {
                ERRPRINTF("SMART : Error in sending input arguments remotely to file\n");
                return -1;
             }
           }

           if(dest_srvrs[i]) gs_server_free(dest_srvrs[i]);
           }
           if(dest_srvrs) free(dest_srvrs);
         }
       }
       arg_count++;
    }
  }
  return 0;
}








int gs_smart_send_output_args_remotely(SOCKET sock, gs_server_t * src_server, gs_problem_t *problem, int my_dsig){
  gs_argument_t *argptr;
  int  s_mem_full;
  if(!problem)
    return -1;

  if(gs_send_tag(sock, problem->major) < 0) {
    ERRPRINTF("failed to send major\n");
    return -1;
  }

  if(gs_receiver_compute_arg_sizes(problem, GS_OUT) < 0)
  {
    ERRPRINTF("could not compute argument sizes.\n");
    return -1;
  }

  int arg_count=0;
  for(argptr = problem->arglist; argptr != NULL; argptr = argptr->next)
  {
    if((argptr->inout == GS_INOUT) || (argptr->inout == GS_OUT) ||
       (argptr->inout == GS_VAROUT))
    {
      if(argptr->dsi) {
        if(gs_send_dsi_output_arg(sock, argptr, my_dsig) < 0) {
          ERRPRINTF("error in sending output DSI argument\n");
          return -1;
        }
      }
      else {
        if(argptr->output_arg_sent_remotely==1){
          if(argptr->output_cached){
            LOGPRINTF("SMART : Problem %s storing arg %s in local server memory\n", problem->name, argptr->name);
            if(gs_smart_send_arg_remotely_to_s_mem(src_server, argptr,  problem->major, my_dsig, &s_mem_full) < 0) {
              ERRPRINTF("error in sending output arguments\n");
              return -1;
             }
             if(s_mem_full==1){
                LOGPRINTF("SMART : Problem %s storing arg %s in local file %s as server memory full\n", problem->name, argptr->name, argptr->output_arg_file);
               if(gs_smart_write_arg_to_file(argptr->output_arg_file, argptr)<0){
                ERRPRINTF("SMART : Error writing arg to file\n");
                return -1;
               }
             }

          }

          
         gs_server_t ** dest_srvrs;
         int nb_dest_servers;
         nb_dest_servers=argptr->output_nb_dest_servers;

         dest_srvrs=(gs_server_t **) CALLOC(nb_dest_servers,sizeof(gs_server_t *));

         int i; 
         for(i=0;i<nb_dest_servers;i++){
          
           dest_srvrs[i]=(gs_server_t *) CALLOC(1,sizeof(gs_server_t));  
           if(gs_decode_server(argptr->enc_output_dest_servers[i], dest_srvrs[i]) < 0) {
             ERRPRINTF("Error decoding server string\n");
             FREE(argptr->enc_output_dest_servers[i]);
             return -1;
           }
             
            LOGPRINTF("SMART : Problem %s sending arg %s remotely to server %s to store in server memory\n", problem->name, argptr->name, dest_srvrs[i]->hostname);
           if(gs_smart_send_arg_remotely_to_s_mem(dest_srvrs[i], argptr,  problem->major, my_dsig, &s_mem_full) < 0) {
             ERRPRINTF("error in sending output arguments\n");
             return -1;
            }
            
            if(s_mem_full==1){
              LOGPRINTF("SMART : Problem %s sending arg %s remotely to server %s store in file %s\n", problem->name, argptr->name, dest_srvrs[i]->hostname, argptr->output_arg_file);

              if(gs_smart_send_arg_remotely_to_file(dest_srvrs[i], argptr,  problem->major, 
                                                  my_dsig) < 0) {
                ERRPRINTF("error in sending output arguments\n");
                return -1;
             }
           }

           if(dest_srvrs[i]) gs_server_free(dest_srvrs[i]);
           }
           if(dest_srvrs) free(dest_srvrs);
         }
       }
       arg_count++;
    }
  }
  return 0;
}



int 
gs_smart_send_output_args_to_client(SOCKET sock, gs_problem_t *problem, int my_dsig)
{
  gs_argument_t *argptr;

  if(!problem)
    return -1;

  if(gs_send_tag(sock, problem->major) < 0) {
    ERRPRINTF("failed to send major\n");
    return -1;
  }

  if(gs_receiver_compute_arg_sizes(problem, GS_OUT) < 0)
  {
    ERRPRINTF("could not compute argument sizes.\n");
    return -1;
  }

  for(argptr = problem->arglist; argptr != NULL; argptr = argptr->next)
  {
    if((argptr->inout == GS_INOUT) || (argptr->inout == GS_OUT) || 
       (argptr->inout == GS_VAROUT)) 
    {
      if(argptr->dsi) {
        if(gs_send_dsi_output_arg(sock, argptr, my_dsig) < 0) {
          ERRPRINTF("error in sending output DSI argument\n");
          return -1;
        }
      }
      else {

        if(!argptr->output_arg_sent_remotely){ 
          if(gs_send_arg(sock, argptr, my_dsig) < 0) {
            ERRPRINTF("error in sending output arguments\n");
            return -1;
          }
          
        }
      }
    }
  }

  return 0;
}



int 
gs_smart_recv_output_args_from_server(SOCKET sock, gs_problem_t *req_problem, gs_problem_t *handle_problem,int sender_dsig, 
       int my_dsig)
{
  gs_argument_t *req_argptr;
  gs_argument_t *handle_argptr;
  
  int sender_major;
  if(!req_problem)
    return -1;
  if(gs_wait_for_output(sock) < 0) {
    ERRPRINTF("error waiting for output to be ready\n");
    return -1;
  }

  if(gs_recv_tag(sock, &sender_major) < 0) {
    ERRPRINTF("error in receiving major\n");
    return -1;
  }

  handle_argptr=handle_problem->arglist;  

  for(req_argptr = req_problem->arglist; req_argptr != NULL; req_argptr = req_argptr->next)
  {
    if((req_argptr->inout == GS_INOUT) || (req_argptr->inout == GS_OUT) ||
       (req_argptr->inout == GS_VAROUT)) 
    {
      if(!handle_argptr->output_arg_sent_remotely){    
        if(gs_recv_arg(sock, req_argptr, sender_major, sender_dsig, my_dsig, 
            GS_CLIENT_SIDE) < 0) 
        {
          ERRPRINTF("error in receiving output arguments\n");
          return -1;
        }
      }
    }
    if(handle_argptr != NULL){
      handle_argptr = handle_argptr->next;
    }
    else{
      ERRPRINTF("Smart:  Error receiving output arguments\n");
      return -1;
    }
  }
  return 0;
}


int
gs_smart_recv_output_args(SOCKET sock, gs_problem_t *problem, int sender_dsig,
       int my_dsig)
{
  gs_argument_t *argptr;
  int sender_major;
  if(!problem)
    return -1;
  
  if(gs_wait_for_output(sock) < 0) {
    ERRPRINTF("error waiting for output to be ready\n");
    return -1;
  }

  if(gs_recv_tag(sock, &sender_major) < 0) {
    ERRPRINTF("error in receiving major\n");
    return -1;
  }
  for(argptr = problem->arglist; argptr != NULL; argptr = argptr->next)
  {
    if((argptr->inout == GS_INOUT) || (argptr->inout == GS_OUT) ||
       (argptr->inout == GS_VAROUT))
    {
      if(!argptr->output_arg_sent_remotely){
        printf("SMART : Receive output arg %s from server\n", argptr->name);
        if(gs_recv_arg(sock, argptr, sender_major, sender_dsig, my_dsig,
            GS_CLIENT_SIDE) < 0)
        {
          ERRPRINTF("error in receiving output arguments\n");
          return -1;
        }
        printf("SMART : Received output arg %s from server\n", argptr->name);
      }
      else{
        printf("SMART : Output arg %s was sent remotely\n", argptr->name);
      }

    }
  }
  return 0;
}




int gs_smart_send_arg_remotely(SOCKET dest_srv_sock, gs_argument_t *arg,
                        int major, int my_dsig){

 
  my_dsig = pvmgetdsig();
  if(gs_send_string(dest_srv_sock, arg->name) < 0) {
    ERRPRINTF("SMART : Error sending arg name\n");
    return -1;
  }

  if(gs_send_int(dest_srv_sock, (int)arg->objecttype) < 0) {
    ERRPRINTF("SMART : Error sending objecttype\n");
    return -1;
  }

  if(gs_send_int(dest_srv_sock,(int)arg->inout) < 0) {
    ERRPRINTF("SMART : Error sending inout\n");
    return -1;
  }

  if(gs_send_int(dest_srv_sock, (int)arg->datatype) < 0) {
    ERRPRINTF("SMART : Error sending datatype\n");
    return -1;
  }

  if(gs_send_int(dest_srv_sock, (int)arg->prob->major) < 0) {
    ERRPRINTF("SMART : Error sending major\n");
    return -1;
  }

  if(gs_send_int(dest_srv_sock, (int)arg->rows) < 0) {
    ERRPRINTF("SMART : Error sending rows\n");
    return -1;
  }

  if(gs_send_int(dest_srv_sock, (int)arg->cols) < 0) {
    ERRPRINTF("SMART : Error sending cols\n");
    return -1;
  }

  if(gs_send_string(dest_srv_sock, arg->colexp) < 0) {
    ERRPRINTF("SMART : Error sending colexp\n");
    return -1;
  }

  if(gs_send_string(dest_srv_sock, arg->rowexp) < 0) {
    ERRPRINTF("SMART : Error sending rowexp\n");
    return -1;
  }

  if(gs_send_int(dest_srv_sock, major) <0){
    ERRPRINTF("SMART : Error sending major\n");
    return -1;
  }

  if(gs_send_int(dest_srv_sock, my_dsig) <0){
    ERRPRINTF("SMART : Error sending my_dsig\n");
    return -1;
  }

  int sender_elsize = gs_get_element_size(arg->datatype, my_dsig);
  int data_size = arg->rows * arg->cols * sender_elsize;

  arg->byte_size= data_size; 
 
  if(gs_send_int(dest_srv_sock, arg->byte_size) <0){
    ERRPRINTF("SMART : Error sending byte size\n");
    return -1;
  }

  if(gs_send_string(dest_srv_sock,  arg->output_arg_file)<0){
    ERRPRINTF("SMART: Error sending arg file loc\n");
    return -1;
  }
 
  if(gs_send_arg(dest_srv_sock, arg, my_dsig)<0){
    ERRPRINTF("SMART : Error sending arg\n");
    return -1;
  }
  return 0;
}





int gs_smart_recv_arg_remotely(SOCKET sock, gs_argument_t *arg, int my_dsig){

  int sender_major;
  int sender_dsig;
  int recv_int;

my_dsig = pvmgetdsig();
  if(gs_recv_string(sock, &arg->name) < 0) {
    proxy_close(sock);
    return -1;
  }

  if(gs_recv_int(sock, &recv_int) < 0) {
    proxy_close(sock);
    return -1;
  }

  arg->objecttype=recv_int;

  if(gs_recv_int(sock, &recv_int) < 0) {
    proxy_close(sock);
    return -1;
  }
  arg->inout=recv_int;

  if(gs_recv_int(sock, &recv_int) < 0) {
    proxy_close(sock);
    return -1;
  }
  arg->datatype=recv_int;

  if(gs_recv_int(sock, &recv_int) < 0) {
    proxy_close(sock);
    return -1;
  }
  arg->prob->major=recv_int;

  if(gs_recv_int(sock, &recv_int) < 0) {
    proxy_close(sock);
    return -1;
  }
  arg->rows=recv_int;

  if(gs_recv_int(sock, &recv_int) < 0) {
    proxy_close(sock);
    return -1;
  }
  arg->cols=recv_int;

  if(gs_recv_string(sock, &arg->colexp) < 0) {
    proxy_close(sock);
    return -1;
  }

  if(gs_recv_string(sock, &arg->rowexp) < 0) {
    proxy_close(sock);
    return -1;
  }

  if(gs_recv_int(sock, &sender_major) < 0) {
    proxy_close(sock);
    return -1;
  }


  if(gs_recv_int(sock, &sender_dsig) < 0) {
    proxy_close(sock);
    return -1;
  }

  if(gs_recv_int(sock, &arg->byte_size) < 0) {
    proxy_close(sock);
    return -1;
  }

  if(gs_recv_string(sock,  &arg->output_arg_file)<0){
    ERRPRINTF("SMART: Error receiving arg file loc\n");
    return -1;
  }

  if(gs_recv_arg(sock, arg, sender_major, sender_dsig, my_dsig, GS_SERVER_SIDE)<0){
    ERRPRINTF("SMART : Error receiving arg\n");
    return -1;
  }

  return 0;

}





int gs_smart_send_arg_remotely_to_s_mem(gs_server_t * dest_srvr, gs_argument_t *arg,
                        int major, int my_dsig, int * s_mem_full){

  SOCKET dest_srv_sock;
  int tag;
  
  dest_srv_sock = gs_connect_to_host(dest_srvr->componentid, dest_srvr->ipaddress, dest_srvr->port, dest_srvr->proxyip, dest_srvr->proxyport);   

  if(dest_srv_sock == INVALID_SOCKET){
    ERRPRINTF("Smart : Invalid connection with server socket\n");
    proxy_close(dest_srv_sock);
    return -1;
  }
   
  if(gs_send_tag(dest_srv_sock,  GS_SMART_STORE_ARG_TO_MEMORY) < 0){
    ERRPRINTF("SMART : Error sending tag\n");
    return -1;
  }

  arg->byte_size=gs_smart_get_argument_size(arg);

  if(gs_send_int(dest_srv_sock, arg->byte_size)<0){
    ERRPRINTF("SMART : Error sending byte size\n");
    return -1;
  }

  if(gs_send_string(dest_srv_sock,  arg->output_arg_file)<0){
    ERRPRINTF("SMART: Error sending arg file loc\n");
    return -1;
  }

  if(gs_recv_tag(dest_srv_sock, &tag) < 0) {
    ERRPRINTF("SMART : Error receiving tag\n");
    return -1;
  }
 

  if(tag==GS_SMART_ARG_BUFF_FULL){
    *s_mem_full=1;
    return 0;  
  }
  else{
    *s_mem_full=0;  
  } 
  if(gs_smart_send_arg_remotely(dest_srv_sock, arg,  major,
                                 my_dsig) < 0) {
    ERRPRINTF("error in sending output arguments\n");
    return -1;
  }
 
  if(gs_recv_tag(dest_srv_sock, &tag) < 0) {
    ERRPRINTF("SMART : Error receiving tag\n");
    return -1;
  }
   
  if(tag!=GS_PROT_OK){
    ERRPRINTF("SMART : Error sending remote arg\n");
    return -1;
  }
   

  gs_close_socket(dest_srv_sock);
  return 0;
}






int gs_smart_recv_arg_from_s_mem(gs_server_t * src_server, gs_argument_t *arg, int sender_major,
  int my_dsig, gs_side_t side, int * arg_in_mem){

  SOCKET src_srv_sock;
  int tag;
  int arg_recvd=0;
  double total_wait_time, wait_time;
  int sender_dsig;
  wait_time=100;
  total_wait_time=1;
  *arg_in_mem=1;
  tag=GS_SMART_ARG_NOT_READY;

   while(tag==GS_SMART_ARG_NOT_READY){
    src_srv_sock = gs_connect_to_host(src_server->componentid, src_server->ipaddress, src_server->port, src_server->proxyip, src_server->proxyport);
    
    if(src_srv_sock == INVALID_SOCKET){
      ERRPRINTF("Smart : Invalid connection with server socket\n");
      proxy_close(src_srv_sock);
      return -1;
    }
    if(gs_send_tag(src_srv_sock,  GS_SMART_RECV_ARG_FROM_MEMORY) < 0){
      ERRPRINTF("SMART : Error sending tag\n");
      return -1;
    }
    if(gs_send_string(src_srv_sock, arg->input_arg_file)<0){
      ERRPRINTF("SMART : Error sending arg file\n");
      return -1;
    }

    if(gs_recv_tag(src_srv_sock, &tag)<0){
      ERRPRINTF("SMART : Error sending tag\n");
      return -1;
    }
    if(tag==GS_SMART_ARG_NOT_READY){
      arg_recvd=0;
      gs_close_socket(src_srv_sock);
    }

 

   if(tag==GS_SMART_ARG_STORED_IN_S_MEM){
     *arg_in_mem=1;
     if(gs_recv_int(src_srv_sock, &sender_dsig)<0){
       ERRPRINTF("SMART: Error receiving data signature\n");
       return -1;
     }

     if(gs_recv_arg(src_srv_sock, arg, sender_major, sender_dsig, my_dsig,
            GS_SERVER_SIDE) < 0){
       ERRPRINTF("error receiving input non-scalar arguments\n");
       return -1;
     }    
      arg_recvd=1;
      gs_close_socket(src_srv_sock);
    }
    if(arg_recvd==0){
      usleep(wait_time);
      total_wait_time+=wait_time;
      wait_time=total_wait_time/10+1;
      if(wait_time>GS_MAX_ARG_POLL)
        wait_time=GS_MAX_ARG_POLL;
     
    }
  }
  if(tag==GS_SMART_ARG_STORED_IN_FILE){

     *arg_in_mem=0;
     return 0;
  }



  SOCKET _src_srv_sock;
  if(arg->input_arg_deleted){

    _src_srv_sock = gs_connect_to_host(src_server->componentid, src_server->ipaddress, src_server->port, src_server->proxyip, src_server->proxyport);

    if(_src_srv_sock == INVALID_SOCKET){
      ERRPRINTF("Smart : Invalid connection with server socket\n");
      proxy_close(src_srv_sock);
      return -1;
    }
    if(gs_send_tag(_src_srv_sock,  GS_SMART_DELETE_ARG_FROM_MEMORY) < 0){
      ERRPRINTF("SMART : Error sending tag\n");
      return -1;
    }
    if(gs_send_string(_src_srv_sock, arg->input_arg_file)<0){
      ERRPRINTF("SMART : Error sending arg file\n");
      return -1;
    }
    if(gs_recv_tag(_src_srv_sock, &tag) < 0){
      ERRPRINTF("Smart: Error sending tag\n");
      return -1;
    }
    gs_close_socket(_src_srv_sock);

  }

  return 0;
}







int 
gs_smart_save_output_args_to_file(SOCKET sock, gs_server_t * src_server, 
       int fd, gs_problem_t *problem, int my_dsig)
{
  gs_argument_t *argptr;
  int s_mem_full;
  if(!problem)
    return -1;

  if(gs_send_tag(fd, problem->major) < 0) {
    ERRPRINTF("failed to send major\n");
    return -1;
  }


  for(argptr = problem->arglist; argptr != NULL; argptr = argptr->next)
  {
    if(((argptr->inout == GS_IN) || (argptr->inout == GS_INOUT) ||
        (argptr->inout == GS_OUT)) && (argptr->objecttype == GS_SCALAR))
    {
      if(gs_send_arg(fd, argptr, my_dsig) < 0) {
        ERRPRINTF("error in writing input argument\n");
        return -1;
      }
    }
  }

  if(gs_receiver_compute_arg_sizes(problem, GS_OUT) < 0)
  {
    ERRPRINTF("could not compute argument sizes.\n");
    return -1;
  }



  for(argptr = problem->arglist; argptr != NULL; argptr = argptr->next)
  {
    if((argptr->inout == GS_INOUT) || (argptr->inout == GS_OUT) ||
       (argptr->inout == GS_VAROUT))
    {
      if(argptr->objecttype == GS_FILE) {
        if(gs_send_string(fd, argptr->data) < 0) {
          ERRPRINTF("error in writing file name\n");
          return -1;
        }
      }
      else if(argptr->objecttype == GS_PACKEDFILE) {
        int i;

        if(gs_send_int(fd, argptr->rows) < 0) {
          ERRPRINTF("Error sending string length\n");
          return -1;
        }

        for(i = 0; i < argptr->rows; i++) {
          if(gs_send_string(fd, ((char **)(argptr->data))[i]) < 0) {
            ERRPRINTF("error in writing file name\n");
            return -1;
          }
        }
      }
      else {

        if(argptr->output_arg_sent_remotely==1){
          if(argptr->output_cached){
            LOGPRINTF("SMART : Problem %s storing arg %s to local server memory\n", problem->name, argptr->name); 
            if(gs_smart_send_arg_remotely_to_s_mem(src_server, argptr,  problem->major, 
                                                            my_dsig, &s_mem_full) < 0) {
              ERRPRINTF("error in sending output arguments\n");
              return -1;
             }
             if(s_mem_full==1){
            LOGPRINTF("SMART : Problem %s storing arg %s to local file %s as server memory full\n", problem->name, argptr->name, argptr->output_arg_file); 
               if(gs_smart_write_arg_to_file(argptr->output_arg_file, argptr)<0){
                 ERRPRINTF("SMART : Error writing arg to file\n");
                 return -1;
               }
             }

          }

         gs_server_t ** dest_srvrs;
         int nb_dest_servers;
         nb_dest_servers=argptr->output_nb_dest_servers;

         dest_srvrs=(gs_server_t **) CALLOC(nb_dest_servers,sizeof(gs_server_t *));
         int i;
         for(i=0;i<nb_dest_servers;i++){
          
           dest_srvrs[i]=(gs_server_t *) CALLOC(1,sizeof(gs_server_t));  
           if(gs_decode_server(argptr->enc_output_dest_servers[i], dest_srvrs[i]) < 0) {
             ERRPRINTF("Error decoding server string\n");
             FREE(argptr->enc_output_dest_servers[i]);
             return -1;
           }
            LOGPRINTF("SMART : Problem %s sending arg %s remotely to server %s to store in memory of server\n", problem->name, argptr->name, dest_srvrs[i]->hostname); 
           if(gs_smart_send_arg_remotely_to_s_mem(dest_srvrs[i],argptr, problem->major, 
                                              my_dsig, &s_mem_full) < 0) {
             ERRPRINTF("error in sending output arguments\n");
             return -1;
           }
           if(s_mem_full==1){
              LOGPRINTF("SMART : Server buffer full : Problem %s sending arg %s remotely to server %s to store in file %s\n", problem->name, argptr->name, dest_srvrs[i]->hostname, argptr->output_arg_file); 
             if(gs_smart_send_arg_remotely_to_file(dest_srvrs[i],argptr, problem->major, 
                                                my_dsig) < 0) {
               ERRPRINTF("error in sending output arguments\n");
               return -1;
             }
           }

           if(dest_srvrs[i]) gs_server_free(dest_srvrs[i]);
          }
          
          if(dest_srvrs) free(dest_srvrs);
        }
        else{
          if(gs_send_arg(fd, argptr, my_dsig) < 0) {
            ERRPRINTF("error in writing input argument\n");
            return -1;
          }
        }
      }
    }
  }
  return 0;
}



int 
gs_smart_restore_output_args_from_file(int fd, gs_problem_t *problem, 
  int my_dsig)
{   
  gs_argument_t *argptr;
  int sender_major;

  if(!problem)
    return -1;

  if(gs_recv_tag(fd, &sender_major) < 0) {
    ERRPRINTF("error in receiving major\n");
    return -1;
  }
  
  for(argptr = problem->arglist; argptr != NULL; argptr = argptr->next)
  {
    if(((argptr->inout == GS_IN) || (argptr->inout == GS_INOUT) ||
        (argptr->inout == GS_OUT)) && (argptr->objecttype == GS_SCALAR))
    {
      argptr->rows = argptr->cols = 1;
      argptr->data = NULL;

      if(gs_recv_arg(fd, argptr, sender_major, my_dsig, my_dsig,
            GS_CLIENT_SIDE) < 0)
      {
        ERRPRINTF("error in reading input argument\n");
        return -1;
      }
    }
  }

  if(gs_receiver_compute_arg_sizes(problem, GS_OUT) < 0)
  {
    ERRPRINTF("could not compute argument sizes.\n");
    return -1;
  }

  for(argptr = problem->arglist; argptr != NULL; argptr = argptr->next)
  { 
    if((argptr->inout == GS_INOUT) || (argptr->inout == GS_OUT) ||
       (argptr->inout == GS_VAROUT)) 
    {
      if(argptr->objecttype == GS_FILE) {
        char *fname;

        if(gs_recv_string(fd, &fname) < 0) {
          ERRPRINTF("error in reading file name\n");
          return -1;
        }

        argptr->data = fname;
      }
      else if(argptr->objecttype == GS_PACKEDFILE) {
        char *fname;
        int i, n;

        if(gs_recv_int(fd, &n) < 0) {
          ERRPRINTF("error in reading number of packed files\n");
          return -1;
        }

        argptr->data = malloc(n * sizeof(char *));

        for(i = 0; i < n; i++) {
          if(gs_recv_string(fd, &fname) < 0) {
            ERRPRINTF("error in reading file name\n");
            return -1;
          }

          ((char **)(argptr->data))[i] = fname;
        }
      }
      else {
        if(!argptr->output_arg_sent_remotely){
          if(gs_recv_arg(fd, argptr, sender_major, my_dsig, my_dsig,
             GS_CLIENT_SIDE) < 0)
          {
            ERRPRINTF("error in reading input argument\n");
            return -1;
          }
        }
      }
    }
  }

  return 0;
}





int gs_smart_send_arg_remotely_to_file(gs_server_t * dest_srvr, gs_argument_t *arg,
                        int major, int my_dsig){

  SOCKET dest_srv_sock;
  int tag;
 
  dest_srv_sock = gs_connect_to_host(dest_srvr->componentid, dest_srvr->ipaddress, dest_srvr->port, dest_srvr->proxyip, dest_srvr->proxyport);
  
  if(dest_srv_sock == INVALID_SOCKET){
    ERRPRINTF("Smart : Invalid connection with server socket\n");
    proxy_close(dest_srv_sock);
    return -1;
  }

  if(gs_send_tag(dest_srv_sock,   GS_SMART_STORE_ARG_TO_FILE) < 0){
    ERRPRINTF("SMART : Error sending tag\n");
    return -1;
  }

  if(gs_smart_send_arg_remotely(dest_srv_sock, arg,  major,
                                 my_dsig) < 0) {
    ERRPRINTF("error in sending output arguments\n");
    return -1;
  }
  if(gs_recv_tag(dest_srv_sock, &tag) < 0) {
    ERRPRINTF("SMART : Error receiving tag\n");
    return -1;
  }
   
  if(tag!=GS_PROT_OK){
    ERRPRINTF("SMART : Error sending remote arg\n");
    return -1;
  }
  gs_close_socket(dest_srv_sock);
  return 0;
}


/*
 * This function stores the argument (argptr) to file (filename).
 *
 * @param filename -- the location of the file to store the argument
 * @param arg -- pointer to the argument struct to be sent.
 *
 * @returns 0 on success, -1 on failure.
 */


int gs_smart_write_arg_to_file(char * filename, gs_argument_t * argptr){
 int  my_dsig = pvmgetdsig();

  int fd;

  if((fd = gs_open_locked_file(filename, F_WRLCK, O_WRONLY | O_CREAT)) < 0) {
    ERRPRINTF("SMART : Could not open file to write argument\n");
    return -1;
  }

  if(gs_send_arg(fd, argptr, my_dsig )<0){
    ERRPRINTF("SMART : Error writing argument to file\n");
    return -1;
  }
  
  gs_unlock_file(fd);
  close(fd);
  return 0;
}


/*
 * This function receives the argument (argptr) from file (filename).
 *
 * @param filename -- the location of the file to receive the argument
 * @param argptr -- pointer to the argument struct to be sent.
 * @param sender_major -- specifies how the sender stores matrices.
 *   'r' represents row-major and 'c' represents column-major.
 * @param sender_dsig -- the sender's data signature.
 * @param my_dsig -- this machine's data signature.
 *
 * @returns 0 on success, -1 on failure.
 */



int gs_smart_read_arg_from_file(gs_server_t * src_server, char * filename, gs_argument_t * argptr, int sender_major, int my_dsig){

  struct stat stbuf;
  int fd;

  double total_wait_time, wait_time;
  wait_time=100;
  total_wait_time=1;

  do{
    usleep(wait_time);
    total_wait_time+=wait_time;
    wait_time=total_wait_time/10+1;
    if(wait_time>GS_MAX_ARG_POLL)
      wait_time=GS_MAX_ARG_POLL;
  }while(stat(filename, &stbuf)!=0);

   if((fd = gs_open_locked_file(filename,  F_RDLCK, O_RDONLY))==-1){
    ERRPRINTF("SMART : Error opening argument file to read\n");
    return -1;
  }

  /*
   * sender_dsig is the same as my_dsig as the file is received from
   * the local server's filesystem
   */ 
  if(gs_recv_arg(fd, argptr, sender_major, my_dsig, my_dsig, GS_CLIENT_SIDE)<0){
    ERRPRINTF("SMART : Error receiving argument from file\n");
    return -1;
  }

  if(argptr->input_arg_deleted==1){
 
    SOCKET _src_srv_sock;
     int tag;  
     _src_srv_sock = gs_connect_to_host(src_server->componentid, src_server->ipaddress, src_server->port, src_server->proxyip, src_server->proxyport);

     if(_src_srv_sock == INVALID_SOCKET){
       ERRPRINTF("Smart : Invalid connection with server socket\n");
       proxy_close(_src_srv_sock);
       return -1;
     }
     if(gs_send_tag(_src_srv_sock,  GS_SMART_DELETE_ARG_FROM_FILE) < 0){
       ERRPRINTF("SMART : Error sending tag\n");
       return -1;
     }
     if(gs_send_string(_src_srv_sock, filename)<0){
       ERRPRINTF("SMART : Error sending arg file\n");
       return -1;
     }
     if(gs_recv_tag(_src_srv_sock, &tag) < 0){
       ERRPRINTF("Smart: Error sending tag\n");
       return -1;
     }
     gs_close_socket(_src_srv_sock);

     if(remove(filename)<0){
      ERRPRINTF("SMART : Error removing file %s\n", argptr->name);
      return -1;
    }
  }
  gs_unlock_file(fd);
  close(fd);
  return 0;
}



/*
 * This function creates a file name to store an argument. 
 * This file name is based on the argument id (every argument which
 * is sent remotely is given a unique id),  the application id 
 * (every time an application is run it is given a new id).
 *
 *
 * @param _arg_file_name -- (out) the generated filename
 * @param arg_file_prefix -- (in) the prefix of the filename.
 * @param app_id -- (in) the id of the running application 
 * @param arg_id -- (in) the id of the argument sent remotely
 *
 * @returns 0 on success, -1 on failure.
 *
 */

int gs_smart_get_obj_file_name(char ** _arg_file_name, char * arg_file_prefix ,int app_id, int arg_id){
  char * int_string= "%d";
  char  * arg_id_str=(char *)calloc(sizeof(int_string),sizeof(char) );  
  char  * app_id_str=(char *)calloc(sizeof(int_string),sizeof(char) ); 
  char * arg_file_name;
  sprintf(arg_id_str,"%d" ,  arg_id);
  sprintf(app_id_str,"%d" ,  app_id);
  arg_file_name = (char *)calloc(strlen(arg_file_prefix) + strlen("__app_id__") + strlen(app_id_str) + strlen("__arg_id__") + strlen(arg_id_str)+3, sizeof(char));
  strcat(arg_file_name, arg_file_prefix);
  strcat(arg_file_name, "__app_id__");
  strcat(arg_file_name, app_id_str);
  strcat(arg_file_name, "__arg_id__");
  strcat(arg_file_name, arg_id_str);
  strcat(arg_file_name, "\0");
  *_arg_file_name=arg_file_name;
  return 0;
}



/*
 * This function reads the unique id given to the running application.
 * Each time an application is run on a machine it is given a new id.
 * This function also updates the current application id, by incrementing
 * the number by 1.
 * This id is used to generate unique file names to store remote argument.
 * This is prevent conflicts when multiple applications are running on the
 * same network at the same time.
 * If the application number file has not been created this function will
 * create a new application number file.
 *
 *
 * @param file-- (in) the name of the file where the unique application id is
 *                stored
 * @param arg_file_prefix -- (out) application id.
 *
 * @returns 0 on success, -1 on failure.
 *
 */



int gs_smart_read_update_app_no_file(char * file, int * app_num){
  int fd, count, n, exists;
  struct stat stbuf;
  exists=1;
  count=0;
  if(stat(file, &stbuf) != 0){
    count=0;
    exists=0;
  }

  if((fd = gs_open_locked_file(file, F_WRLCK, O_RDWR | O_CREAT)) < 0) {
    ERRPRINTF("Failed to open/lock file '%s' for job count\n", file);
    return -1;
  }
  if(exists==1){
    if((n = read(fd, &count, sizeof(count))) < 0) {
      ERRPRINTF("read failure getting job count\n");
      gs_unlock_file(fd);
      close(fd);
      return -1;
    }
  }
  *app_num=count;
  count++;
  lseek(fd, 0, SEEK_SET);
  write(fd, &count, sizeof(count));
  gs_unlock_file(fd);
  close(fd);
  return 0;
}

/*
 * This function reads but does not update the unique id given to the 
 * running application.
 *
 * @param file-- (in) the name of the file where the unique application id is
 *                stored
 * @param arg_file_prefix -- (out) application id.
 *
 * @returns 0 on success, -1 on failure.
 *
 */


int gs_smart_read_app_no_file(char * file, int * app_num){
  int fd, count, n, exists;
  struct stat stbuf;
  exists=1;
  if(stat(file, &stbuf) != 0){
    count=0;
    exists=0;
  }
  if((fd = gs_open_locked_file(file, F_WRLCK, O_RDWR | O_CREAT)) < 0) {
    ERRPRINTF("Failed to open/lock file '%s' for job count\n", file);
    return -1;
  }
  if(exists==1){
    if((n = read(fd, &count, sizeof(count))) < 0) {
      ERRPRINTF("read failure getting job count\n");
      gs_unlock_file(fd);
      close(fd);
      return -1;
    }
  }
  *app_num=count;
  gs_unlock_file(fd);
  close(fd);
  return 0;
}


#endif