A first cut at the large data problem with TCP. As long

as the writev and readv support a sum larger than a uint32_t
this version will work. For the other OSes a different patch
is required.

Author: bosilca

opal/mca/btl/tcp/btl_tcp_frag.c

@@ -109,11 +109,11 @@ size_t mca_btl_tcp_frag_dump(mca_btl_tcp_frag_t* frag, char* msg, char* buf, siz
 
 bool mca_btl_tcp_frag_send(mca_btl_tcp_frag_t* frag, int sd)
 {
-    int cnt=-1;
+    ssize_t cnt = -1;
     size_t i, num_vecs;
 
     /* non-blocking write, but continue if interrupted */
-    while(cnt < 0) {
+    do {
         cnt = writev(sd, frag->iov_ptr, frag->iov_cnt);
         if(cnt < 0) {
             switch(opal_socket_errno) {
@@ -137,12 +137,12 @@ bool mca_btl_tcp_frag_send(mca_btl_tcp_frag_t* frag, int sd)
                 return false;
             }
         }
-    }
+    } while(cnt < 0);
 
     /* if the write didn't complete - update the iovec state */
     num_vecs = frag->iov_cnt;
-    for(i=0; i<num_vecs; i++) {
-        if(cnt >= (int)frag->iov_ptr->iov_len) {
+    for( i = 0; i < num_vecs; i++) {
+        if(cnt >= frag->iov_ptr->iov_len) {
             cnt -= frag->iov_ptr->iov_len;
             frag->iov_ptr++;
             frag->iov_idx++;
@@ -159,15 +159,15 @@ bool mca_btl_tcp_frag_send(mca_btl_tcp_frag_t* frag, int sd)
 
 bool mca_btl_tcp_frag_recv(mca_btl_tcp_frag_t* frag, int sd)
 {
-    int cnt, dont_copy_data = 0;
-    size_t i, num_vecs;
     mca_btl_base_endpoint_t* btl_endpoint = frag->endpoint;
+    ssize_t cnt;
+    int32_t i, num_vecs, dont_copy_data = 0;
 
  repeat:
     num_vecs = frag->iov_cnt;
 #if MCA_BTL_TCP_ENDPOINT_CACHE
     if( 0 != btl_endpoint->endpoint_cache_length ) {
-        size_t length;
+        ssize_t length;
         /* It's strange at the first look but cnt have to be set to the full amount of data
          * available. After going to advance_iov_position we will use cnt to detect if there
          * is still some data pending.
@@ -203,20 +203,20 @@ bool mca_btl_tcp_frag_recv(mca_btl_tcp_frag_t* frag, int sd)
 
     /* non-blocking read, but continue if interrupted */
     cnt = -1;
-    while( cnt < 0 ) {
+    do {
         cnt = readv(sd, frag->iov_ptr, num_vecs);
-	if( 0 < cnt ) goto advance_iov_position;
-	if( cnt == 0 ) {
+        if( 0 < cnt ) goto advance_iov_position;
+        if( cnt == 0 ) {
             btl_endpoint->endpoint_state = MCA_BTL_TCP_FAILED;
-	    mca_btl_tcp_endpoint_close(btl_endpoint);
-	    return false;
-	}
-	switch(opal_socket_errno) {
-	case EINTR:
-	    continue;
-	case EWOULDBLOCK:
-	    return false;
-	case EFAULT:
+            mca_btl_tcp_endpoint_close(btl_endpoint);
+            return false;
+        }
+        switch(opal_socket_errno) {
+        case EINTR:
+            continue;
+        case EWOULDBLOCK:
+            return false;
+        case EFAULT:
             BTL_ERROR(("mca_btl_tcp_frag_recv: readv error (%p, %lu)\n\t%s(%lu)\n",
                        frag->iov_ptr[0].iov_base, (unsigned long) frag->iov_ptr[0].iov_len,
                        strerror(opal_socket_errno), (unsigned long) frag->iov_cnt));
@@ -231,23 +231,23 @@ bool mca_btl_tcp_frag_recv(mca_btl_tcp_frag_t* frag, int sd)
             mca_btl_tcp_endpoint_close(btl_endpoint);
             return false;
         }
-    }
+    } while( cnt < 0 );
 
  advance_iov_position:
     /* if the read didn't complete - update the iovec state */
     num_vecs = frag->iov_cnt;
     for( i = 0; i < num_vecs; i++ ) {
-        if( cnt < (int)frag->iov_ptr->iov_len ) {
+        if( cnt < frag->iov_ptr->iov_len ) {
             frag->iov_ptr->iov_base = (opal_iov_base_ptr_t)
                 (((unsigned char*)frag->iov_ptr->iov_base) + cnt);
             frag->iov_ptr->iov_len -= cnt;
             cnt = 0;
             break;
-    }
-    cnt -= frag->iov_ptr->iov_len;
-    frag->iov_idx++;
-    frag->iov_ptr++;
-    frag->iov_cnt--;
+        }
+        cnt -= frag->iov_ptr->iov_len;
+        frag->iov_idx++;
+        frag->iov_ptr++;
+        frag->iov_cnt--;
     }
 #if MCA_BTL_TCP_ENDPOINT_CACHE
     btl_endpoint->endpoint_cache_length = cnt;

opal/mca/btl/tcp/btl_tcp_frag.h

@@ -53,8 +53,8 @@ struct mca_btl_tcp_frag_t {
     mca_btl_tcp_hdr_t hdr;
     struct iovec iov[MCA_BTL_TCP_FRAG_IOVEC_NUMBER + 1];
     struct iovec *iov_ptr;
-    size_t iov_cnt;
-    size_t iov_idx;
+    uint32_t iov_cnt;
+    uint32_t iov_idx;
     size_t size;
     int rc;
     opal_free_list_t* my_list;