core (grpc) use retry strategy

lib/common/grpc-service.js

@@ -20,11 +20,13 @@
 
 'use strict';
 
+var extend = require('extend');
 var googleProtoFiles = require('google-proto-files');
 var grpc = require('grpc');
 var is = require('is');
 var nodeutil = require('util');
 var path = require('path');
+var retryRequest = require('retry-request');
 
 /**
  * @type {module:common/service}
@@ -151,6 +153,8 @@ function GrpcService(config, options) {
     this.grpcCredentials = grpc.credentials.createInsecure();
   }
 
+  this.maxRetries = config.maxRetries;
+
   var apiVersion = config.apiVersion;
   var service = this.service = config.service;
   var rootDir = googleProtoFiles('..');
@@ -241,19 +245,47 @@ GrpcService.prototype.request = function(protoOpts, reqOpts, callback) {
     grpcOpts.deadline = new Date(Date.now() + protoOpts.timeout);
   }
 
-  service[protoOpts.method](reqOpts, function(err, resp) {
-    if (err) {
-      if (HTTP_ERROR_CODE_MAP[err.code]) {
-        var httpError = HTTP_ERROR_CODE_MAP[err.code];
-        err.code = httpError.code;
-      }
-
-      callback(err);
-      return;
+  // Retains a reference to an error from the response. If the final callback is
+  // executed with this as the "response", we return it to the user as an error.
+  var respError;
+
+  retryRequest(null, {
+    shouldRetryFn: function(resp) {
+      return [429, 500, 502, 503].indexOf(resp.code) > -1;
+    },
+
+    maxRetries: this.maxRetries,
+
+    // retry-request determines if it should retry from the incoming HTTP
+    // response status. gRPC always returns an error proto message. We pass that
+    // "error" into retry-request to act as the HTTP response, so it can use the
+    // status code to determine if it should retry.
+    request: function(_, onResponse) {
+      respError = null;
+
+      service[protoOpts.method](reqOpts, function(err, resp) {
+        if (err) {
+          if (HTTP_ERROR_CODE_MAP[err.code]) {
+            respError = extend(err, HTTP_ERROR_CODE_MAP[err.code]);
+            onResponse(null, respError);
+            return;
+          }
+
+          onResponse(err);
+          return;
+        }
+
+        onResponse(null, resp);
+      }, null, grpcOpts);
+    }
+  }, function(err, resp) {
+    if (!err && resp === respError) {
+      err = respError;
+      resp = null;
     }
 
-    callback(null, resp);
-  }, null, grpcOpts);
+    callback(err, resp);
+  });
 };
 
 /**