Add retryAfterMSec option

Author: vlad-ignatov

README.md

@@ -132,6 +132,7 @@ The Bulk Data Client uses `js` configuration files, but you can think of them as
 - *boolean* **`log.enabled`** - Set this to false to disable logging. Optional (defaults to true).
 - *string* **`log.file`** - Path to the log file. Absolute, or relative to process CWD. If not provided, the file will be called log.ndjson and will be stored in the downloads folder.
 - *object* **`log.metadata`** - Key/value pairs to be added to every log entry. Can be used to add useful information (for example which site imported this data).
+- *number* **`retryAfterMSec`** - If the server does not provide `Retry-after` header use this number of milliseconds before checking the status again.
 
 
 ### Environment Variables
@@ -173,6 +174,7 @@ Note that you can pass a `--help` parameter to see this listed in your terminal
 |       | `--config`       | Relative path to config file |
 |       | `--reporter`     | Reporter to use to render the output. "cli" renders fancy progress bars and tables. "text" is better for log files. Defaults to "cli" |
 | `-c`  | `--custom`       | Custom parameters to be passed to the kick-off endpoint. Example: `-c a=1 b=c` |
+|       | `--status`       | If a status request fails for some reason the client will exit. However, if the status endpoint is printed in the output, you can retry by passing it as `--status` option here |
 
 
 Features

built/lib/BulkDataClient.js

@@ -312,15 +312,15 @@ class BulkDataClient extends events_1.EventEmitter {
                     const progress = String(res.headers["x-progress"] || "").trim();
                     const retryAfter = String(res.headers["retry-after"] || "").trim();
                     const progressPct = parseInt(progress, 10);
-                    let retryAfterMSec = 1000;
+                    let retryAfterMSec = this.options.retryAfterMSec;
                     if (retryAfter.match(/\d+/)) {
                         retryAfterMSec = parseInt(retryAfter, 10) * 1000;
                     }
                     else {
                         let d = new Date(retryAfter);
                         retryAfterMSec = Math.ceil(d.getTime() - now);
                     }
-                    const poolDelay = Math.min(Math.max(retryAfterMSec / 10, 1000), 10000);
+                    const poolDelay = Math.min(Math.max(retryAfterMSec, 100), 1000 * 60 * 60 * 24);
                     Object.assign(status, {
                         percentComplete: isNaN(progressPct) ? -1 : progressPct,
                         nextCheckAfter: poolDelay,

config/defaults.js

@@ -284,5 +284,11 @@
         metadata: {
             // siteId: "localhost"
         }
-    }
+    },
+
+    /**
+     * If the server does not provide `Retry-after` header use this number of
+     * milliseconds before checking the status again
+     */
+    retryAfterMSec: 200
 }

index.d.ts

@@ -211,6 +211,12 @@ export declare namespace BulkDataClient {
         awsSecretAccessKey?: string
 
         log?: LoggingOptions
+
+        /**
+         * If the server does not provide `Retry-after` header use this number of
+         * milliseconds before checking the status again
+         */
+        retryAfterMSec?: number
     }
 
     interface LoggingOptions
@@ -463,6 +469,12 @@ export declare namespace BulkDataClient {
         awsSecretAccessKey: string
 
         log: LoggingOptions
+
+        /**
+         * If the server does not provide `Retry-after` header use this number of
+         * milliseconds before checking the status again
+         */
+        retryAfterMSec: number
     }
 
     interface JWK {

package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulk-data-client",
-  "version": "1.1.2",
+  "version": "1.1.3",
   "description": "",
   "main": "built/app.js",
   "engines": {

src/lib/BulkDataClient.ts

@@ -470,7 +470,7 @@ class BulkDataClient extends EventEmitter
                     const retryAfter  = String(res.headers["retry-after"] || "").trim();
                     const progressPct = parseInt(progress, 10);
 
-                    let retryAfterMSec = 1000;
+                    let retryAfterMSec = this.options.retryAfterMSec;
                     if (retryAfter.match(/\d+/)) {
                         retryAfterMSec = parseInt(retryAfter, 10) * 1000
                     } else {