Skip to content

Commit 4c542aa

Browse files
secDre4mersecDre4mer
authored
Merge pull request #5 from NextronSystems/feat/more-retries
add more retries to the different collector variants
2 parents bb5ee30 + e8aeb43 commit 4c542aa

5 files changed

+121
-66
lines changed

Diff for: ‎go/collector.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -239,7 +239,7 @@ func (c *Collector) uploadToThunderstorm(info *infoWithPath) (redo bool) {
239239
if info.retries < 3 {
240240
c.logger.Printf("Could not send file %s to thunderstorm, will try again: %v", info.path, err)
241241
info.retries++
242-
time.Sleep(time.Second)
242+
time.Sleep(4 * time.Second * time.Duration(1<<info.retries))
243243
return true
244244
} else {
245245
c.logger.Printf("Could not send file %s to thunderstorm, canceling it.", info.path)

Diff for: ‎scripts/thunderstorm-collector.pl

100644100755
+26-13
Original file line numberDiff line numberDiff line change
@@ -127,19 +127,32 @@ sub processDir {
127127
sub submitSample {
128128
my ($filepath) = shift;
129129
print "[SUBMIT] Submitting $filepath ...\n";
130-
eval {
131-
my $req = $ua->post($api_endpoint,
132-
Content_Type => 'form-data',
133-
Content => [
134-
"file" => [ $filepath ],
135-
],
136-
);
137-
$num_submitted++;
138-
print "\nError: ", $req->status_line unless $req->is_success;
139-
} or do {
140-
my $error = $@ || 'Unknown failure';
141-
warn "Could not submit '$filepath' - $error";
142-
};
130+
my $retry = 0;
131+
for ($retry = 0; $retry < 4; $retry++) {
132+
if ($retry > 0) {
133+
my $sleep_time = 2 << $retry;
134+
print "[SUBMIT] Waiting $sleep_time seconds to retry submitting $filepath ...\n";
135+
sleep($sleep_time)
136+
}
137+
my $successful = 0;
138+
eval {
139+
my $req = $ua->post($api_endpoint,
140+
Content_Type => 'form-data',
141+
Content => [
142+
"file" => [ $filepath ],
143+
],
144+
);
145+
$successful = $req->is_success;
146+
$num_submitted++;
147+
print "\nError: ", $req->status_line unless $successful;
148+
} or do {
149+
my $error = $@ || 'Unknown failure';
150+
warn "Could not submit '$filepath' - $error";
151+
};
152+
if ($successful) {
153+
last;
154+
}
155+
}
143156
}
144157

145158
# MAIN ----------------------------------------------------------------

Diff for: ‎scripts/thunderstorm-collector.ps1

+9-2
Original file line numberDiff line numberDiff line change
@@ -274,6 +274,7 @@ try {
274274

275275
# Submitting the request
276276
$StatusCode = 0
277+
$Retries = 0
277278
while ( $($StatusCode) -ne 200 ) {
278279
try {
279280
Write-Log "Submitting to Thunderstorm server: $($_.FullName) ..." -Level "Info"
@@ -291,8 +292,14 @@ try {
291292
Write-Log "503: Server seems busy - retrying in $($WaitSecs) seconds"
292293
Start-Sleep -Seconds $($WaitSecs)
293294
} else {
294-
Write-Log "$($StatusCode): Server has problems - retrying in 3 seconds"
295-
Start-Sleep -Seconds 3
295+
if ( $Retries -eq 3) {
296+
Write-Log "$($StatusCode): Server still has problems - giving up"
297+
break
298+
}
299+
$Retries = $Retries + 1
300+
$SleepTime = 2 * [Math]::Pow(2, $Retries)
301+
Write-Log "$($StatusCode): Server has problems - retrying in $SleepTime seconds"
302+
Start-Sleep -Seconds $($SleepTime)
296303
}
297304
}
298305
}

Diff for: ‎scripts/thunderstorm-collector.py

100644100755
+60-42
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,7 @@
55
import os
66
import re
77
import ssl
8-
import sys
98
import time
10-
import urllib.parse
119
import uuid
1210

1311
# Configuration
@@ -34,6 +32,11 @@
3432
num_submitted = 0
3533
num_processed = 0
3634

35+
# URL to use for submission
36+
api_endpoint = ""
37+
38+
# Original args
39+
args = {}
3740

3841
# Functions
3942
def process_dir(workdir):
@@ -106,53 +109,70 @@ def skip_file(filepath):
106109
def submit_sample(filepath):
107110
print("[SUBMIT] Submitting {} ...".format(filepath))
108111

112+
headers = {
113+
"Content-Type": "application/octet-stream",
114+
"Content-Disposition": f"attachment; filename={filepath}",
115+
}
116+
109117
try:
110-
headers = {
111-
"Content-Type": "application/octet-stream",
112-
"Content-Disposition": f"attachment; filename={filepath}",
113-
}
114118

115119
with open(filepath, "rb") as f:
116120
data = f.read()
117121

118-
boundary = str(uuid.uuid4())
119-
headers = {
120-
"Content-Type": f"multipart/form-data; boundary={boundary}",
121-
}
122-
123-
# Create multipart/form-data payload
124-
payload = (
125-
f"--{boundary}\r\n"
126-
f'Content-Disposition: form-data; name="file"; filename="{filepath}"\r\n'
127-
f"Content-Type: application/octet-stream\r\n\r\n"
128-
).encode("utf-8")
129-
payload += data
130-
payload += f"\r\n--{boundary}--\r\n".encode("utf-8")
131-
132-
if args.tls:
133-
if args.insecure:
134-
context = ssl._create_unverified_context()
122+
except Exception as e:
123+
print("[ERROR] Could not read '{}' - {}".format(filepath, e))
124+
return
125+
126+
boundary = str(uuid.uuid4())
127+
headers = {
128+
"Content-Type": f"multipart/form-data; boundary={boundary}",
129+
}
130+
131+
# Create multipart/form-data payload
132+
payload = (
133+
f"--{boundary}\r\n"
134+
f'Content-Disposition: form-data; name="file"; filename="{filepath}"\r\n'
135+
f"Content-Type: application/octet-stream\r\n\r\n"
136+
).encode("utf-8")
137+
payload += data
138+
payload += f"\r\n--{boundary}--\r\n".encode("utf-8")
139+
140+
retries = 0
141+
while retries < 3:
142+
try:
143+
if args.tls:
144+
if args.insecure:
145+
context = ssl._create_unverified_context()
146+
else:
147+
context = ssl._create_default_https_context()
148+
conn = http.client.HTTPSConnection(args.server, args.port, context=context)
135149
else:
136-
context = ssl._create_default_https_context()
137-
conn = http.client.HTTPSConnection(args.server, args.port, context=context)
138-
else:
139-
conn = http.client.HTTPConnection(args.server, args.port)
140-
conn.request("POST", api_endpoint, body=payload, headers=headers)
150+
conn = http.client.HTTPConnection(args.server, args.port)
151+
conn.request("POST", api_endpoint, body=payload, headers=headers)
141152

142-
resp = conn.getresponse()
153+
resp = conn.getresponse()
143154

144-
global num_submitted
145-
num_submitted += 1
155+
except Exception as e:
156+
print("[ERROR] Could not submit '{}' - {}".format(filepath, e))
157+
retries += 1
158+
time.sleep(2 << retries)
159+
continue
146160

147-
if resp.status != 200:
148-
print(
149-
"[ERROR] HTTP return status: {}, reason: {}".format(
150-
resp.status, resp.reason
151-
)
161+
# pylint: disable=no-else-continue
162+
if resp.status == 503: # Service unavailable
163+
retry_time = resp.headers.get("Retry-After", 30)
164+
time.sleep(retry_time)
165+
continue
166+
elif resp.status == 200:
167+
break
168+
print(
169+
"[ERROR] HTTP return status: {}, reason: {}".format(
170+
resp.status, resp.reason
152171
)
172+
)
153173

154-
except Exception as e:
155-
print("Could not submit '{}' - {}".format(filepath, e))
174+
global num_submitted
175+
num_submitted += 1
156176

157177

158178
# Main
@@ -188,13 +208,11 @@ def submit_sample(filepath):
188208
)
189209
parser.add_argument("--debug", action="store_true", help="Enable debug logging.")
190210

191-
global args
192211
args = parser.parse_args()
193212

194213
if args.tls:
195214
schema = "https"
196215

197-
global api_endpoint
198216
api_endpoint = "{}://{}:{}/api/checkAsync".format(schema, args.server, args.port)
199217

200218
print("=" * 80)
@@ -214,8 +232,8 @@ def submit_sample(filepath):
214232
print("Starting the walk at: {} ...".format(", ".join(args.dirs)))
215233

216234
# Walk directory
217-
for dir in args.dirs:
218-
process_dir(dir)
235+
for walkdir in args.dirs:
236+
process_dir(walkdir)
219237

220238
# End message
221239
end_date = time.time()

Diff for: ‎scripts/thunderstorm-collector.sh

+25-8
Original file line numberDiff line numberDiff line change
@@ -137,14 +137,31 @@ do
137137
continue
138138
fi
139139
log debug "Submitting ${file_path} ..."
140-
# Submit sample
141-
result=$(curl -s -X POST \
142-
"$scheme://$THUNDERSTORM_SERVER:8080/api/$api_endpoint" \
143-
--form "file=@${file_path};filename=${file_path}")
144-
# If not 'id' in result
145-
error="reason"
146-
if [ "${result/$error}" != "$result" ]; then
147-
log error "$result"
140+
successful=0
141+
142+
for retry in {1..3}; do
143+
# Submit sample
144+
result=$(curl -s -X POST \
145+
"$scheme://$THUNDERSTORM_SERVER:8080/api/$api_endpoint" \
146+
--form "file=@${file_path};filename=${file_path}")
147+
curl_exit=$?
148+
if [ $curl_exit -ne 0 ]; then
149+
log error "Upload failed with code $curl_exit"
150+
sleep $((2 << retry))
151+
continue
152+
fi
153+
154+
# If 'reason' in result
155+
if [ "${result/reason}" != "$result" ]; then
156+
log error "$result"
157+
sleep $((2 << retry))
158+
continue
159+
fi
160+
successful=1
161+
break
162+
done
163+
if [ $successful -ne 1 ]; then
164+
log error "Could not upload ${file_path}"
148165
fi
149166
fi
150167
done

0 commit comments

Comments
 (0)