Skip to content

Commit 854abc7

Browse files
author
Drew Kerrigan
committed
Merge pull request #95 from basho/ack-http
Add a generic HTTP driver
2 parents 7b6fd1a + 63e535c commit 854abc7

File tree

2 files changed

+341
-0
lines changed

2 files changed

+341
-0
lines changed

‎examples/http.config

+27
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
{mode, max}.
2+
3+
{duration, 1}.
4+
5+
{concurrent, 1}.
6+
7+
{driver, basho_bench_driver_http}.
8+
9+
{key_generator, {int_to_str, {uniform_int, 50000}}}.
10+
11+
{value_generator, {fixed_bin, 10}}.
12+
13+
{operations, [
14+
%% Get without a key
15+
{{get, {"localhost", 4567, "/"}, []}, 1},
16+
%% Get with a key and headers
17+
{{get_re, {"localhost", 4567, "/%%K"}, [{'Content-Type', 'application/json'}]}, 1},
18+
%% Put with a json object and value
19+
{{put_re, {"localhost", 4567, "/",
20+
"{\"this\":\"is_json_%%V\"}"}, [{'Content-Type', 'application/json'}]}, 1},
21+
%% Post with an xml object and value
22+
{{post_re, {"localhost", 4567, "/%%K",
23+
"<?xml version=\"1.0\"?><catalog><book><author>%%V</author></book></catalog>"},
24+
[{'Content-Type', 'application/xml'}]}, 1},
25+
%% Delete with a key
26+
{{delete_re, {"localhost", 4567, "/%%K"}, []}, 1}
27+
]}.

‎src/basho_bench_driver_http.erl

+314
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,314 @@
1+
%% -------------------------------------------------------------------
2+
%%
3+
%% basho_bench: Benchmarking Suite
4+
%%
5+
%% Copyright (c) 2009-2013 Basho Techonologies
6+
%%
7+
%% This file is provided to you under the Apache License,
8+
%% Version 2.0 (the "License"); you may not use this file
9+
%% except in compliance with the License. You may obtain
10+
%% a copy of the License at
11+
%%
12+
%% http://www.apache.org/licenses/LICENSE-2.0
13+
%%
14+
%% Unless required by applicable law or agreed to in writing,
15+
%% software distributed under the License is distributed on an
16+
%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17+
%% KIND, either express or implied. See the License for the
18+
%% specific language governing permissions and limitations
19+
%% under the License.
20+
%%
21+
%% -------------------------------------------------------------------
22+
-module(basho_bench_driver_http).
23+
24+
-export([new/1,
25+
run/4]).
26+
27+
-include("basho_bench.hrl").
28+
29+
-record(url, {abspath, host, port, username, password, path, protocol, host_type}).
30+
31+
-record(state, {path_params}). % Params to append on the path
32+
33+
34+
%% ====================================================================
35+
%% API
36+
%% ====================================================================
37+
38+
new(Id) ->
39+
40+
?DEBUG("ID: ~p\n", [Id]),
41+
42+
%% Make sure ibrowse is available
43+
case code:which(ibrowse) of
44+
non_existing ->
45+
?FAIL_MSG("~s requires ibrowse to be installed.\n", [?MODULE]);
46+
_ ->
47+
ok
48+
end,
49+
50+
application:start(ibrowse),
51+
52+
Params = basho_bench_config:get(http_params, ""),
53+
Disconnect = basho_bench_config:get(http_disconnect_frequency, infinity),
54+
55+
case Disconnect of
56+
infinity -> ok;
57+
Seconds when is_integer(Seconds) -> ok;
58+
{ops, Ops} when is_integer(Ops) -> ok;
59+
_ -> ?FAIL_MSG("Invalid configuration for http_disconnect_frequency: ~p~n", [Disconnect])
60+
end,
61+
62+
%% Uses pdict to avoid threading state record through lots of functions
63+
erlang:put(disconnect_freq, Disconnect),
64+
65+
{ok, #state {path_params = Params}}.
66+
67+
run({get_re, {Host, Port, Path}, Headers}, KeyGen, _ValueGen, _State) ->
68+
Path1 = re:replace(Path, "%%K", KeyGen(), [global, {return, list}]),
69+
run({get, {Host, Port, Path1}, Headers}, KeyGen, _ValueGen, _State);
70+
71+
run({get, {Host, Port, Path}, Headers}, _KeyGen, _ValueGen, _State) ->
72+
PUrl = #url{host=Host, port=Port, path=Path},
73+
74+
case do_get(PUrl, Headers) of
75+
{not_found, _Url} ->
76+
{ok, 1};
77+
{ok, _Url, _Header} ->
78+
{ok, 1};
79+
{error, Reason} ->
80+
{error, Reason, 1}
81+
end;
82+
83+
run({put_re, {Host, Port, Path, Data}, Headers}, KeyGen, ValueGen, _State) ->
84+
Path1 = re:replace(Path, "%%K", KeyGen(), [global, {return, list}]),
85+
Value = re:replace(Data, "%%V", ValueGen(), [global, {return, list}]),
86+
run({put, {Host, Port, Path1, Value}, Headers}, KeyGen, ValueGen, _State);
87+
88+
run({put, {Host, Port, Path, Data}, Headers}, _KeyGen, _ValueGen, _State) ->
89+
PUrl = #url{host=Host, port=Port, path=Path},
90+
91+
case do_put(PUrl, Headers, Data) of
92+
ok ->
93+
{ok, 1};
94+
{error, Reason} ->
95+
{error, Reason, 1}
96+
end;
97+
98+
run({post_re, {Host, Port, Path, Data}, Headers}, KeyGen, ValueGen, _State) ->
99+
Path1 = re:replace(Path, "%%K", KeyGen(), [global, {return, list}]),
100+
Value = re:replace(Data, "%%V", ValueGen(), [global, {return, list}]),
101+
run({post, {Host, Port, Path1, Value}, Headers}, KeyGen, ValueGen, _State);
102+
103+
run({post, {Host, Port, Path, Data}, Headers}, _KeyGen, _ValueGen, _State) ->
104+
PUrl = #url{host=Host, port=Port, path=Path},
105+
106+
case do_post(PUrl, Headers, Data) of
107+
ok ->
108+
{ok, 1};
109+
{error, Reason} ->
110+
{error, Reason, 1}
111+
end;
112+
113+
run({delete_re, {Host, Port, Path}, Headers}, KeyGen, _ValueGen, _State) ->
114+
Path1 = re:replace(Path, "%%K", KeyGen(), [global, {return, list}]),
115+
run({delete, {Host, Port, Path1}, Headers}, KeyGen, _ValueGen, _State);
116+
117+
run({delete, {Host, Port, Path}, Headers}, _KeyGen, _ValueGen, _State) ->
118+
PUrl = #url{host=Host, port=Port, path=Path},
119+
120+
case do_delete(PUrl, Headers) of
121+
ok ->
122+
{ok, 1};
123+
{error, Reason} ->
124+
{error, Reason, 1}
125+
end.
126+
127+
%% ====================================================================
128+
%% Internal functions
129+
%% ====================================================================
130+
131+
do_get(Url, Headers) ->
132+
%%case send_request(Url, [], get, [], [{response_format, binary}]) of
133+
case send_request(Url, Headers, get, [], [{response_format, binary}]) of
134+
{ok, "404", _Header, _Body} ->
135+
{not_found, Url};
136+
{ok, "300", Header, _Body} ->
137+
{ok, Url, Header};
138+
{ok, "200", Header, _Body} ->
139+
{ok, Url, Header};
140+
{ok, Code, _Header, _Body} ->
141+
{error, {http_error, Code}};
142+
{error, Reason} ->
143+
{error, Reason}
144+
end.
145+
146+
do_put(Url, Headers, ValueGen) ->
147+
Val = if is_function(ValueGen) ->
148+
ValueGen();
149+
true ->
150+
ValueGen
151+
end,
152+
case send_request(Url, Headers,
153+
put, Val, [{response_format, binary}]) of
154+
{ok, "200", _Header, _Body} ->
155+
ok;
156+
{ok, "201", _Header, _Body} ->
157+
ok;
158+
{ok, "202", _Header, _Body} ->
159+
ok;
160+
{ok, "204", _Header, _Body} ->
161+
ok;
162+
{ok, Code, _Header, _Body} ->
163+
{error, {http_error, Code}};
164+
{error, Reason} ->
165+
{error, Reason}
166+
end.
167+
168+
do_post(Url, Headers, ValueGen) ->
169+
Val = if is_function(ValueGen) ->
170+
ValueGen();
171+
true ->
172+
ValueGen
173+
end,
174+
case send_request(Url, Headers,
175+
post, Val, [{response_format, binary}]) of
176+
{ok, "200", _Header, _Body} ->
177+
ok;
178+
{ok, "201", _Header, _Body} ->
179+
ok;
180+
{ok, "202", _Header, _Body} ->
181+
ok;
182+
{ok, "204", _Header, _Body} ->
183+
ok;
184+
{ok, Code, _Header, _Body} ->
185+
{error, {http_error, Code}};
186+
{error, Reason} ->
187+
{error, Reason}
188+
end.
189+
190+
do_delete(Url, Headers) ->
191+
case send_request(Url, Headers, delete, [], []) of
192+
{ok, "200", _Header, _Body} ->
193+
ok;
194+
{ok, "201", _Header, _Body} ->
195+
ok;
196+
{ok, "202", _Header, _Body} ->
197+
ok;
198+
{ok, "204", _Header, _Body} ->
199+
ok;
200+
{ok, "404", _Header, _Body} ->
201+
ok;
202+
{ok, "410", _Header, _Body} ->
203+
ok;
204+
{ok, Code, _Header, _Body} ->
205+
{error, {http_error, Code}};
206+
{error, Reason} ->
207+
{error, Reason}
208+
end.
209+
210+
connect(Url) ->
211+
case erlang:get({ibrowse_pid, Url#url.host}) of
212+
undefined ->
213+
{ok, Pid} = ibrowse_http_client:start({Url#url.host, Url#url.port}),
214+
erlang:put({ibrowse_pid, Url#url.host}, Pid),
215+
Pid;
216+
Pid ->
217+
case is_process_alive(Pid) of
218+
true ->
219+
Pid;
220+
false ->
221+
erlang:erase({ibrowse_pid, Url#url.host}),
222+
connect(Url)
223+
end
224+
end.
225+
226+
227+
disconnect(Url) ->
228+
case erlang:get({ibrowse_pid, Url#url.host}) of
229+
undefined ->
230+
ok;
231+
OldPid ->
232+
catch(ibrowse_http_client:stop(OldPid))
233+
end,
234+
erlang:erase({ibrowse_pid, Url#url.host}),
235+
ok.
236+
237+
maybe_disconnect(Url) ->
238+
case erlang:get(disconnect_freq) of
239+
infinity -> ok;
240+
{ops, Count} -> should_disconnect_ops(Count,Url) andalso disconnect(Url);
241+
Seconds -> should_disconnect_secs(Seconds,Url) andalso disconnect(Url)
242+
end.
243+
244+
should_disconnect_ops(Count, Url) ->
245+
Key = {ops_since_disconnect, Url#url.host},
246+
case erlang:get(Key) of
247+
undefined ->
248+
erlang:put(Key, 1),
249+
false;
250+
Count ->
251+
erlang:put(Key, 0),
252+
true;
253+
Incr ->
254+
erlang:put(Key, Incr + 1),
255+
false
256+
end.
257+
258+
should_disconnect_secs(Seconds, Url) ->
259+
Key = {last_disconnect, Url#url.host},
260+
case erlang:get(Key) of
261+
undefined ->
262+
erlang:put(Key, erlang:now()),
263+
false;
264+
Time when is_tuple(Time) andalso size(Time) == 3 ->
265+
Diff = timer:now_diff(erlang:now(), Time),
266+
if
267+
Diff >= Seconds * 1000000 ->
268+
erlang:put(Key, erlang:now()),
269+
true;
270+
true -> false
271+
end
272+
end.
273+
274+
clear_disconnect_freq(Url) ->
275+
case erlang:get(disconnect_freq) of
276+
infinity -> ok;
277+
{ops, _Count} -> erlang:put({ops_since_disconnect, Url#url.host}, 0);
278+
_Seconds -> erlang:put({last_disconnect, Url#url.host}, erlang:now())
279+
end.
280+
281+
send_request(Url, Headers, Method, Body, Options) ->
282+
send_request(Url, Headers, Method, Body, Options, 3).
283+
284+
send_request(_Url, _Headers, _Method, _Body, _Options, 0) ->
285+
{error, max_retries};
286+
send_request(Url, Headers, Method, Body, Options, Count) ->
287+
Pid = connect(Url),
288+
case catch(ibrowse_http_client:send_req(Pid, Url, Headers ++ basho_bench_config:get(http_append_headers,[]), Method, Body, Options, basho_bench_config:get(http_request_timeout, 5000))) of
289+
{ok, Status, RespHeaders, RespBody} ->
290+
maybe_disconnect(Url),
291+
{ok, Status, RespHeaders, RespBody};
292+
293+
Error ->
294+
clear_disconnect_freq(Url),
295+
disconnect(Url),
296+
case should_retry(Error) of
297+
true ->
298+
send_request(Url, Headers, Method, Body, Options, Count-1);
299+
300+
false ->
301+
normalize_error(Method, Error)
302+
end
303+
end.
304+
305+
306+
should_retry({error, send_failed}) -> true;
307+
should_retry({error, connection_closed}) -> true;
308+
should_retry({'EXIT', {normal, _}}) -> true;
309+
should_retry({'EXIT', {noproc, _}}) -> true;
310+
should_retry(_) -> false.
311+
312+
normalize_error(Method, {'EXIT', {timeout, _}}) -> {error, {Method, timeout}};
313+
normalize_error(Method, {'EXIT', Reason}) -> {error, {Method, 'EXIT', Reason}};
314+
normalize_error(Method, {error, Reason}) -> {error, {Method, Reason}}.

0 commit comments

Comments
 (0)