Skip to content

Commit 0380a8c

Browse files
mkokryashkinigormunkin
authored andcommitted
tools: add cli flag to run profile dump parsers
It is really inconvenient to use a standalone shell script to parse binary dumps from profilers. That is why this commit introduces a CLI flag for tools in the LuaJIT, so now it is possible to parse a memprof dump as simple as: ``` luajit -tm memprof.bin luajit -tm --leak-only memprof.bin ``` And the sysprof too: ``` luajit -ts sysprof.bin ``` The scripts `luajit-parse-memprof` and `luajit-parse-sysprof` are purged as a result of these changes. Resolves tarantool/tarantool#5688 Reviewed-by: Sergey Kaplun <[email protected]> Reviewed-by: Sergey Bronnikov <[email protected]> Signed-off-by: Igor Munkin <[email protected]>
1 parent 090117d commit 0380a8c

File tree

8 files changed

+194
-124
lines changed

8 files changed

+194
-124
lines changed

Makefile.original

Lines changed: 1 addition & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -57,8 +57,6 @@ INSTALL_DYLIBSHORT1= libluajit-$(ABIVER).dylib
5757
INSTALL_DYLIBSHORT2= libluajit-$(ABIVER).$(MAJVER).dylib
5858
INSTALL_DYLIBNAME= libluajit-$(ABIVER).$(MAJVER).$(MINVER).$(RELVER).dylib
5959
INSTALL_PCNAME= luajit.pc
60-
INSTALL_TMEMPROFNAME= luajit-$(VERSION)-parse-memprof
61-
INSTALL_TMEMPROFSYMNAME= luajit-parse-memprof
6260

6361
INSTALL_STATIC= $(INSTALL_LIB)/$(INSTALL_ANAME)
6462
INSTALL_DYN= $(INSTALL_LIB)/$(INSTALL_SONAME)
@@ -67,8 +65,6 @@ INSTALL_SHORT2= $(INSTALL_LIB)/$(INSTALL_SOSHORT2)
6765
INSTALL_T= $(INSTALL_BIN)/$(INSTALL_TNAME)
6866
INSTALL_TSYM= $(INSTALL_BIN)/$(INSTALL_TSYMNAME)
6967
INSTALL_PC= $(INSTALL_PKGCONFIG)/$(INSTALL_PCNAME)
70-
INSTALL_TMEMPROF= $(INSTALL_BIN)/$(INSTALL_TMEMPROFNAME)
71-
INSTALL_TMEMPROFSYM= $(INSTALL_BIN)/$(INSTALL_TMEMPROFSYMNAME)
7268

7369
INSTALL_DIRS= $(INSTALL_BIN) $(INSTALL_LIB) $(INSTALL_INC) $(INSTALL_MAN) \
7470
$(INSTALL_PKGCONFIG) $(INSTALL_JITLIB) $(INSTALL_LMOD) $(INSTALL_CMOD) \
@@ -87,8 +83,6 @@ UNINSTALL= $(RM)
8783
LDCONFIG= ldconfig -n
8884
SED_PC= sed -e "s|@LUAJIT_PC_PREFIX@|$(PREFIX)|" \
8985
-e "s|@LUAJIT_PC_MULTILIB@|$(MULTILIB)|"
90-
SED_TMEMPROF= sed -e "s|@LUAJIT_TOOLS_DIR@|$(INSTALL_TOOLSLIB)|" \
91-
-e "s|@LUAJIT_TOOLS_BIN@|$(INSTALL_T)|"
9286

9387
FILE_T= luajit
9488
FILE_A= libluajit.a
@@ -103,7 +97,6 @@ FILES_JITLIB= bc.lua bcsave.lua dump.lua p.lua v.lua zone.lua \
10397
FILES_UTILSLIB= avl.lua bufread.lua symtab.lua
10498
FILES_MEMPROFLIB= parse.lua humanize.lua
10599
FILES_TOOLSLIB= memprof.lua
106-
FILE_TMEMPROF= luajit-parse-memprof
107100

108101
ifeq (,$(findstring Windows,$(OS)))
109102
HOST_SYS:= $(shell uname -s)
@@ -148,22 +141,18 @@ install: $(INSTALL_DEP)
148141
cd tools/utils && $(INSTALL_F) $(FILES_UTILSLIB) $(INSTALL_UTILSLIB)
149142
cd tools/memprof && $(INSTALL_F) $(FILES_MEMPROFLIB) $(INSTALL_MEMPROFLIB)
150143
cd tools && $(INSTALL_F) $(FILES_TOOLSLIB) $(INSTALL_TOOLSLIB)
151-
cd tools && $(SED_TMEMPROF) $(FILE_TMEMPROF).in > $(FILE_TMEMPROF) && \
152-
$(INSTALL_X) $(FILE_TMEMPROF) $(INSTALL_TMEMPROF) && \
153-
$(RM) $(FILE_TMEMPROF)
154144
@echo "==== Successfully installed LuaJIT $(VERSION) to $(PREFIX) ===="
155145
@echo ""
156146
@echo "Note: the development releases deliberately do NOT install a symlink for luajit"
157147
@echo "You can do this now by running these commands (with sudo):"
158148
@echo ""
159149
@echo " $(SYMLINK) $(INSTALL_TNAME) $(INSTALL_TSYM)"
160-
@echo " $(SYMLINK) $(INSTALL_TMEMPROFNAME) $(INSTALL_TMEMPROFSYM)"
161150
@echo ""
162151

163152

164153
uninstall:
165154
@echo "==== Uninstalling LuaJIT $(VERSION) from $(PREFIX) ===="
166-
$(UNINSTALL) $(INSTALL_T) $(INSTALL_STATIC) $(INSTALL_DYN) $(INSTALL_SHORT1) $(INSTALL_SHORT2) $(INSTALL_MAN)/$(FILE_MAN) $(INSTALL_PC) $(INSTALL_TMEMPROF)
155+
$(UNINSTALL) $(INSTALL_T) $(INSTALL_STATIC) $(INSTALL_DYN) $(INSTALL_SHORT1) $(INSTALL_SHORT2) $(INSTALL_MAN)/$(FILE_MAN) $(INSTALL_PC)
167156
for file in $(FILES_JITLIB); do \
168157
$(UNINSTALL) $(INSTALL_JITLIB)/$$file; \
169158
done
@@ -190,19 +179,8 @@ amalg: tools
190179
$(MAKE) -C src -f Makefile.original amalg
191180

192181
clean:
193-
$(RM) tools/$(FILE_TMEMPROF)
194182
$(MAKE) -C src -f Makefile.original clean
195183

196-
tools: tools/$(FILE_TMEMPROF)
197-
198-
# FIXME: This is an ugly hack to manually configure an auxiliary
199-
# tools/luajit-parse-memprof. This file should go away in scope of
200-
# https://github.com/tarantool/tarantool/issues/5688.
201-
tools/$(FILE_TMEMPROF): src/luajit
202-
@sed -e "s|@LUAJIT_TOOLS_DIR@|$(realpath tools)|" \
203-
-e "s|@LUAJIT_TOOLS_BIN@|$(realpath src/luajit)|" \
204-
205-
@chmod +x $@
206184

207185
.PHONY: all install amalg clean tools
208186

src/luajit.c

Lines changed: 52 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,12 +72,24 @@ static void print_usage(void)
7272
" -O[opt] Control LuaJIT optimizations.\n"
7373
" -i Enter interactive mode after executing " LUA_QL("script") ".\n"
7474
" -v Show version information.\n"
75+
" -t<cmd> Execute tool.\n"
7576
" -E Ignore environment variables.\n"
7677
" -- Stop handling options.\n"
7778
" - Execute stdin and stop handling options.\n", stderr);
7879
fflush(stderr);
7980
}
8081

82+
static void print_tools_usage(void)
83+
{
84+
fputs("usage: ", stderr);
85+
fputs(progname, stderr);
86+
fputs(" -t<cmd>\n"
87+
"Available tools are:\n"
88+
" -m [--leak-only] input Memprof profile data parser.\n"
89+
" -s input Sysprof profile data parser.\n", stderr);
90+
fflush(stderr);
91+
}
92+
8193
static void l_message(const char *pname, const char *msg)
8294
{
8395
if (pname) { fputs(pname, stderr); fputc(':', stderr); fputc(' ', stderr); }
@@ -361,6 +373,37 @@ static int dojitcmd(lua_State *L, const char *cmd)
361373
return runcmdopt(L, opt ? opt+1 : opt);
362374
}
363375

376+
static int runtoolcmd(lua_State *L, const char *tool_name)
377+
{
378+
lua_getglobal(L, "require");
379+
lua_pushstring(L, tool_name);
380+
if (lua_pcall(L, 1, 1, 0)) {
381+
const char *msg = lua_tostring(L, -1);
382+
if (msg) {
383+
if (!strncmp(msg, "module ", 7))
384+
msg = "unknown luaJIT command or tools not installed";
385+
l_message(progname, msg);
386+
}
387+
return 1;
388+
}
389+
lua_getglobal(L, "arg");
390+
return report(L, lua_pcall(L, 1, 1, 0));
391+
}
392+
393+
static int dotoolcmd(lua_State *L, const char *cmd)
394+
{
395+
switch (cmd[0]) {
396+
case 'm':
397+
return runtoolcmd(L, "memprof");
398+
case 's':
399+
return runtoolcmd(L, "sysprof");
400+
default:
401+
print_tools_usage();
402+
break;
403+
}
404+
return -1;
405+
}
406+
364407
/* Optimization flags. */
365408
static int dojitopt(lua_State *L, const char *opt)
366409
{
@@ -398,6 +441,7 @@ static int dobytecode(lua_State *L, char **argv)
398441
#define FLAGS_EXEC 4
399442
#define FLAGS_OPTION 8
400443
#define FLAGS_NOENV 16
444+
#define FLAGS_TOOL 32
401445

402446
static int collectargs(char **argv, int *flags)
403447
{
@@ -419,6 +463,9 @@ static int collectargs(char **argv, int *flags)
419463
notail(argv[i]);
420464
*flags |= FLAGS_VERSION;
421465
break;
466+
case 't':
467+
*flags |= FLAGS_TOOL;
468+
return i + 1;
422469
case 'e':
423470
*flags |= FLAGS_EXEC;
424471
/* fallthrough */
@@ -475,6 +522,10 @@ static int runargs(lua_State *L, char **argv, int argn)
475522
return 1;
476523
break;
477524
}
525+
case 't': { /* Tarantool's fork extension. */
526+
const char *cmd = argv[i] + 2;
527+
return dotoolcmd(L, cmd) != LUA_OK;
528+
}
478529
case 'O': /* LuaJIT extension. */
479530
if (dojitopt(L, argv[i] + 2))
480531
return 1;
@@ -536,7 +587,7 @@ static int pmain(lua_State *L)
536587
luaL_openlibs(L);
537588
lua_gc(L, LUA_GCRESTART, -1);
538589

539-
createargtable(L, argv, s->argc, argn);
590+
createargtable(L, argv, s->argc, (flags & FLAGS_TOOL) ? argn - 1 : argn);
540591

541592
if (!(flags & FLAGS_NOENV)) {
542593
s->status = handle_luainit(L);
Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
local tap = require('tap')
2+
local test = tap.test('gh-5688-tool-cli-flag'):skipcond({
3+
['Profile tools are implemented for x86_64 only'] = jit.arch ~= 'x86' and
4+
jit.arch ~= 'x64',
5+
['Profile tools are implemented for Linux only'] = jit.os ~= 'Linux',
6+
-- XXX: Tarantool integration is required to run this test properly.
7+
-- luacheck: no global
8+
['No profile tools CLI option integration'] = _TARANTOOL,
9+
})
10+
11+
test:plan(3)
12+
13+
jit.off()
14+
jit.flush()
15+
16+
local table_new = require('table.new')
17+
local utils = require('utils')
18+
19+
local BAD_PATH = utils.tools.profilename('bad-path-tmp.bin')
20+
local TMP_BINFILE_MEMPROF = utils.tools.profilename('memprofdata.tmp.bin')
21+
local TMP_BINFILE_SYSPROF = utils.tools.profilename('sysprofdata.tmp.bin')
22+
23+
local EXECUTABLE = utils.exec.luacmd(arg)
24+
local MEMPROF_PARSER = EXECUTABLE .. ' -tm '
25+
local SYSPROF_PARSER = EXECUTABLE .. ' -ts '
26+
27+
local REDIRECT_OUTPUT = ' 2>&1'
28+
29+
local TABLE_SIZE = 20
30+
31+
local SMOKE_CMD_SET = {
32+
{
33+
cmd = EXECUTABLE .. ' -t ' .. BAD_PATH,
34+
like = '.+Available tools.+',
35+
},
36+
{
37+
cmd = EXECUTABLE .. ' -ta ' .. BAD_PATH,
38+
like = '.+Available tools.+',
39+
},
40+
}
41+
42+
local MEMPROF_CMD_SET = {
43+
{
44+
cmd = MEMPROF_PARSER .. BAD_PATH,
45+
like = 'fopen, errno: 2',
46+
},
47+
{
48+
cmd = MEMPROF_PARSER .. TMP_BINFILE_MEMPROF,
49+
like = 'ALLOCATIONS.+',
50+
},
51+
{
52+
cmd = MEMPROF_PARSER .. ' --wrong ' .. TMP_BINFILE_MEMPROF,
53+
like = 'unrecognized option',
54+
},
55+
{
56+
cmd = MEMPROF_PARSER .. ' --leak-only ' .. TMP_BINFILE_MEMPROF,
57+
like = 'HEAP SUMMARY:.+',
58+
},
59+
}
60+
61+
local SYSPROF_CMD_SET = {
62+
{
63+
cmd = SYSPROF_PARSER .. BAD_PATH,
64+
like = 'fopen, errno: 2',
65+
},
66+
{
67+
cmd = SYSPROF_PARSER .. TMP_BINFILE_SYSPROF,
68+
like = '[%w_@:;]+%s%d+\n*.*',
69+
},
70+
{
71+
cmd = SYSPROF_PARSER .. ' --wrong ' .. TMP_BINFILE_SYSPROF,
72+
like = 'unrecognized option',
73+
},
74+
}
75+
76+
local function memprof_payload()
77+
local _ = table_new(TABLE_SIZE, 0)
78+
_ = nil
79+
collectgarbage()
80+
end
81+
82+
local function sysprof_payload()
83+
local function fib(n)
84+
if n <= 1 then
85+
return n
86+
end
87+
return fib(n - 1) + fib(n - 2)
88+
end
89+
return fib(32)
90+
end
91+
92+
local function generate_profiler_output(opts, payload, profiler)
93+
local res, err = profiler.start(opts)
94+
-- Should start succesfully.
95+
assert(res, err)
96+
97+
payload()
98+
99+
res, err = profiler.stop()
100+
-- Should stop succesfully.
101+
assert(res, err)
102+
end
103+
104+
local function tool_test_case(case_name, cmd_set)
105+
test:test(case_name, function(subtest)
106+
subtest:plan(#cmd_set)
107+
108+
for idx = 1, #cmd_set do
109+
local output = io.popen(cmd_set[idx].cmd .. REDIRECT_OUTPUT):read('*all')
110+
subtest:like(output, cmd_set[idx].like, cmd_set[idx].cmd)
111+
end
112+
end)
113+
end
114+
115+
tool_test_case('smoke', SMOKE_CMD_SET)
116+
117+
generate_profiler_output(TMP_BINFILE_MEMPROF, memprof_payload, misc.memprof)
118+
tool_test_case('memprof parsing tool', MEMPROF_CMD_SET)
119+
120+
local sysprof_opts = { mode = 'C', path = TMP_BINFILE_SYSPROF }
121+
generate_profiler_output(sysprof_opts, sysprof_payload, misc.sysprof)
122+
tool_test_case('sysprof parsing tool', SYSPROF_CMD_SET)
123+
124+
os.remove(TMP_BINFILE_MEMPROF)
125+
os.remove(TMP_BINFILE_SYSPROF)
126+
test:done(true)

0 commit comments

Comments
 (0)