tools: add cli flag to run profile dump parsers

It is really inconvenient to use a standalone shell script to parse
binary dumps from profilers. That is why this commit introduces a
CLI flag for tools in the LuaJIT, so now it is possible to parse
a memprof dump as simple as:
```
luajit -tm memprof.bin
luajit -tm --leak-only memprof.bin
```

And the sysprof too:
```
luajit -ts sysprof.bin
```

The scripts `luajit-parse-memprof` and `luajit-parse-sysprof`
are purged as a result of these changes.

Resolves tarantool/tarantool#5688

Reviewed-by: Sergey Kaplun <[email protected]>
Reviewed-by: Sergey Bronnikov <[email protected]>
Signed-off-by: Igor Munkin <[email protected]>

Author: mkokryashkin

Makefile.original

@@ -57,8 +57,6 @@ INSTALL_DYLIBSHORT1= libluajit-$(ABIVER).dylib
 INSTALL_DYLIBSHORT2= libluajit-$(ABIVER).$(MAJVER).dylib
 INSTALL_DYLIBNAME= libluajit-$(ABIVER).$(MAJVER).$(MINVER).$(RELVER).dylib
 INSTALL_PCNAME= luajit.pc
-INSTALL_TMEMPROFNAME= luajit-$(VERSION)-parse-memprof
-INSTALL_TMEMPROFSYMNAME= luajit-parse-memprof
 
 INSTALL_STATIC= $(INSTALL_LIB)/$(INSTALL_ANAME)
 INSTALL_DYN= $(INSTALL_LIB)/$(INSTALL_SONAME)
@@ -67,8 +65,6 @@ INSTALL_SHORT2= $(INSTALL_LIB)/$(INSTALL_SOSHORT2)
 INSTALL_T= $(INSTALL_BIN)/$(INSTALL_TNAME)
 INSTALL_TSYM= $(INSTALL_BIN)/$(INSTALL_TSYMNAME)
 INSTALL_PC= $(INSTALL_PKGCONFIG)/$(INSTALL_PCNAME)
-INSTALL_TMEMPROF= $(INSTALL_BIN)/$(INSTALL_TMEMPROFNAME)
-INSTALL_TMEMPROFSYM= $(INSTALL_BIN)/$(INSTALL_TMEMPROFSYMNAME)
 
 INSTALL_DIRS= $(INSTALL_BIN) $(INSTALL_LIB) $(INSTALL_INC) $(INSTALL_MAN) \
   $(INSTALL_PKGCONFIG) $(INSTALL_JITLIB) $(INSTALL_LMOD) $(INSTALL_CMOD) \
@@ -87,8 +83,6 @@ UNINSTALL= $(RM)
 LDCONFIG= ldconfig -n
 SED_PC= sed -e "s|@LUAJIT_PC_PREFIX@|$(PREFIX)|" \
             -e "s|@LUAJIT_PC_MULTILIB@|$(MULTILIB)|"
-SED_TMEMPROF= sed -e "s|@LUAJIT_TOOLS_DIR@|$(INSTALL_TOOLSLIB)|" \
-                  -e "s|@LUAJIT_TOOLS_BIN@|$(INSTALL_T)|"
 
 FILE_T= luajit
 FILE_A= libluajit.a
@@ -103,7 +97,6 @@ FILES_JITLIB= bc.lua bcsave.lua dump.lua p.lua v.lua zone.lua \
 FILES_UTILSLIB= avl.lua bufread.lua symtab.lua
 FILES_MEMPROFLIB= parse.lua humanize.lua
 FILES_TOOLSLIB= memprof.lua
-FILE_TMEMPROF= luajit-parse-memprof
 
 ifeq (,$(findstring Windows,$(OS)))
   HOST_SYS:= $(shell uname -s)
@@ -148,22 +141,18 @@ install: $(INSTALL_DEP)
 	cd tools/utils && $(INSTALL_F) $(FILES_UTILSLIB) $(INSTALL_UTILSLIB)
 	cd tools/memprof && $(INSTALL_F) $(FILES_MEMPROFLIB) $(INSTALL_MEMPROFLIB)
 	cd tools && $(INSTALL_F) $(FILES_TOOLSLIB) $(INSTALL_TOOLSLIB)
-	cd tools && $(SED_TMEMPROF) $(FILE_TMEMPROF).in > $(FILE_TMEMPROF) && \
-	  $(INSTALL_X) $(FILE_TMEMPROF) $(INSTALL_TMEMPROF) && \
-	  $(RM) $(FILE_TMEMPROF)
 	@echo "==== Successfully installed LuaJIT $(VERSION) to $(PREFIX) ===="
 	@echo ""
 	@echo "Note: the development releases deliberately do NOT install a symlink for luajit"
 	@echo "You can do this now by running these commands (with sudo):"
 	@echo ""
 	@echo "  $(SYMLINK) $(INSTALL_TNAME) $(INSTALL_TSYM)"
-	@echo "  $(SYMLINK) $(INSTALL_TMEMPROFNAME) $(INSTALL_TMEMPROFSYM)"
 	@echo ""
 
 
 uninstall:
 	@echo "==== Uninstalling LuaJIT $(VERSION) from $(PREFIX) ===="
-	$(UNINSTALL) $(INSTALL_T) $(INSTALL_STATIC) $(INSTALL_DYN) $(INSTALL_SHORT1) $(INSTALL_SHORT2) $(INSTALL_MAN)/$(FILE_MAN) $(INSTALL_PC) $(INSTALL_TMEMPROF)
+	$(UNINSTALL) $(INSTALL_T) $(INSTALL_STATIC) $(INSTALL_DYN) $(INSTALL_SHORT1) $(INSTALL_SHORT2) $(INSTALL_MAN)/$(FILE_MAN) $(INSTALL_PC)
 	for file in $(FILES_JITLIB); do \
 	  $(UNINSTALL) $(INSTALL_JITLIB)/$$file; \
 	  done
@@ -190,19 +179,8 @@ amalg: tools
 	$(MAKE) -C src -f Makefile.original amalg
 
 clean:
-	$(RM) tools/$(FILE_TMEMPROF)
 	$(MAKE) -C src -f Makefile.original clean
 
-tools: tools/$(FILE_TMEMPROF)
-
-# FIXME: This is an ugly hack to manually configure an auxiliary
-# tools/luajit-parse-memprof. This file should go away in scope of
-# https://github.com/tarantool/tarantool/issues/5688.
-tools/$(FILE_TMEMPROF): src/luajit
-	@sed -e "s|@LUAJIT_TOOLS_DIR@|$(realpath tools)|" \
-	     -e "s|@LUAJIT_TOOLS_BIN@|$(realpath src/luajit)|" \
-	     [email protected] > $@
-	@chmod +x $@
 
 .PHONY: all install amalg clean tools
 

src/luajit.c

@@ -72,12 +72,24 @@ static void print_usage(void)
   "  -O[opt]   Control LuaJIT optimizations.\n"
   "  -i        Enter interactive mode after executing " LUA_QL("script") ".\n"
   "  -v        Show version information.\n"
+  "  -t<cmd>   Execute tool.\n"
   "  -E        Ignore environment variables.\n"
   "  --        Stop handling options.\n"
   "  -         Execute stdin and stop handling options.\n", stderr);
   fflush(stderr);
 }
 
+static void print_tools_usage(void)
+{
+  fputs("usage: ", stderr);
+  fputs(progname, stderr);
+  fputs(" -t<cmd>\n"
+  "Available tools are:\n"
+  "  -m [--leak-only] input  Memprof profile data parser.\n"
+  "  -s input                Sysprof profile data parser.\n", stderr);
+  fflush(stderr);
+}
+
 static void l_message(const char *pname, const char *msg)
 {
   if (pname) { fputs(pname, stderr); fputc(':', stderr); fputc(' ', stderr); }
@@ -361,6 +373,37 @@ static int dojitcmd(lua_State *L, const char *cmd)
   return runcmdopt(L, opt ? opt+1 : opt);
 }
 
+static int runtoolcmd(lua_State *L, const char *tool_name)
+{
+  lua_getglobal(L, "require");
+  lua_pushstring(L, tool_name);
+  if (lua_pcall(L, 1, 1, 0)) {
+    const char *msg = lua_tostring(L, -1);
+    if (msg) {
+      if (!strncmp(msg, "module ", 7))
+	msg = "unknown luaJIT command or tools not installed";
+      l_message(progname, msg);
+    }
+    return 1;
+  }
+  lua_getglobal(L, "arg");
+  return report(L, lua_pcall(L, 1, 1, 0));
+}
+
+static int dotoolcmd(lua_State *L, const char *cmd)
+{
+  switch (cmd[0]) {
+  case 'm':
+    return runtoolcmd(L, "memprof");
+  case 's':
+    return runtoolcmd(L, "sysprof");
+  default:
+    print_tools_usage();
+    break;
+  }
+  return -1;
+}
+
 /* Optimization flags. */
 static int dojitopt(lua_State *L, const char *opt)
 {
@@ -398,6 +441,7 @@ static int dobytecode(lua_State *L, char **argv)
 #define FLAGS_EXEC		4
 #define FLAGS_OPTION		8
 #define FLAGS_NOENV		16
+#define FLAGS_TOOL		32
 
 static int collectargs(char **argv, int *flags)
 {
@@ -419,6 +463,9 @@ static int collectargs(char **argv, int *flags)
       notail(argv[i]);
       *flags |= FLAGS_VERSION;
       break;
+    case 't':
+      *flags |= FLAGS_TOOL;
+      return i + 1;
     case 'e':
       *flags |= FLAGS_EXEC;
       /* fallthrough */
@@ -475,6 +522,10 @@ static int runargs(lua_State *L, char **argv, int argn)
 	return 1;
       break;
       }
+    case 't': { /* Tarantool's fork extension. */
+      const char *cmd = argv[i] + 2;
+      return dotoolcmd(L, cmd) != LUA_OK;
+    }
     case 'O':  /* LuaJIT extension. */
       if (dojitopt(L, argv[i] + 2))
 	return 1;
@@ -536,7 +587,7 @@ static int pmain(lua_State *L)
   luaL_openlibs(L);
   lua_gc(L, LUA_GCRESTART, -1);
 
-  createargtable(L, argv, s->argc, argn);
+  createargtable(L, argv, s->argc, (flags & FLAGS_TOOL) ? argn - 1 : argn);
 
   if (!(flags & FLAGS_NOENV)) {
     s->status = handle_luainit(L);

test/tarantool-tests/gh-5688-tool-cli-flag.test.lua

@@ -0,0 +1,126 @@
+local tap = require('tap')
+local test = tap.test('gh-5688-tool-cli-flag'):skipcond({
+  ['Profile tools are implemented for x86_64 only'] = jit.arch ~= 'x86' and
+                                                      jit.arch ~= 'x64',
+  ['Profile tools are implemented for Linux only'] = jit.os ~= 'Linux',
+  -- XXX: Tarantool integration is required to run this test properly.
+  -- luacheck: no global
+  ['No profile tools CLI option integration'] = _TARANTOOL,
+})
+
+test:plan(3)
+
+jit.off()
+jit.flush()
+
+local table_new = require('table.new')
+local utils = require('utils')
+
+local BAD_PATH = utils.tools.profilename('bad-path-tmp.bin')
+local TMP_BINFILE_MEMPROF = utils.tools.profilename('memprofdata.tmp.bin')
+local TMP_BINFILE_SYSPROF = utils.tools.profilename('sysprofdata.tmp.bin')
+
+local EXECUTABLE = utils.exec.luacmd(arg)
+local MEMPROF_PARSER = EXECUTABLE .. ' -tm '
+local SYSPROF_PARSER = EXECUTABLE .. ' -ts '
+
+local REDIRECT_OUTPUT = ' 2>&1'
+
+local TABLE_SIZE = 20
+
+local SMOKE_CMD_SET = {
+  {
+    cmd = EXECUTABLE .. ' -t ' .. BAD_PATH,
+    like = '.+Available tools.+',
+  },
+  {
+    cmd = EXECUTABLE .. ' -ta ' .. BAD_PATH,
+    like = '.+Available tools.+',
+  },
+}
+
+local MEMPROF_CMD_SET = {
+  {
+    cmd = MEMPROF_PARSER .. BAD_PATH,
+    like = 'fopen, errno: 2',
+  },
+  {
+    cmd = MEMPROF_PARSER .. TMP_BINFILE_MEMPROF,
+    like = 'ALLOCATIONS.+',
+  },
+  {
+    cmd = MEMPROF_PARSER .. ' --wrong ' .. TMP_BINFILE_MEMPROF,
+    like = 'unrecognized option',
+  },
+  {
+    cmd = MEMPROF_PARSER .. ' --leak-only ' .. TMP_BINFILE_MEMPROF,
+    like = 'HEAP SUMMARY:.+',
+  },
+}
+
+local SYSPROF_CMD_SET = {
+  {
+    cmd = SYSPROF_PARSER .. BAD_PATH,
+    like = 'fopen, errno: 2',
+  },
+  {
+    cmd = SYSPROF_PARSER .. TMP_BINFILE_SYSPROF,
+    like = '[%w_@:;]+%s%d+\n*.*',
+  },
+  {
+    cmd = SYSPROF_PARSER .. ' --wrong ' .. TMP_BINFILE_SYSPROF,
+    like = 'unrecognized option',
+  },
+}
+
+local function memprof_payload()
+  local _ = table_new(TABLE_SIZE, 0)
+   _ = nil
+  collectgarbage()
+end
+
+local function sysprof_payload()
+  local function fib(n)
+    if n <= 1 then
+      return n
+    end
+    return fib(n - 1) + fib(n - 2)
+  end
+  return fib(32)
+end
+
+local function generate_profiler_output(opts, payload, profiler)
+  local res, err = profiler.start(opts)
+  -- Should start succesfully.
+  assert(res, err)
+
+  payload()
+
+  res, err = profiler.stop()
+  -- Should stop succesfully.
+  assert(res, err)
+end
+
+local function tool_test_case(case_name, cmd_set)
+  test:test(case_name, function(subtest)
+    subtest:plan(#cmd_set)
+
+    for idx = 1, #cmd_set do
+      local output = io.popen(cmd_set[idx].cmd .. REDIRECT_OUTPUT):read('*all')
+      subtest:like(output, cmd_set[idx].like, cmd_set[idx].cmd)
+    end
+  end)
+end
+
+tool_test_case('smoke', SMOKE_CMD_SET)
+
+generate_profiler_output(TMP_BINFILE_MEMPROF, memprof_payload, misc.memprof)
+tool_test_case('memprof parsing tool', MEMPROF_CMD_SET)
+
+local sysprof_opts = { mode = 'C', path = TMP_BINFILE_SYSPROF }
+generate_profiler_output(sysprof_opts, sysprof_payload, misc.sysprof)
+tool_test_case('sysprof parsing tool', SYSPROF_CMD_SET)
+
+os.remove(TMP_BINFILE_MEMPROF)
+os.remove(TMP_BINFILE_SYSPROF)
+test:done(true)