working on pypy verion, relocations and other changes

Author: planrich

README.md

@@ -31,28 +31,31 @@ Setting up development can be done using the following commands:
 	$ python setup.py develop
 
 Now it is time to write a test and implement your feature. If you want
-your changes to affect vmprof.com, please head over to
+your changes to affect vmprof.com, head over to
 https://github.com/vmprof/vmprof-server and follow the setup instructions.
 
-Please also consult our section for development at https://vmprof.readthedocs.org.
+Consult our section for development at https://vmprof.readthedocs.org for more
+information.
 
 ## vmprofshow
 
-`vmprofshow` is a command line tool that comes with **VMprof** which can read profile files generated by **VMprof** and produce a nicely formatted output.
+`vmprofshow` is a command line tool that comes with **VMProf** which can read profile files
+and produce a formatted output.
 
-Here is an example of how to use:
+Here is an example of how to use `vmprofshow`:
 
 Clone the vmprof repo first to use a minimalistic cpuburn.py:
 
 ```console
 git clone https://github.com/vmprof/vmprof-python
 cd vmprof-python
+pip install .
 ```
 
 Run that smallish program which burns CPU cycles (with vmprof enabled):
 
 ```console
-python vmprof/test/cpuburn.py
+pypy vmprof/test/cpuburn.py
 ```
 
 This will produce a profile file `vmprof_cpuburn.dat`.

setup.py

@@ -58,7 +58,7 @@
                                'src/_vmprof.c',
                                'src/machine.c',
                                'src/compat.c',
-                               'src/stack.c',
+                               'src/vmp_stack.c',
                                ] + extra_source_files,
                            depends=[
                                'src/vmprof_main.h',

src/_vmprof.c

@@ -21,7 +21,7 @@ static volatile int is_enabled = 0;
 #else
 #include "vmprof_main_win32.h"
 #endif
-#include "stack.h"
+#include "vmp_stack.h"
 
 static destructor Original_code_dealloc = 0;
 PyObject* (*_default_eval_loop)(PyFrameObject *, int) = 0;
@@ -32,12 +32,12 @@ __attribute__((disable_tail_calls))
 #elif defined(__GNUC__)
 __attribute__((optimize("O1")))
 #endif
-PyObject* vmprof_eval(PyFrameObject *f, int throwflag)
+PY_EVAL_RETURN_T * vmprof_eval(PY_STACK_FRAME_T *f, int throwflag)
 {
 #ifdef X86_64
-    register PyFrameObject * callee_saved asm("rbx");
+    register PY_STACK_FRAME_T * callee_saved asm("rbx");
 #elif defined(X86_32)
-    register PyFrameObject * callee_saved asm("edi");
+    register PY_STACK_FRAME_T * callee_saved asm("edi");
 #else
 #    error "platform not supported"
 #endif
@@ -150,30 +150,6 @@ static void cpyprof_code_dealloc(PyObject *co)
     Original_code_dealloc(co);
 }
 
-#ifdef VMP_SUPPORTS_NATIVE_PROFILING
-static void init_cpyprof(int native)
-{
-    // skip this if native should not be enabled
-    if (!native) {
-        vmp_native_disable();
-        return;
-    }
-#if CPYTHON_HAS_FRAME_EVALUATION
-    PyThreadState *tstate = PyThreadState_GET();
-    tstate->interp->eval_frame = vmprof_eval;
-    _default_eval_loop = _PyEval_EvalFrameDefault;
-#else
-    if (vmp_patch_callee_trampoline(PyEval_EvalFrameEx,
-                vmprof_eval, (void*)&_default_eval_loop) == 0) {
-    } else {
-        fprintf(stderr, "FATAL: could not insert trampline, try with --no-native\n");
-        // TODO dump the first few bytes and tell them to create an issue!
-        exit(-1);
-    }
-#endif
-    vmp_native_enable();
-}
-#endif
 
 void dump_native_symbols(int fileno)
 {
@@ -189,22 +165,6 @@ void dump_native_symbols(int fileno)
     Py_XDECREF(mod);
 }
 
-#ifdef VMP_SUPPORTS_NATIVE_PROFILING
-static void disable_cpyprof(void)
-{
-    vmp_native_disable();
-#if CPYTHON_HAS_FRAME_EVALUATION
-    PyThreadState *tstate = PyThreadState_GET();
-    tstate->interp->eval_frame = _PyEval_EvalFrameDefault;
-#else
-    if (vmp_unpatch_callee_trampoline(PyEval_EvalFrameEx) > 0) {
-        fprintf(stderr, "FATAL: could not remove trampoline\n");
-        exit(-1);
-    }
-#endif
-    dump_native_symbols(vmp_profile_fileno());
-}
-#endif
 
 
 static PyObject *enable_vmprof(PyObject* self, PyObject *args)
@@ -229,10 +189,6 @@ static PyObject *enable_vmprof(PyObject* self, PyObject *args)
 
     vmp_profile_lines(lines);
 
-#ifdef VMP_SUPPORTS_NATIVE_PROFILING
-    init_cpyprof(native);
-#endif
-
     if (!Original_code_dealloc) {
         Original_code_dealloc = PyCode_Type.tp_dealloc;
         PyCode_Type.tp_dealloc = &cpyprof_code_dealloc;
@@ -244,7 +200,7 @@ static PyObject *enable_vmprof(PyObject* self, PyObject *args)
         return NULL;
     }
 
-    if (vmprof_enable(memory) < 0) {
+    if (vmprof_enable(memory, native) < 0) {
         PyErr_SetFromErrno(PyExc_OSError);
         return NULL;
     }
@@ -265,9 +221,6 @@ disable_vmprof(PyObject *module, PyObject *noarg)
     is_enabled = 0;
     vmprof_ignore_signals(1);
     emit_all_code_objects();
-#ifdef VMP_SUPPORTS_NATIVE_PROFILING
-    disable_cpyprof();
-#endif
 
     if (vmprof_disable() < 0) {
         PyErr_SetFromErrno(PyExc_OSError);
@@ -319,7 +272,7 @@ sample_stack_now(PyObject *module, PyObject *args)
         vmprof_ignore_signals(0);
         return NULL;
     }
-    entry_count = vmp_walk_and_record_stack(tstate->frame, m, MAX_STACK_DEPTH-1, 0);
+    entry_count = vmp_walk_and_record_stack(tstate->frame, m, MAX_STACK_DEPTH-1, 0, 0);
 
     for (i = 0; i < entry_count; i++) {
         routine_ip = m[i];

src/_vmprof.h

@@ -1,14 +1,14 @@
 #pragma once
 
-#include <Python.h>
-#include <frameobject.h>
+#include "vmprof.h"
 
 #ifdef VMPROF_WINDOWS
 #include "msiinttypes/inttypes.h"
 #include "msiinttypes/stdint.h"
 #else
 #include <inttypes.h>
 #include <stdint.h>
+#include <stddef.h>
 #endif
 
 /**
@@ -43,37 +43,4 @@
 
 #define CPYTHON_HAS_FRAME_EVALUATION PY_VERSION_HEX >= 0x30600B0
 
-PyObject* vmprof_eval(PyFrameObject *f, int throwflag);
-
-#ifdef VMPROF_UNIX
-#define VMP_SUPPORTS_NATIVE_PROFILING
-#endif
-
-#ifdef __x86_64__
-#define X86_64
-#elif defined(__i386__)
-#define X86_32
-#endif
-
-#define MARKER_STACKTRACE '\x01'
-#define MARKER_VIRTUAL_IP '\x02'
-#define MARKER_TRAILER '\x03'
-#define MARKER_INTERP_NAME '\x04'   /* deprecated */
-#define MARKER_HEADER '\x05'
-#define MARKER_TIME_N_ZONE '\x06'
-#define MARKER_META '\x07'
-#define MARKER_NATIVE_SYMBOLS '\x08'
-
-#define VERSION_BASE '\x00'
-#define VERSION_THREAD_ID '\x01'
-#define VERSION_TAG '\x02'
-#define VERSION_MEMORY '\x03'
-#define VERSION_MODE_AWARE '\x04'
-#define VERSION_DURATION '\x05'
-#define VERSION_TIMESTAMP '\x06'
-
-#define PROFILE_MEMORY '\x01'
-#define PROFILE_LINES  '\x02'
-#define PROFILE_NATIVE '\x04'
-
 int vmp_write_all(const char *buf, size_t bufsize);

src/compat.c

@@ -1,10 +1,13 @@
 #include "compat.h"
 
-#include "_vmprof.h"
-
+#include <string.h>
+#include <assert.h>
 #if VMPROF_WINDOWS
 #define WIN32_LEAN_AND_MEAN
 #include <Windows.h>
+#else
+#include <time.h>
+#include <sys/time.h>
 #endif
 
 static int _vmp_profile_fileno = -1;

src/compat.h

@@ -1,17 +1,19 @@
 #pragma once
 
-INCLUDE_MODULE_HEADER
+#include "vmprof.h"
 
-#if PY_MAJOR_VERSION >= 3
-    #define PyStr_AS_STRING PyBytes_AS_STRING
-    #define PyStr_GET_SIZE PyBytes_GET_SIZE
-    #define PyStr_NEW      PyUnicode_FromString
-    #define PyLong_NEW     PyLong_FromLong
-#else
-    #define PyStr_AS_STRING PyString_AS_STRING
-    #define PyStr_GET_SIZE PyString_GET_SIZE
-    #define PyStr_NEW      PyString_FromString
-    #define PyLong_NEW     PyInt_FromLong
+#ifndef RPYTHON_VMPROF
+#  if PY_MAJOR_VERSION >= 3
+      #define PyStr_AS_STRING PyBytes_AS_STRING
+      #define PyStr_GET_SIZE PyBytes_GET_SIZE
+      #define PyStr_NEW      PyUnicode_FromString
+      #define PyLong_NEW     PyLong_FromLong
+#  else
+      #define PyStr_AS_STRING PyString_AS_STRING
+      #define PyStr_GET_SIZE PyString_GET_SIZE
+      #define PyStr_NEW      PyString_FromString
+      #define PyLong_NEW     PyInt_FromLong
+#  endif
 #endif
 
 int vmp_write_all(const char *buf, size_t bufsize);

src/machine.c

@@ -1,6 +1,6 @@
 #include "machine.h"
 
-#include "_vmprof.h"
+#include "vmprof.h"
 #include <stdio.h>
 
 int vmp_machine_bits(void)

src/rss_unix.h

@@ -1,3 +1,5 @@
+#include <stdlib.h>
+
 /* On normal Unices we can get RSS from '/proc/<pid>/status'. */
 static int proc_file = -1;
 

src/symboltable.c

@@ -1,16 +1,16 @@
 #include "symboltable.h"
 
-#include "_vmprof.h"
+#include "vmprof.h"
 #include "machine.h"
 
+#include <string.h>
 #include <stdio.h>
 #include <stdlib.h>
 
+#include <dlfcn.h>
 #ifdef VMPROF_LINUX
-#define _GNU_SOURCE 1
 #include <link.h>
 #endif
-#include <dlfcn.h>
 
 #ifdef _PY_TEST
 #define LOG(...) printf(__VA_ARGS__)
@@ -220,7 +220,7 @@ int vmp_resolve_addr(void * addr, char * name, int name_len, int * lineno, char
         name[name_len-1] = 0;
     }
     lookup_vmprof_debug_info(name, info.dli_fbase, srcfile, srcfile_len, lineno);
-#elif defined(__unix__)
+#elif defined(VMPROF_LINUX)
     if (bstate == NULL) {
         bstate = backtrace_create_state (NULL, 1, backtrace_error_cb, NULL);
     }

src/symboltable.h

@@ -1,15 +1,19 @@
 #pragma once
 
+#define _GNU_SOURCE 1
+
 /**
  * Extract all the known symbols from the current process and
  * log them to the file descriptor. To read them see binary.py funcs:
  *
  * # encoded as a mapping
  * addr = read_word(fd); name = read_string(fd)
  *
- * A) It is not allowed to have two addresses (virtual ones only valid
+ * A) It is not allowed to have two addresses (virtual ones are only valid
  * in the curent process) in this mapping to point to several symbols.
  * B) No duplicates are logged
+ *
+ * Deprecated, do not use
  */
 void dump_all_known_symbols(int fd);