Update debug, perf, trace and page_cache

Fix issue:
freelancer-leon#2

Signed-off-by: Leon <[email protected]>

Author: freelancer-leon

kernel/debug/debug.md

@@ -1,5 +1,56 @@
 # 调试
 
+# 常用调试选项
+## lib/Kconfig.debug
+```c
+config DEBUG_INFO
+    bool "Compile the kernel with debug info"
+    depends on DEBUG_KERNEL && !COMPILE_TEST
+    help
+      If you say Y here the resulting kernel image will include
+      debugging info resulting in a larger kernel image.
+      This adds debug symbols to the kernel and modules (gcc -g), and
+      is needed if you intend to use kernel crashdump or binary object
+      tools like crash, kgdb, LKCD, gdb, etc on the kernel.
+      Say Y here only if you plan to debug the kernel.
+
+      If unsure, say N.
+config DEBUG_KERNEL
+    bool "Kernel debugging"
+    help
+      Say Y here if you are developing drivers or trying to debug and
+      identify kernel problems.
+
+```
+* 不开启`COINFIG_DEBUG_INFO`无法在反汇编`vmlinux`时内联 C 源代码
+## init/Kconfig
+```c
+config KALLSYMS
+    bool "Load all symbols for debugging/ksymoops" if EXPERT
+    default y
+    help
+      Say Y here to let the kernel print out symbolic crash information and
+      symbolic stack backtraces. This increases the size of the kernel
+      somewhat, as all symbols have to be loaded into the kernel image.
+
+config KALLSYMS_ALL
+    bool "Include all symbols in kallsyms"
+    depends on DEBUG_KERNEL && KALLSYMS
+    help
+      Normally kallsyms only contains the symbols of functions for nicer
+      OOPS messages and backtraces (i.e., symbols from the text and inittext
+      sections). This is sufficient for most cases. And only in very rare
+      cases (e.g., when a debugger is used) all symbols are required (e.g.,
+      names of variables from the data sections, etc).
+
+      This option makes sure that all symbols are loaded into the kernel
+      image (i.e., symbols from all sections) in cost of increased kernel
+      size (depending on the kernel configuration, it may be 300KiB or
+      something like this).
+
+      Say N unless you really need all symbols.
+```
+
 # 打印（print）
 
 ## printk
@@ -13,7 +64,7 @@
   Append ``",keep"`` to not disable it when the real console takes over.
   ```
 
-  ```
+  ```c
   keep_bootcon    [KNL]
           Do not unregister boot console at start. This is only
           useful for debugging when something happens in the window
@@ -28,7 +79,7 @@
     printk(KERN_DEBUG “error=%d\n”, error);
   ```
 * 调整
-  ```
+  ```sh
   echo 5 > /proc/sys/kernel/printk_ratelimit        # Wait sec before re-open printk
   echo 10 > /proc/sys/kernel/printk_ratelimit_burst # Message number before limiting the rate
   ```
@@ -37,7 +88,7 @@
 
 ### loglevel
 
-```
+```c
 loglevel=   All Kernel Messages with a loglevel smaller than the
         console loglevel will be printed to the console. It can
         also be changed with klogd or other programs. The
@@ -141,7 +192,7 @@ static int __init loglevel(char *str)
 early_param("loglevel", loglevel);
 ...*```
 ```
-### 调整
+### 调整 log level
 
 * 如果不指定 printk 打印的 log level，那么它缺省的级别是 `DEFAULT_MESSAGE_LOGLEVEL`（通常 "4"=KERN_WARNING）
 * 该缺省值可以通过 `CONFIG_DEFAULT_MESSAGE_LOGLEVEL` kernel config 选项 (make menuconfig-> Kernel Hacking -> Default message log level) 调整。
@@ -158,7 +209,7 @@ early_param("loglevel", loglevel);
   # echo 8 > /proc/sys/kernel/printk
   ```
 ### logbuffer
-```
+```c
 log_buf_len=n[KMG]  Sets the size of the printk ring buffer,
         in bytes.  n must be a power of two and greater
         than the minimal size. The minimal size is defined
@@ -194,7 +245,7 @@ log_buf_len=n[KMG]  Sets the size of the printk ring buffer,
 * sysrq 的键映射表见 drivers/tty/sysrq.c 的`struct sysrq_key_op *sysrq_key_table[]`数组
 
 ## git二分法查找
-```
+```sh
 git bisect start
 git bisect bad [revision bug on]
 git bisect good [revision no bug]
@@ -209,7 +260,80 @@ git bisect start - arch/x86
 ```
 
 # Dynamic Debug
-* CONFIG_DYNAMIC_DEBUG
+* `CONFIG_DYNAMIC_DEBUG` (lib/Kconfig.debug)
+```c
+config DYNAMIC_DEBUG
+    bool "Enable dynamic printk() support"
+    default n
+    depends on PRINTK
+    depends on (DEBUG_FS || PROC_FS)
+    select DYNAMIC_DEBUG_CORE
+    help
+
+      Compiles debug level messages into the kernel, which would not
+      otherwise be available at runtime. These messages can then be
+      enabled/disabled based on various levels of scope - per source file,
+      function, module, format string, and line number. This mechanism
+      implicitly compiles in all pr_debug() and dev_dbg() calls, which
+      enlarges the kernel text size by about 2%.
+
+      If a source file is compiled with DEBUG flag set, any
+      pr_debug() calls in it are enabled by default, but can be
+      disabled at runtime as below.  Note that DEBUG flag is
+      turned on by many CONFIG_*DEBUG* options.
+
+      Usage:
+
+      Dynamic debugging is controlled via the 'dynamic_debug/control' file,
+      which is contained in the 'debugfs' filesystem or procfs.
+      Thus, the debugfs or procfs filesystem must first be mounted before
+      making use of this feature.
+      We refer the control file as: <debugfs>/dynamic_debug/control. This
+      file contains a list of the debug statements that can be enabled. The
+      format for each line of the file is:
+
+        filename:lineno [module]function flags format
+
+      filename : source file of the debug statement
+      lineno : line number of the debug statement
+      module : module that contains the debug statement
+      function : function that contains the debug statement
+      flags : '=p' means the line is turned 'on' for printing
+      format : the format used for the debug statement
+
+      From a live system:
+
+        nullarbor:~ # cat <debugfs>/dynamic_debug/control
+        # filename:lineno [module]function flags format
+        fs/aio.c:222 [aio]__put_ioctx =_ "__put_ioctx:\040freeing\040%p\012"
+        fs/aio.c:248 [aio]ioctx_alloc =_ "ENOMEM:\040nr_events\040too\040high\012"
+        fs/aio.c:1770 [aio]sys_io_cancel =_ "calling\040cancel\012"
+
+        Example usage:
+
+          // enable the message at line 1603 of file svcsock.c
+          nullarbor:~ # echo -n 'file svcsock.c line 1603 +p' >
+                          <debugfs>/dynamic_debug/control
+
+          // enable all the messages in file svcsock.c
+          nullarbor:~ # echo -n 'file svcsock.c +p' >
+                          <debugfs>/dynamic_debug/control
+
+          // enable all the messages in the NFS server module
+          nullarbor:~ # echo -n 'module nfsd +p' >
+                          <debugfs>/dynamic_debug/control
+
+          // enable all 12 messages in the function svc_process()
+          nullarbor:~ # echo -n 'func svc_process +p' >
+                          <debugfs>/dynamic_debug/control
+
+          // disable all 12 messages in the function svc_process()
+          nullarbor:~ # echo -n 'func svc_process -p' >
+                          <debugfs>/dynamic_debug/control
+
+        See Documentation/admin-guide/dynamic-debug-howto.rst for additional
+        information.
+```
 
 # References
 * [Debugging by printing](https://elinux.org/Debugging_by_printing)

kernel/page_cache.md

@@ -61,22 +61,22 @@
 * include/linux/fs.h
 ```c
 struct address_space {
-    struct inode        *host;      /* owner: inode, block_device */
+    struct inode        *host;          /* owner: inode, block_device */
     struct radix_tree_root  page_tree;  /* radix tree of all pages */
-    spinlock_t      tree_lock;  /* and lock protecting it */
-    atomic_t        i_mmap_writable;/* count VM_SHARED mappings */
-    struct rb_root      i_mmap;     /* tree of private and shared mappings */
+    spinlock_t          tree_lock;      /* and lock protecting it */
+    atomic_t            i_mmap_writable;/* count VM_SHARED mappings */
+    struct rb_root      i_mmap;         /* tree of private and shared mappings */
     struct rw_semaphore i_mmap_rwsem;   /* protect tree, count, list */
     /* Protected by tree_lock together with the radix tree */
-    unsigned long       nrpages;    /* number of total pages */
+    unsigned long       nrpages;        /* number of total pages */
     /* number of shadow or DAX exceptional entries */
     unsigned long       nrexceptional;
-    pgoff_t         writeback_index;/* writeback starts here */
+    pgoff_t             writeback_index;/* writeback starts here */
     const struct address_space_operations *a_ops;   /* methods */
-    unsigned long       flags;      /* error bits/gfp mask */
-    spinlock_t      private_lock;   /* for use by the address_space */
+    unsigned long       flags;          /* error bits/gfp mask */
+    spinlock_t          private_lock;   /* for use by the address_space */
     struct list_head    private_list;   /* ditto */
-    void            *private_data;  /* ditto */
+    void                *private_data;  /* ditto */
 } __attribute__((aligned(sizeof(long))));
 ```
 * `i_mmap` 一个优先搜索树，它的搜索范围包含了在`address_space`中所有共享的 *私有的* 映射页面。
@@ -150,11 +150,83 @@ struct address_space_operations {
 
 ![page_cache_file_mm_mapping.png](pic/page_cache_file_mm_mapping.png)
 
-## Radix Tree
-
-* 因为任何页I/O操作前内核都要检查page是否已在page cache中，内核通过 **radix tree** 数据结构来快速检索希望得到的page。
-* 每个`struct address_space`对象的都有唯一的radix tree。
-  * radix tree即`struct radix_tree_root  page_tree`域。
+## Radix Tree / Xarray
+* 因为任何页 I/O 操作前内核都要检查 page 是否已在 page cache 中，内核通过 **radix tree** 数据结构来快速检索希望得到的 page。
+* 每个`struct address_space`对象的都有唯一的 radix tree。
+  * radix tree 即`struct radix_tree_root  page_tree`域。
+* 在 v4.20 后，radix tree 已被改为 xarray。
+  * include/linux/xarray.h
+  ```c
+  /*
+   * @count is the count of every non-NULL element in the ->slots array
+   * whether that is a value entry, a retry entry, a user pointer,
+   * a sibling entry or a pointer to the next level of the tree.
+   * @nr_values is the count of every element in ->slots which is
+   * either a value entry or a sibling of a value entry.
+   */
+  struct xa_node {
+      unsigned char   shift;      /* Bits remaining in each slot */
+      unsigned char   offset;     /* Slot offset in parent */
+      unsigned char   count;      /* Total entry count */
+      unsigned char   nr_values;  /* Value entry count */
+      struct xa_node __rcu *parent;   /* NULL at top of tree */
+      struct xarray   *array;     /* The array we belong to */
+      union {
+          struct list_head private_list;  /* For tree user */
+          struct rcu_head rcu_head;   /* Used when freeing node */
+      };
+      void __rcu  *slots[XA_CHUNK_SIZE];
+      union {
+          unsigned long   tags[XA_MAX_MARKS][XA_MARK_LONGS];
+          unsigned long   marks[XA_MAX_MARKS][XA_MARK_LONGS];
+      };
+  };
+  ```
+  * `shift` 当前节点的单位
+* slot 里存储的指针的最后两位指示该 node 的类型
+  * `00`：node 为数据指针（叶子节点）
+  * `10`：node 指向下一级 node，老版本为`01`
+  * 老版本为`10`：node 为 exceptional entry，用于存储 shmem/tmpfs 的 swap entries
+  * include/linux/radix-tree.h
+  ```c
+  /*
+   * The bottom two bits of the slot determine how the remaining bits in the
+   * slot are interpreted:
+   *
+   * 00 - data pointer
+   * 10 - internal entry
+   * x1 - value entry
+   *
+   * The internal entry may be a pointer to the next level in the tree, a
+   * sibling entry, or an indicator that the entry in this slot has been moved
+   * to another location in the tree and the lookup should be restarted.  While
+   * NULL fits the 'data pointer' pattern, it means that there is no entry in
+   * the tree for this index (no matter what level of the tree it is found at).
+   * This means that storing a NULL entry in the tree is the same as deleting
+   * the entry from the tree.
+   */
+  #define RADIX_TREE_ENTRY_MASK       3UL
+  #define RADIX_TREE_INTERNAL_NODE    2UL
+
+  static inline bool radix_tree_is_internal_node(void *ptr)
+  {
+      return ((unsigned long)ptr & RADIX_TREE_ENTRY_MASK) ==
+                  RADIX_TREE_INTERNAL_NODE;
+  }
+  ```
+* 这样在使用 node 前需要用函数`entry_to_node()`转换一下，存储 entry 前需要做相反的操作
+  * lib/radix-tree.c
+  ```c
+  static inline struct radix_tree_node *entry_to_node(void *ptr)
+  {
+      return (void *)((unsigned long)ptr & ~RADIX_TREE_INTERNAL_NODE);
+  }
+
+  static inline void *node_to_entry(void *ptr)
+  {
+      return (void *)((unsigned long)ptr | RADIX_TREE_INTERNAL_NODE);
+  }
+  ```
 
 # The Buffer Cache
 * 独立的磁盘块通过 block I/O buffer 也要被存入 page cache 中。
@@ -223,3 +295,5 @@ laptop_mode | A Boolean value controlling laptop mode. See the following section
 * [Linux Storage Cache](https://msreekan.com/2015/04/24/linux-storage-cache/)
 * [How Linux Kernel Manages Application Memory](http://techblog.cloudperf.net/2016/07/how-linux-kernel-manages-application_18.html)
 * [FREE命令显示的BUFFERS与CACHED的区别](http://linuxperf.com/?p=32)
+* [The XArray data structure - LWN.net](https://lwn.net/Articles/745073/)
+* [详解Linux内核Radix树算法的实现](http://sourcelink.top/2019/09/26/linux-kernel-radix-tree-analysis/)

kernel/profiling/perf.md

@@ -455,7 +455,7 @@ SYSCALL_DEFINE5(perf_event_open,
 * 运行时间越 **多** 的函数，被中断击中的机会越 **大**，从而推测，那个函数（或者`pid`等）的CPU占用率就越 **高**
 * 如果某个进程/函数运气特别好，它每次都刚好躲过你发起探测的位置，则统计结果可能就完全是错的。这是所有采样统计都有可能遇到的问题。
 * 如何可以获得更精确的采样结果
-	* 延长采样的事件以获得更多的样本
+	* 延长采样的时间以获得更多的样本
 	* 提高采样的频率，然而对系统而言也会有额外的开销
 
 ### 基于时间的性能分析受 CPU idle 的限制

kernel/trace/ftrace-design.md

@@ -1244,6 +1244,66 @@ static __init int init_function_trace(void)
 core_initcall(init_function_trace);
 ```
 
+## 栈跟踪
+* 核心函数是`__ftrace_trace_stack()`，`__trace_stack()`对其有一个简单的封装
+
+### func_stack_trace (function tracer)
+```c
+kernel/trace/trace_functions.c
+function_stack_trace_call()
+   kernel/trace/trace.c
+-> trace_function()
+-> __trace_stack()
+   -> __ftrace_trace_stack()
+      kernel/stacktrace.c
+      -> stack_trace_save()
+        stack_trace_consume_fn consume_entry = stack_trace_consume_entry;
+        arch/x86/kernel/stacktrace.c
+        -> arch_stack_walk(consume_entry, &c, current, NULL)
+           -> unwind_start()
+           -> unwind_done()
+           -> unwind_next_frame()
+           -> unwind_get_return_address()
+           -> consume_entry() => stack_trace_consume_entry()
+```
+### stacktrace (tracepoint)
+```c
+kernel/trace/trace_events_trigger.c
+stacktrace_trigger()
+   kernel/trace/trace.c
+-> trace_dump_stack(STACK_SKIP)
+   -> __ftrace_trace_stack()
+```
+### function-trace (latency tracer)
+```c
+kernel/trace/trace_irqsoff.c
+tracer_hardirqs_on()/tracer_preempt_on()
+-> stop_critical_timings()
+   -> check_critical_timing()
+      -> __trace_function()
+      -> __trace_stack()
+```
+
+### ftrace_stacks[4] 和 ftrace_stack[1024] 数组
+```c
+/* Allow 4 levels of nesting: normal, softirq, irq, NMI */
+#define FTRACE_KSTACK_NESTING   4
+
+#define FTRACE_KSTACK_ENTRIES   (PAGE_SIZE / FTRACE_KSTACK_NESTING)
+/*每一级 1024 个条目。对于 64 位系统，占 8192 Byte （2 pages）*/
+struct ftrace_stack {
+    unsigned long       calls[FTRACE_KSTACK_ENTRIES];
+};
+/*每个条目存一个调用函数的地址，故而用的类型是 unsigned long*/
+/*允许 4 级嵌套：normal, softirq, irq, NMI，2*4=8个pages*/
+struct ftrace_stacks {
+    struct ftrace_stack stacks[FTRACE_KSTACK_NESTING];
+};
+/*Per-CPU 变量 ftrace_stacks 占 CPU x 8 个 pages*/
+static DEFINE_PER_CPU(struct ftrace_stacks, ftrace_stacks);
+static DEFINE_PER_CPU(int, ftrace_stack_reserve);
+```
+
 # Reference
 
 * [ftrace 简介](https://www.ibm.com/developerworks/cn/linux/l-cn-ftrace/)