fix(perf/UX): Use num physical cores by default, warn about E/P cores

examples/common.cpp

@@ -10,6 +10,7 @@
 #if defined (_WIN32)
 #include <fcntl.h>
 #include <io.h>
+#include <windows.h>
 #pragma comment(lib,"kernel32.lib")
 extern "C" __declspec(dllimport) void* __stdcall GetStdHandle(unsigned long nStdHandle);
 extern "C" __declspec(dllimport) int __stdcall GetConsoleMode(void* hConsoleHandle, unsigned long* lpMode);
@@ -23,17 +24,41 @@ extern "C" __declspec(dllimport) int __stdcall WideCharToMultiByte(unsigned int
 #define CP_UTF8 65001
 #endif
 
-bool gpt_params_parse(int argc, char ** argv, gpt_params & params) {
-    // determine sensible default number of threads.
-    // std::thread::hardware_concurrency may not be equal to the number of cores, or may return 0.
+int32_t get_num_physical_cores() {
 #ifdef __linux__
     std::ifstream cpuinfo("/proc/cpuinfo");
-    params.n_threads = std::count(std::istream_iterator<std::string>(cpuinfo),
-                                  std::istream_iterator<std::string>(),
-                                  std::string("processor"));
+    std::string line;
+    while (std::getline(cpuinfo, line)) {
+        if (line.find("cpu cores") != std::string::npos) {
+            line.erase(0, line.find(": ") + 2);
+            try {
+                return (int32_t) std::stoul(line);
+            } catch (std::invalid_argument& e) {} // Ignore if we could not parse
+        }
+    }
+#elif defined(__APPLE__) && defined(__MACH__)
+    int num_physical_cores;
+    size_t len = sizeof(num_physical_cores);
+    int result = sysctlbyname("hw.perflevel0.physicalcpu", &num_physical_cores, &len, NULL, 0);
+    if (result == 0) {
+        return (int32_t) num_physical_cores;
+    }
+    result = sysctlbyname("hw.physicalcpu", &num_physical_cores, &len, NULL, 0);
+    if (result == 0) {
+        return (int32_t) num_physical_cores;
+    }
+#elif defined(_WIN32)
+    SYSTEM_INFO sysinfo;
+    GetNativeSystemInfo(&sysinfo);
+    return (in32_t) sysinfo.dwNumberOfProcessors;
 #endif
-    if (params.n_threads == 0) {
-        params.n_threads = std::max(1, (int32_t) std::thread::hardware_concurrency());
+    return -1;
+}
+
+bool gpt_params_parse(int argc, char ** argv, gpt_params & params) {
+    // Clip if not a valid number of threads
+    if (params.n_threads <= 0) {
+        params.n_threads = std::max(1, std::min(8, (int32_t) std::thread::hardware_concurrency()));
     }
 
     bool invalid_param = false;

examples/common.h

@@ -13,9 +13,11 @@
 // CLI argument parsing
 //
 
+int32_t get_num_physical_cores();
+
 struct gpt_params {
     int32_t seed          = -1;   // RNG seed
-    int32_t n_threads     = std::min(4, (int32_t) std::thread::hardware_concurrency());
+    int32_t n_threads     = get_num_physical_cores(); // (if <= 0, = clip(num_logical_cores, 1, 8))
     int32_t n_predict     = 128;  // new tokens to predict
     int32_t repeat_last_n = 64;   // last n tokens to penalize
     int32_t n_parts       = -1;   // amount of model parts (-1 = determine from model dimensions)