From 3d4dc42fe99572d0b33f77149cab348cbbb8d887 Mon Sep 17 00:00:00 2001
From: Lars Viklund <zao@zao.se>
Date: Wed, 10 Nov 2021 01:58:15 +0100
Subject: [PATCH 01/16] feat: add OpenFile function to open UTF-8 paths

On Windows this function opens a placeholder file with the standard
io:open function to get a proper file object, then replacing the
internal C file pointer with the actual file opened through _wfopen.
---
 engine/common.h          |  11 ++++
 engine/common/common.cpp |  58 ++++++++++++++++++
 ui_api.cpp               | 124 +++++++++++++++++++++++++++++++++++++++
 ui_main.h                |   1 +
 4 files changed, 194 insertions(+)

diff --git a/engine/common.h b/engine/common.h
index 3c1c292..3eb720e 100644
--- a/engine/common.h
+++ b/engine/common.h
@@ -485,6 +485,17 @@ char*	_AllocStringLen(size_t len, const char* file, int line);
 void	FreeString(const char* str);
 dword	StringHash(const char* str, int mask);
 
+#ifdef _WIN32
+wchar_t* WidenANSIString(const char* str);
+wchar_t* WidenOEMString(const char* str);
+wchar_t* WidenUTF8String(const char* str);
+void FreeWideString(wchar_t* str);
+
+char* NarrowANSIString(const wchar_t* str);
+char* NarrowOEMString(const wchar_t* str);
+char* NarrowUTF8String(const wchar_t* str);
+#endif
+
 #ifndef _WIN32
 #define _stricmp strcasecmp
 #define _strnicmp strncasecmp
diff --git a/engine/common/common.cpp b/engine/common/common.cpp
index 099d497..3147076 100644
--- a/engine/common/common.cpp
+++ b/engine/common/common.cpp
@@ -279,3 +279,61 @@ dword StringHash(const char* str, int mask)
 	}
 	return hash & mask;
 }
+
+#ifdef _WIN32
+#include <Windows.h>
+
+static wchar_t* WidenCodepageString(const char* str, UINT codepage)
+{
+	int cch = MultiByteToWideChar(codepage, 0, str, -1, nullptr, 0);
+	wchar_t* wstr = new wchar_t[cch];
+	MultiByteToWideChar(codepage, 0, str, -1, wstr, cch);
+	return wstr;
+}
+
+wchar_t* WidenANSIString(const char* str)
+{
+	return WidenCodepageString(str, CP_ACP);
+}
+
+wchar_t* WidenOEMString(const char* str)
+{
+	return WidenCodepageString(str, CP_OEMCP);
+}
+
+wchar_t* WidenUTF8String(const char* str)
+{
+	return WidenCodepageString(str, CP_UTF8);
+}
+
+char* NarrowCodepageString(const wchar_t* str, UINT codepage)
+{
+	int cb = WideCharToMultiByte(codepage, 0, str, -1, nullptr, 0, nullptr, nullptr);
+	char* nstr = new char[cb];
+	WideCharToMultiByte(codepage, 0, str, -1, nstr, cb, nullptr, nullptr);
+	return nstr;
+}
+
+void FreeWideString(wchar_t* str)
+{
+	if (str) {
+		delete[] str;
+	}
+}
+
+char* NarrowANSIString(const wchar_t* str)
+{
+	return NarrowCodepageString(str, CP_ACP);
+}
+
+char* NarrowOEMString(const wchar_t* str)
+{
+	return NarrowCodepageString(str, CP_OEMCP);
+}
+
+char* NarrowUTF8String(const wchar_t* str)
+{
+	return NarrowCodepageString(str, CP_UTF8);
+}
+
+#endif
diff --git a/ui_api.cpp b/ui_api.cpp
index 7ccd7e4..2e1802e 100644
--- a/ui_api.cpp
+++ b/ui_api.cpp
@@ -1133,6 +1133,116 @@ static int l_GetAsyncCount(lua_State* L)
 	return 1;
 }
 
+// ============
+// File Handles
+// ============
+
+#ifdef _WIN32
+#include <Windows.h>
+
+static void stackDump(lua_State* L)
+{
+	char buf[4096]{};
+	char* p = buf;
+	int i;
+	int top = lua_gettop(L);
+	for (i = 1; i <= top; i++) {  /* repeat for each level */
+		int t = lua_type(L, i);
+		switch (t) {
+
+		case LUA_TSTRING:  /* strings */
+			p += sprintf(p, "`%s'", lua_tostring(L, i));
+			break;
+
+		case LUA_TBOOLEAN:  /* booleans */
+			p += sprintf(p, lua_toboolean(L, i) ? "true" : "false");
+			break;
+
+		case LUA_TNUMBER:  /* numbers */
+			p += sprintf(p, "%g", lua_tonumber(L, i));
+			break;
+
+		default:  /* other values */
+			p += sprintf(p, "%s", lua_typename(L, t));
+			break;
+
+		}
+		p += sprintf(p, "  ");  /* put a separator */
+	}
+	p += sprintf(p, "\n");  /* end the listing */
+	OutputDebugStringA(buf);
+}
+#endif
+
+/* io:open replacement that opens a file with UTF-8 paths instead of the user codepage.
+*  In order to reduce implementation effort this function opens a well-known readable file
+*  with regular io:open and swaps out the internal FILE* pointer to a file opened via the
+*  Unicode-aware _wfopen on Windows.
+* 
+*  The resulting file object has all the functionality of the standard library file
+*  object due to actually being such an object.
+*/
+static int l_OpenFile(lua_State* L)
+{
+	ui_main_c* ui = GetUIPtr(L);
+	
+	int n = lua_gettop(L);
+	ui->LAssert(L, n == 2, "Usage: OpenFile(path, mode)");
+	ui->LAssert(L, lua_isstring(L, 1), "OpenFile() argument 1: expected string, got %s", luaL_typename(L, 1));
+	ui->LAssert(L, lua_isstring(L, 2), "OpenFile() argument 2: expected string, got %s", luaL_typename(L, 2));
+
+#ifndef _WIN32
+	lua_rawgeti(L, LUA_REGISTRYINDEX, ui->ioOpenf);
+	lua_pushstring(L, lua_tostring(L, 1));
+	lua_pushstring(L, lua_tostring(L, 2));
+	lua_call(L, 2, 2);
+	return lua_gettop(L) - 2;
+#else
+	wchar_t* widePath = WidenUTF8String(lua_tostring(L, 1));
+	wchar_t* wideMode = WidenUTF8String(lua_tostring(L, 2));
+	FILE* fp = _wfopen(widePath, wideMode);
+	FreeWideString(widePath);
+	FreeWideString(wideMode);
+	if (!fp) {
+		return 0;
+	}
+
+	static const std::string placeholder = [] {
+		char buf[MAX_PATH]{};
+		HMODULE mod = LoadLibraryA("ntdll.dll");
+		GetModuleFileNameA(mod, buf, sizeof(buf));
+		FreeLibrary(mod);
+		return std::string(buf);
+	}();
+
+	{
+		lua_rawgeti(L, LUA_REGISTRYINDEX, ui->ioOpenf);
+		lua_pushstring(L, placeholder.c_str());
+		lua_pushstring(L, "r");
+		lua_call(L, 2, 2);
+
+
+		if (lua_isnil(L, -2)) {
+			fclose(fp);
+			ui->LAssert(L, !lua_isnil(L, -2), "OpenFile(): failed to open placeholder path %s: %s", placeholder.c_str(), luaL_checkstring(L, -1));
+		}
+	}
+
+	lua_pop(L, 1);
+
+	struct luajitInternalFileHandlePart {
+		FILE* f;
+	};
+
+	luajitInternalFileHandlePart* ljData = (luajitInternalFileHandlePart*)luaL_checkudata(L, -1, "FILE*");
+
+	fclose(ljData->f);
+	ljData->f = fp;
+
+	return 1;
+#endif
+}
+
 // ==============
 // Search Handles
 // ==============
@@ -1913,6 +2023,17 @@ int ui_main_c::InitAPI(lua_State* L)
 	sol::state_view lua(L);
 	luaL_openlibs(L);
 
+	{
+		ui_main_c* ui = GetUIPtr(L);
+		lua_getglobal(L, "io");
+		if (!lua_isnil(L, -1)) {
+			lua_getfield(L, -1, "open");
+			ui->ioOpenf = luaL_ref(L, LUA_REGISTRYINDEX);
+		}
+
+		lua_pop(L, 1);
+	}
+
 	// Add "lua/" subdir for non-JIT Lua
 	{
 		lua_getglobal(L, "package");
@@ -2020,6 +2141,9 @@ int ui_main_c::InitAPI(lua_State* L)
 	ADDFUNC(GetAsyncCount);
 	ADDFUNC(RenderInit);
 
+	// Wide file I/O
+	ADDFUNC(OpenFile);
+
 	// Search handles
 	lua_newtable(L);	// Search handle metatable
 	lua_pushvalue(L, -1);	// Push search handle metatable
diff --git a/ui_main.h b/ui_main.h
index 23d607a..ce8d36e 100644
--- a/ui_main.h
+++ b/ui_main.h
@@ -49,6 +49,7 @@ class ui_main_c: public ui_IMain {
 	int		cursorY = 0;
 	int		framesSinceWindowHidden = 0;
 	volatile bool	inLua = false;
+	int		ioOpenf = LUA_NOREF;
 
 	static int InitAPI(lua_State* L);
 

From f6e82a30673f8d5a4be5d17eea4809de8465c8df Mon Sep 17 00:00:00 2001
From: Lars Viklund <zao@zao.se>
Date: Sat, 23 Mar 2024 20:11:59 +0100
Subject: [PATCH 02/16] feat: yield UTF-8 paths from file searches

Previously the file search objects exposed to Lua accepted and generated
paths encoded in the user's narrow codepage. As a move toward UTF-8
awareness for paths and strings on the Lua side, we now instead accept
and generate UTF-8 strings.

This should be robust against legacy paths represented in the user's
codepage, the only source of such paths should be if they're
historically persisted in locations like `Settings.xml`.
---
 engine/system/win/sys_main.cpp | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/engine/system/win/sys_main.cpp b/engine/system/win/sys_main.cpp
index db43577..fa1d0c9 100644
--- a/engine/system/win/sys_main.cpp
+++ b/engine/system/win/sys_main.cpp
@@ -139,7 +139,7 @@ find_c::~find_c()
 bool GlobMatch(const std::filesystem::path& glob, const std::filesystem::path& file)
 {
 	using namespace std::literals::string_view_literals;
-	auto globStr = glob.generic_string();
+	auto globStr = glob.generic_u8string();
 
 	// Deal with traditional "everything" wildcards.
 	if (glob == "*" || glob == "*.*") {
@@ -160,7 +160,7 @@ bool GlobMatch(const std::filesystem::path& glob, const std::filesystem::path& f
 	else {
 		// Otherwise build a regular expression from the glob and use that to match files.
 		auto it = fmt::appender(buf);
-		for (char ch : glob.generic_string()) {
+		for (char ch : globStr) {
 			if (ch == '*') {
 				it = fmt::format_to(it, ".*");
 			}
@@ -185,19 +185,26 @@ bool GlobMatch(const std::filesystem::path& glob, const std::filesystem::path& f
 	reOpts.set_case_sensitive(false);
 	RE2 reGlob{to_string(buf), reOpts};
 
-	return RE2::FullMatch(file.generic_string(), reGlob);
+	auto fileStr = file.generic_u8string();
+	return RE2::FullMatch(fileStr, reGlob);
 }
 
 bool find_c::FindFirst(const char* fileSpec)
 {
+#ifdef _WIN32
+	wchar_t* wideSpec = WidenUTF8String(fileSpec);
+	std::filesystem::path p(wideSpec);
+	FreeWideString(wideSpec);
+#else
 	std::filesystem::path p(fileSpec);
+#endif
 	glob = p.filename();
 
 	std::error_code ec;
 	for (iter = std::filesystem::directory_iterator(p.parent_path(), ec); iter != std::filesystem::directory_iterator{}; ++iter) {
 		auto candFilename = iter->path().filename();
 		if (GlobMatch(glob, candFilename)) {
-			fileName = candFilename.string();
+			fileName = candFilename.u8string();
 			isDirectory = iter->is_directory();
 			fileSize = iter->file_size();
 			auto mod = iter->last_write_time();
@@ -217,7 +224,7 @@ bool find_c::FindNext()
 	for (++iter; iter != std::filesystem::directory_iterator{}; ++iter) {
 		auto candFilename = iter->path().filename();
 		if (GlobMatch(glob, candFilename)) {
-			fileName = candFilename.string();
+			fileName = candFilename.u8string();
 			isDirectory = iter->is_directory();
 			fileSize = iter->file_size();
 			auto mod = iter->last_write_time();

From c55f5b7a79ad5ea3a67af3f9234515bcbbe4215a Mon Sep 17 00:00:00 2001
From: Lars Viklund <zao@zao.se>
Date: Sat, 23 Mar 2024 20:17:05 +0100
Subject: [PATCH 03/16] feat: monkey-patch `io.open` to take UTF-8 paths

By squirreling away the original `io.open` function from Lua and
replacing the entry in the interpreter with our own wrapper that
interprets the path as UTF-8, we can now open files in locations that
cannot be represented in the user's codepage, as long as the path is
supplied as UTF-8.

Lua itself is very encoding agnostic, assuming that all strings are
soups of bytes and that paths are encoded in whatever the platform's
`FILE*` accepts. On Linux and macOS that's fairly friendly and in the
past dependent on the locale, but these days is de-facto standardized
on UTF-8.

On Windows the story is worse and narrow paths are assumed to be in the
user's current codepage, which can only express a fixed subset of
Unicode and which breaks if paths from the filesystem cannot be encoded
in the codepage. On Windows with NTFS the underlying filesystem is
UTF-16LE and supports all of Unicode, so it's good for consistency and
future work if we standardize on UTF-8, as Lua is very byte-oriented
when it comes to text and paths.
---
 ui_api.cpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/ui_api.cpp b/ui_api.cpp
index 2e1802e..a9ca321 100644
--- a/ui_api.cpp
+++ b/ui_api.cpp
@@ -2029,6 +2029,8 @@ int ui_main_c::InitAPI(lua_State* L)
 		if (!lua_isnil(L, -1)) {
 			lua_getfield(L, -1, "open");
 			ui->ioOpenf = luaL_ref(L, LUA_REGISTRYINDEX);
+			lua_pushcfunction(L, l_OpenFile);
+			lua_setfield(L, -2, "open");
 		}
 
 		lua_pop(L, 1);

From d28a5777527ff6cc8e7ba904df0b12280f155c20 Mon Sep 17 00:00:00 2001
From: Lars Viklund <zao@zao.se>
Date: Mon, 1 Apr 2024 01:25:10 +0200
Subject: [PATCH 04/16] feat: add luautf8 for Unicode string operations

The luautf8 library provides analogues of string functions that operate
on UTF-8 strings instead of byte soups, allowing for intelligent editing
and iterating through such strings, needed for things like Unicode paths
and other internationalization.
---
 .gitmodules    |  3 +++
 CMakeLists.txt | 20 ++++++++++++++++++++
 libs/luautf8   |  1 +
 3 files changed, 24 insertions(+)
 create mode 160000 libs/luautf8

diff --git a/.gitmodules b/.gitmodules
index b8b7c9a..0c90bb4 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -10,3 +10,6 @@
 [submodule "dep/glm"]
 	path = dep/glm
 	url = https://github.com/g-truc/glm.git
+[submodule "libs/luautf8"]
+	path = libs/luautf8
+	url = https://github.com/starwing/luautf8.git
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 959f2cf..f89a281 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -271,6 +271,26 @@ target_link_libraries(lcurl
 install(TARGETS lcurl RUNTIME DESTINATION ".")
 install(FILES $<TARGET_RUNTIME_DLLS:lcurl> DESTINATION ".")
 
+# luautf8 module
+
+add_library(lua-utf8 SHARED libs/luautf8/lutf8lib.c)
+
+target_compile_definitions(lua-utf8
+    PRIVATE
+    LUA_BUILD_AS_DLL
+)
+
+target_include_directories(lua-utf8
+    PRIVATE
+)
+
+target_link_libraries(lua-utf8
+    PRIVATE
+    LuaJIT::LuaJIT
+)
+
+install(TARGETS lua-utf8 RUNTIME DESTINATION ".")
+install(FILES $<TARGET_RUNTIME_DLLS:lua-utf8> DESTINATION ".")
 
 # lzip module
 
diff --git a/libs/luautf8 b/libs/luautf8
new file mode 160000
index 0000000..bdd3d7f
--- /dev/null
+++ b/libs/luautf8
@@ -0,0 +1 @@
+Subproject commit bdd3d7fb6ef22334fde028ba792d3a16309a4de8

From 6d669e6c24568ed685c8356f69a1c1ccc85f28e5 Mon Sep 17 00:00:00 2001
From: Lars Viklund <zao@zao.se>
Date: Tue, 2 Apr 2024 22:13:52 +0200
Subject: [PATCH 05/16] feat: expose and expect UTF-8 paths for Lua

Store the user path and base path as canonicalized std::filesystem paths
and expect the Lua side to be aware enough of UTF-8 to manipulate them
properly.

This together with the monkeypatched UTF-8 `io.open` and functions like
MakeDir and RemoveDir allows for use of a wider variety of installation
and user profile locations.
---
 engine/system/sys_main.h       |  5 ++-
 engine/system/win/sys_main.cpp | 61 +++++++++++++---------------------
 ui_api.cpp                     | 20 +++++------
 ui_main.cpp                    |  4 +--
 4 files changed, 35 insertions(+), 55 deletions(-)

diff --git a/engine/system/sys_main.h b/engine/system/sys_main.h
index 4d830b8..7e3b94b 100644
--- a/engine/system/sys_main.h
+++ b/engine/system/sys_main.h
@@ -64,9 +64,8 @@ class sys_IMain {
 	bool		debug = false;
 	bool		debuggerRunning = false;
 	int			processorCount = 0;
-	std::string basePath;
-	std::optional<std::string> userPath;
-	std::optional<std::string> invalidUserPath;
+	std::filesystem::path basePath;
+	std::optional<std::filesystem::path> userPath;
 
 	virtual int		GetTime() = 0;
 	virtual void	Sleep(int msec) = 0;
diff --git a/engine/system/win/sys_main.cpp b/engine/system/win/sys_main.cpp
index fa1d0c9..1d80da7 100644
--- a/engine/system/win/sys_main.cpp
+++ b/engine/system/win/sys_main.cpp
@@ -382,10 +382,19 @@ char* sys_main_c::ClipboardPaste()
 
 bool sys_main_c::SetWorkDir(const char* newCwd)
 {
+#ifdef _WIN32
+	auto changeDir = [](std::filesystem::path const& p) {
+		return _wchdir(p.c_str());
+	};
+#else
+	auto changeDir = [](std::filesystem::path const& p) {
+		return _chdir(p.c_str());
+	};
+#endif
 	if (newCwd) {
-		return _chdir(newCwd) != 0;
+		return changeDir(newCwd) != 0;
 	} else {
-		return _chdir(basePath.c_str()) != 0;
+		return changeDir(basePath.c_str()) != 0;
 	}
 }
 
@@ -501,31 +510,31 @@ void sys_main_c::Restart()
 	exitFlag = true;
 }
 
-std::string FindBasePath()
+std::filesystem::path FindBasePath()
 {
+	std::filesystem::path progPath;
 #ifdef _WIN32
-	std::vector<char> basePath(512);
-	GetModuleFileNameA(NULL, basePath.data(), basePath.size());
-	*strrchr(basePath.data(), '\\') = 0;
-	return basePath.data();
+	std::vector<wchar_t> basePath(1u << 16);
+	GetModuleFileNameW(NULL, basePath.data(), basePath.size());
+	progPath = basePath.data();
 #elif __linux__
 	char basePath[PATH_MAX];
     ssize_t len = ::readlink("/proc/self/exe", basePath, sizeof(basePath));
     if (len == -1 || len == sizeof(basePath))
     	len = 0;
 	basePath[len] = '\0';
-	*strrchr(basePath, '/') = 0;
-	return basePath;
+	progPath = basePath;
 #elif __APPLE__ && __MACH__
 	pid_t pid = getpid();
 	char basePath[PROC_PIDPATHINFO_MAXSIZE]{};
 	proc_pidpath(pid, basePath, sizeof(basePath));
-	*strrchr(basePath, '/') = 0;
-	return basePath;
+	progPath = basePath;
 #endif
+	progPath = canonical(progPath);
+	return progPath.parent_path();
 }
 
-std::optional<std::string> FindUserPath(std::optional<std::string>& invalidPath)
+std::optional<std::filesystem::path> FindUserPath()
 {
 #ifdef _WIN32
     PWSTR osPath{};
@@ -533,36 +542,12 @@ std::optional<std::string> FindUserPath(std::optional<std::string>& invalidPath)
 	if (FAILED(hr)) {
 		// The path may be inaccessible due to malfunctioning cloud providers.
 		CoTaskMemFree(osPath);
-		invalidPath.reset();
 		return {};
 	}
 	std::wstring pathStr = osPath;
 	CoTaskMemFree(osPath);
 	std::filesystem::path path(pathStr);
-	try {
-    	return path.string();
-	}
-	catch (std::system_error) {
-		// The path could not be converted into the narrow representation, convert the path
-		// string lossily for use in an ASCII error message on the Lua side.
-		invalidPath.reset();
-		std::wstring pathStr = path.wstring();
-		char defGlyph = '?';
-		BOOL defGlyphUsed{};
-		DWORD convFlags = WC_COMPOSITECHECK | WC_NO_BEST_FIT_CHARS | WC_DEFAULTCHAR;
-		int cb = WideCharToMultiByte(CP_ACP, 0, pathStr.c_str(), -1, nullptr, 0, &defGlyph, &defGlyphUsed);
-		if (cb) {
-			std::vector<char> buf(cb);
-			WideCharToMultiByte(CP_ACP, 0, pathStr.c_str(), -1, buf.data(), cb, &defGlyph, &defGlyphUsed);
-			for (auto& ch : buf) {
-				if ((unsigned char)ch >= 0x80) {
-					ch = '?'; // Substitute characters that we can represent but can't draw.
-				}
-			}
-			invalidPath = buf.data();
-		}
-		return {};
-	}
+	return canonical(path);
 #else
     if (char const* data_home_path = getenv("XDG_DATA_HOME")) {
         return data_home_path;
@@ -598,7 +583,7 @@ sys_main_c::sys_main_c()
 
 	// Set the local system information
 	basePath = FindBasePath();
-	userPath = FindUserPath(invalidUserPath);
+	userPath = FindUserPath();
 }
 
 bool sys_main_c::Run(int argc, char** argv)
diff --git a/ui_api.cpp b/ui_api.cpp
index a9ca321..44b9adf 100644
--- a/ui_api.cpp
+++ b/ui_api.cpp
@@ -87,7 +87,7 @@
 ** msec = GetTime()
 ** path = GetScriptPath()
 ** path = GetRuntimePath()
-** path, missingPath = GetUserPath() -- may fail, then returns nil and and approximation of the bad path
+** path = GetUserPath() -- may return nil if the user path could not be determined
 ** SetWorkDir("<path>")
 ** path = GetWorkDir()
 ** ssID = LaunchSubScript("<scriptText>", "<funcList>", "<subList>"[, ...])
@@ -1618,7 +1618,7 @@ static int l_GetScriptPath(lua_State* L)
 static int l_GetRuntimePath(lua_State* L)
 {
 	ui_main_c* ui = GetUIPtr(L);
-	lua_pushstring(L, ui->sys->basePath.c_str());
+	lua_pushstring(L, ui->sys->basePath.u8string().c_str());
 	return 1;
 }
 
@@ -1627,16 +1627,10 @@ static int l_GetUserPath(lua_State* L)
 	ui_main_c* ui = GetUIPtr(L);
 	auto& userPath = ui->sys->userPath;
 	if (userPath) {
-		lua_pushstring(L, userPath->c_str());
+		lua_pushstring(L, userPath->u8string().c_str());
 		return 1;
 	}
-	
-	lua_pushnil(L);
-	if (auto& invalidUserPath = ui->sys->invalidUserPath) {
-		lua_pushstring(L, invalidUserPath->c_str());
-		return 2;
-	}
-	return 1;
+	return 0;
 }
 
 static int l_MakeDir(lua_State* L)
@@ -1645,7 +1639,8 @@ static int l_MakeDir(lua_State* L)
 	int n = lua_gettop(L);
 	ui->LAssert(L, n >= 1, "Usage: MakeDir(path)");
 	ui->LAssert(L, lua_isstring(L, 1), "MakeDir() argument 1: expected string, got %s", luaL_typename(L, 1));
-	std::filesystem::path path(lua_tostring(L, 1));
+	char const* givenPath = lua_tostring(L, 1);
+	auto path = std::filesystem::u8path(givenPath);
 	std::error_code ec;
 	if (!create_directory(path, ec)) {
 		lua_pushnil(L);
@@ -1664,7 +1659,8 @@ static int l_RemoveDir(lua_State* L)
 	int n = lua_gettop(L);
 	ui->LAssert(L, n >= 1, "Usage: l_RemoveDir(path)");
 	ui->LAssert(L, lua_isstring(L, 1), "l_RemoveDir() argument 1: expected string, got %s", luaL_typename(L, 1));
-	std::filesystem::path path(lua_tostring(L, 1));
+	char const* givenPath = lua_tostring(L, 1);
+	auto path = std::filesystem::u8path(givenPath);
 	std::error_code ec;
 	if (!is_directory(path, ec) || ec || !remove(path, ec) || ec) {
 		lua_pushnil(L);
diff --git a/ui_main.cpp b/ui_main.cpp
index 87540c3..79a1f05 100644
--- a/ui_main.cpp
+++ b/ui_main.cpp
@@ -223,8 +223,8 @@ void ui_main_c::Init(int argc, char** argv)
 		scriptPath = AllocString(tmpDir);
 		scriptWorkDir = AllocString(tmpDir);
 	} else {
-		scriptPath = AllocString(sys->basePath.c_str());
-		scriptWorkDir = AllocString(sys->basePath.c_str());
+		scriptPath = AllocString(sys->basePath.u8string().c_str());
+		scriptWorkDir = AllocString(sys->basePath.u8string().c_str());
 	}
 	scriptArgc = argc;
 	scriptArgv = new char*[argc];

From 99d1713fd1f9d917466053669f6193ec605ad60e Mon Sep 17 00:00:00 2001
From: Lars Viklund <zao@zao.se>
Date: Tue, 2 Apr 2024 22:26:11 +0200
Subject: [PATCH 06/16] feat: draw UTF-8 text with sprite font and tofu

As we need the ability to display UTF-8 encoded text onward in Lua code
this change alters text rendering to transcode human strings to UTF-32
codepoints keeping an association between original byte offset and
codepoint index.

This allows for font rendering that can intelligently translate
unhandled codepoints as tofu symbols, for now represented as a bracketed
codepoint ordinal in a smaller font like `[U+0123]`.

This slightly misaligns mouse selection and navigation of text areas if
they're in "scaled" font heights in-between or outside the "crisp" font
sizes available in the runtime. The Lua code predominantly uses crisp
fonts, a survey only found a few scaled/zero/negative heights, all of
empty strings.

The handling of zero-height and negative height strings has also been
made more robust, fixing a division by zero and an array overflow.
---
 engine/common.h          |  17 ++-
 engine/common/common.cpp | 125 ++++++++++++++++-
 engine/render/r_font.cpp | 294 +++++++++++++++++++++++++++------------
 engine/render/r_font.h   |  26 +++-
 engine/render/r_main.cpp |  22 ++-
 5 files changed, 377 insertions(+), 107 deletions(-)

diff --git a/engine/common.h b/engine/common.h
index 3eb720e..d34dfa2 100644
--- a/engine/common.h
+++ b/engine/common.h
@@ -26,6 +26,10 @@
 #include "common/memtrak3.h"
 #endif
 
+#include <string>
+#include <string_view>
+#include <vector>
+
 // =======
 // Classes
 // =======
@@ -475,8 +479,10 @@ T clamp(T &v, T l, T u)
 // Common Functions
 // ================
 
-int		IsColorEscape(const char* str);
-void	ReadColorEscape(const char* str, col3_t out);
+int		IsColorEscape(char const* str);
+int		IsColorEscape(std::u32string_view str);
+void	ReadColorEscape(char const* str, col3_t out);
+std::u32string_view ReadColorEscape(std::u32string_view str, col3_t out);
 
 char*	_AllocString(const char* str, const char* file, int line);
 #define AllocString(s) _AllocString(s, __FILE__, __LINE__)
@@ -485,6 +491,13 @@ char*	_AllocStringLen(size_t len, const char* file, int line);
 void	FreeString(const char* str);
 dword	StringHash(const char* str, int mask);
 
+struct IndexedUTF32String {
+	std::u32string text;
+	std::vector<size_t> sourceCodeUnitOffsets;
+};
+
+IndexedUTF32String IndexUTF8ToUTF32(std::string_view str);
+
 #ifdef _WIN32
 wchar_t* WidenANSIString(const char* str);
 wchar_t* WidenOEMString(const char* str);
diff --git a/engine/common/common.cpp b/engine/common/common.cpp
index 3147076..b26aa7f 100644
--- a/engine/common/common.cpp
+++ b/engine/common/common.cpp
@@ -205,14 +205,45 @@ int IsColorEscape(const char* str)
 	}
 	if (isdigit(str[1])) {
 		return 2;
-	} else if (str[1] == 'x' || str[1] == 'X') {
+	}
+	else if (str[1] == 'x' || str[1] == 'X') {
+		for (int c = 0; c < 6; c++) {
+			if (!isxdigit(str[c + 2])) {
+				return 0;
+			}
+		}
+		return 8;
+	}
+	return 0;
+}
+
+int IsColorEscape(std::u32string_view str)
+{
+	if (str.size() < 2 || str[0] != '^') {
+		return 0;
+	}
+
+	auto discrim = str[1];
+
+	// Check for indexed colour escape like ^7.
+	// Avoid using isdigit as we only accept arabic numerals.
+	if (discrim >= U'0' && discrim <= U'9') {
+		return 2;
+	}
+
+	// Check for direct colour escape like ^x123ABC.
+	if (str.size() >= 8 && (discrim == 'x' || discrim == 'X')) {
 		for (int c = 0; c < 6; c++) {
-			if ( !isxdigit(str[c + 2]) ) {
+			auto ch = str[c + 2];
+			bool const isHexDigit = (ch >= U'0' && ch <= U'9') || (ch >= U'A' && ch <= U'F') || (ch >= U'a' && ch <= U'f');
+			if (!isHexDigit) {
 				return 0;
 			}
 		}
 		return 8;
 	}
+
+	// Fallthrough indicates no recognized colour code.
 	return 0;
 }
 
@@ -223,16 +254,40 @@ void ReadColorEscape(const char* str, col3_t out)
 	case 2:
 		VectorCopy(colorEscape[str[1] - '0'], out);
 		break;
+	case 8:
+	{
+		int xr, xg, xb;
+		sscanf(str + 2, "%2x%2x%2x", &xr, &xg, &xb);
+		out[0] = xr / 255.0f;
+		out[1] = xg / 255.0f;
+		out[2] = xb / 255.0f;
+	}
+	break;
+	}
+}
+
+std::u32string_view ReadColorEscape(std::u32string_view str, col3_t out)
+{
+	int len = IsColorEscape(str);
+	switch (len) {
+	case 2:
+		VectorCopy(colorEscape[str[1] - U'0'], out);
+		break;
 	case 8:
 		{
 			int xr, xg, xb;
-			sscanf(str + 2, "%2x%2x%2x", &xr, &xg, &xb);
+			char buf[7]{};
+			for (size_t i = 0; i < 6; ++i) {
+				buf[i] = (char)str[i + 2];
+			}
+			sscanf(buf, "%2x%2x%2x", &xr, &xg, &xb);
 			out[0] = xr / 255.0f;
 			out[1] = xg / 255.0f;
 			out[2] = xb / 255.0f;
 		}
 		break;
 	}
+	return str.substr(len);
 }
 
 // ================
@@ -336,4 +391,68 @@ char* NarrowUTF8String(const wchar_t* str)
 	return NarrowCodepageString(str, CP_UTF8);
 }
 
+IndexedUTF32String IndexUTF8ToUTF32(std::string_view input)
+{
+	IndexedUTF32String ret{};
+
+	size_t byteCount = input.size();
+	auto& offsets = ret.sourceCodeUnitOffsets;
+	offsets.reserve(byteCount); // conservative reservation
+	std::vector<char32_t> codepoints;
+
+	auto bytes = (uint8_t const*)input.data();
+	for (size_t byteIdx = 0; byteIdx < byteCount;) {
+		uint8_t const* b = bytes + byteIdx;
+		size_t left = byteCount - byteIdx;
+		offsets.push_back(byteIdx);
+
+		char32_t codepoint{};
+		if (*b >> 7 == 0b0) { // 0xxx'xxxx
+			codepoint = *b;
+			byteIdx += 1;
+		}
+		else if (left >= 2 &&
+			b[0] >> 5 == 0b110 &&
+			b[1] >> 6 == 0b10)
+		{
+			auto p0 = (uint32_t)b[0] & 0b1'1111;
+			auto p1 = (uint32_t)b[1] & 0b11'1111;
+			codepoint = p0 << 6 | p1;
+			byteIdx += 2;
+		}
+		else if (left >= 3 &&
+			b[0] >> 4 == 0b1110 &&
+			b[1] >> 6 == 0b10 &&
+			b[2] >> 6 == 0b10)
+		{
+			auto p0 = (uint32_t)b[0] & 0b1111;
+			auto p1 = (uint32_t)b[1] & 0b11'1111;
+			auto p2 = (uint32_t)b[2] & 0b11'1111;
+			codepoint = p0 << 12 | p1 << 6 | p2;
+			byteIdx += 3;
+		}
+		else if (left >= 4 &&
+			b[0] >> 3 == 0b11110 &&
+			b[1] >> 6 == 0b10 &&
+			b[2] >> 6 == 0b10 &&
+			b[3] >> 6 == 0b10)
+		{
+			auto p0 = (uint32_t)b[0] & 0b111;
+			auto p1 = (uint32_t)b[1] & 0b11'1111;
+			auto p2 = (uint32_t)b[2] & 0b11'1111;
+			auto p3 = (uint32_t)b[2] & 0b11'1111;
+			codepoint = p0 << 18 | p1 << 12 | p2 << 6 | p3;
+			byteIdx += 4;
+		}
+		else {
+			codepoints.push_back(0xFFFDu);
+			byteIdx += 1;
+		}
+		codepoints.push_back(codepoint);
+	}
+
+	ret.text = std::u32string(codepoints.begin(), codepoints.end());
+	return ret;
+}
+
 #endif
diff --git a/engine/render/r_font.cpp b/engine/render/r_font.cpp
index deb564e..81f35a7 100644
--- a/engine/render/r_font.cpp
+++ b/engine/render/r_font.cpp
@@ -80,7 +80,8 @@ r_font_c::r_font_c(r_renderer_c* renderer, const char* fontName)
 				maxHeight = h;
 			}
 			fh->numGlyph = 0;
-		} else if (fh && sscanf(sub.c_str(), "GLYPH %u %u %u %d %d;", &x, &y, &w, &sl, &sr) == 5) {
+		}
+		else if (fh && sscanf(sub.c_str(), "GLYPH %u %u %u %d %d;", &x, &y, &w, &sl, &sr) == 5) {
 			// Add glyph
 			if (fh->numGlyph >= 128) continue;
 			f_glyph_s* glyph = &fh->glyphs[fh->numGlyph++];
@@ -126,128 +127,222 @@ r_font_c::~r_font_c()
 // Font Renderer
 // =============
 
-int r_font_c::StringWidthInternal(f_fontHeight_s* fh, const char* str)
+std::u32string BuildTofuString(char32_t cp) {
+	// Format unhandled Unicode codepoints like U+0123, higher planes have wider numbers.
+	fmt::memory_buffer buf;
+	fmt::format_to(fmt::appender(buf), "[U+{:04X}]", (uint32_t)cp);
+	std::u32string ret;
+	ret.reserve(buf.size());
+	std::copy(buf.begin(), buf.end(), std::back_inserter(ret));
+	return ret;
+}
+
+int const tofuSizeReduction = 3;
+
+int r_font_c::StringWidthInternal(f_fontHeight_s* fh, std::u32string_view str, int height, float scale)
 {
+	int heightIdx = (int)(std::find(fontHeights, fontHeights + numFontHeight, fh) - fontHeights);
+	auto tofuFont = FindSmallerFontHeight(height, heightIdx, tofuSizeReduction);
+
+	auto measureCodepoint = [](f_fontHeight_s* fh, char32_t cp) {
+		auto& glyph = fh->Glyph((char)(unsigned char)cp);
+		return glyph.width + glyph.spLeft + glyph.spRight;
+	};
+
 	int width = 0;
-	while (*str && *str != '\n') {
-		int escLen = IsColorEscape(str);
+	for (size_t idx = 0; idx < str.size();) {
+		auto ch = str[idx];
+		int escLen = IsColorEscape(str.substr(idx));
 		if (escLen) {
-			str+= escLen;
-		} else if (*str == '\t') {
+			idx += escLen;
+		}
+		else if (ch >= (unsigned)fh->numGlyph) {
+			auto tofu = BuildTofuString(ch);
+			for (auto ch : tofu) {
+				width += measureCodepoint(tofuFont.fh, ch);
+			}
+			++idx;
+		}
+		else if (ch == U'\t') {
 			auto& glyph = fh->Glyph(' ');
 			int spWidth = glyph.width + glyph.spLeft + glyph.spRight;
-			width += spWidth << 2;
-			str++;
-		} else {
-			auto& glyph = fh->Glyph(*str);
-			width+= glyph.width + glyph.spLeft + glyph.spRight;
-			str++;
+			width += (int)(spWidth * 4 * scale);
+			++idx;
+		}
+		else {
+			width += (int)(measureCodepoint(fh, ch) * scale);
+			++idx;
 		}
 	}
 	return width;
 }
 
-int r_font_c::StringWidth(int height, const char* str)
+int r_font_c::StringWidth(int height, std::u32string_view str)
 {
-	f_fontHeight_s *fh = fontHeights[height > maxHeight? (numFontHeight - 1) : fontHeightMap[height]];
+	auto mainFont = FindFontHeight(height);
+	f_fontHeight_s* fh = mainFont.fh;
 	int max = 0;
-	const char* lptr = str;
-	while (*lptr) {
-		if (*lptr != '\n') {
-			int lw = (int)(StringWidthInternal(fh, lptr) * (double)height / fh->height);
-			if (lw > max) max = lw;
+	float const scale = (float)height / fh->height;
+	for (auto I = str.begin(); I != str.end(); ++I) {
+		auto lineEnd = std::find(I, str.end(), U'\n');
+		if (I != lineEnd) {
+			std::u32string_view line(&*I, std::distance(I, lineEnd));
+			int lw = (int)StringWidthInternal(fh, line, height, scale);
+			max = (std::max)(max, lw);
 		}
-		const char* nptr = strchr(lptr, '\n');
-		if (nptr) {
-			lptr = nptr + 1;
-		} else {
+		if (lineEnd == str.end()) {
 			break;
 		}
+		I = lineEnd;
 	}
 	return max;
 }
 
-const char* r_font_c::StringCursorInternal(f_fontHeight_s* fh, const char* str, int curX)
+size_t r_font_c::StringCursorInternal(f_fontHeight_s* fh, std::u32string_view str, int height, float scale, int curX)
 {
+	int heightIdx = (int)(std::find(fontHeights, fontHeights + numFontHeight, fh) - fontHeights);
+	auto tofuFont = FindSmallerFontHeight(height, heightIdx, tofuSizeReduction);
+
+	auto measureCodepoint = [](f_fontHeight_s* fh, char32_t cp) {
+		auto& glyph = fh->Glyph((char)(unsigned char)cp);
+		return glyph.width + glyph.spLeft + glyph.spRight;
+	};
+
 	int x = 0;
-	while (*str && *str != '\n') {
-		int escLen = IsColorEscape(str);
+	auto I = str.begin();
+	auto lineEnd = std::find(I, str.end(), U'\n');
+	while (I != lineEnd) {
+		auto tail = str.substr(std::distance(str.begin(), I));
+		int escLen = IsColorEscape(tail);
 		if (escLen) {
-			str+= escLen;
-		} else if (*str == '\t') {
+			I += escLen;
+		}
+		else if (*I >= (unsigned)fh->numGlyph) {
+			auto tofu = BuildTofuString(*I);
+			int tofuWidth = 0;
+			for (auto ch : tofu) {
+				tofuWidth += measureCodepoint(tofuFont.fh, ch);
+			}
+			int halfWidth = (int)(tofuWidth / 2.0f);
+			x += halfWidth;
+			if (curX <= x) {
+				break;
+			}
+			x += tofuWidth - halfWidth;
+			++I;
+		}
+		else if (*I == U'\t') {
 			auto& glyph = fh->Glyph(' ');
 			int spWidth = glyph.width + glyph.spLeft + glyph.spRight;
-			x+= spWidth << 1;
+			int fullWidth = (int)(spWidth * 4 * scale);
+			int halfWidth = (int)(fullWidth / 2.0f);
+			x += halfWidth;
 			if (curX <= x) {
 				break;
 			}
-			x+= spWidth << 1;
-			str++;
-		} else {
-			auto& glyph = fh->Glyph(*str);
-			x+= glyph.width + glyph.spLeft + glyph.spRight;
+			x += fullWidth - halfWidth;
+			++I;
+		}
+		else {
+			x += (int)(measureCodepoint(fh, *I) * scale);
 			if (curX <= x) {
 				break;
 			}
-			str++;
+			++I;
 		}
 	}
-	return str;
+	return std::distance(str.begin(), I);
 }
 
-int	r_font_c::StringCursorIndex(int height, const char* str, int curX, int curY)
+int	r_font_c::StringCursorIndex(int height, std::u32string_view str, int curX, int curY)
 {
-	f_fontHeight_s *fh = fontHeights[height > maxHeight? (numFontHeight - 1) : fontHeightMap[height]];
+	auto mainFont = FindFontHeight(height);
+	f_fontHeight_s* fh = mainFont.fh;
 	int lastIndex = 0;
 	int lineY = height;
-	curX = (int)(curX / (double)height * fh->height);
-	const char* lptr = str;
-	while (1) {
-		lastIndex = (int)(StringCursorInternal(fh, lptr, curX) - str);
+	float scale = (float)height / fh->height;
+
+	auto I = str.begin();
+	while (I != str.end()) {
+		auto lineEnd = std::find(I, str.end(), U'\n');
+		auto line = str.substr(std::distance(str.begin(), I), std::distance(I, lineEnd));
+		lastIndex = (int)(StringCursorInternal(fh, line, height, scale, curX));
 		if (curY <= lineY) {
 			break;
 		}
-		const char* nptr = strchr(lptr, '\n');
-		if (nptr) {
-			lptr = nptr + 1;
-		} else {
+		if (lineEnd == str.end()) {
 			break;
 		}
-		lineY+= height;
+		I = lineEnd + 1;
+		lineY += height;
 	}
-	return lastIndex;
+	return (int)std::distance(str.begin(), I) + lastIndex;
 }
 
-void r_font_c::DrawTextLine(scp_t pos, int align, int height, col4_t col, const char* str)
+r_font_c::EmbeddedFontSpec r_font_c::FindSmallerFontHeight(int height, int heightIdx, int sizeReduction) {
+	EmbeddedFontSpec ret{};
+	ret.fh = fontHeights[heightIdx];
+	ret.yPad = 0;
+	for (int tofuIdx = heightIdx - 1; tofuIdx >= 0; --tofuIdx) {
+		auto candFh = fontHeights[tofuIdx];
+		int heightDiff = height - candFh->height;
+		if (heightDiff >= sizeReduction) {
+			ret.fh = candFh;
+			ret.yPad = (int)std::ceil(heightDiff / 2.0f);
+			break;
+		}
+	}
+	return ret;
+}
+
+r_font_c::FontHeightEntry r_font_c::FindFontHeight(int height) {
+	FontHeightEntry ret{};
+	if (height > maxHeight) {
+		// Too large heights get the largest font size.
+		ret.heightIdx = numFontHeight - 1;
+	}
+	else if (height < 0) {
+		// Negative heights get the smallest font size.
+		ret.heightIdx = 0;
+	}
+	else {
+		ret.heightIdx = fontHeightMap[height];
+	}
+	ret.fh = fontHeights[ret.heightIdx];
+	return ret;
+}
+
+void r_font_c::DrawTextLine(scp_t pos, int align, int height, col4_t col, std::u32string_view str)
 {
 	// Check if the line is visible
 	if (pos[Y] >= renderer->sys->video->vid.size[1] || pos[Y] <= -height) {
 		// Just process the colour codes
-		while (*str && *str != '\n') {
+		while (!str.empty()) {
 			// Check for escape character
 			int escLen = IsColorEscape(str);
 			if (escLen) {
-				ReadColorEscape(str, col);
+				str = ReadColorEscape(str, col);
 				col[3] = 1.0f;
 				renderer->curLayer->Color(col);
-				str+= escLen;
 				continue;
 			}
-			str++;
+			str = str.substr(1);
 		}
 		return;
 	}
 
 	// Find best height to use
-	f_fontHeight_s *fh = fontHeights[height > maxHeight? (numFontHeight - 1) : fontHeightMap[height]];
+	auto mainFont = FindFontHeight(height);
+	f_fontHeight_s* fh = mainFont.fh;
 	float scale = (float)height / fh->height;
+	auto tofuFont = FindSmallerFontHeight(height, mainFont.heightIdx, tofuSizeReduction);
 
 	// Calculate the string position
 	float x = pos[X];
 	float y = pos[Y];
 	if (align != F_LEFT) {
 		// Calculate the real width of the string
-		float width = StringWidthInternal(fh, str) * scale;
+		float width = (float)StringWidthInternal(fh, str, height, scale);
 		switch (align) {
 		case F_CENTRE:
 			x = floor((renderer->VirtualScreenWidth() - width) / 2.0f + pos[X]);
@@ -264,57 +359,72 @@ void r_font_c::DrawTextLine(scp_t pos, int align, int height, col4_t col, const
 		}
 	}
 
-	renderer->curLayer->Bind(fh->tex);
+	r_tex_c* curTex{};
+
+	auto drawCodepoint = [this, &curTex, &x, y](f_fontHeight_s* fh, int height, float scale, int yShift, char32_t cp) {
+		float cpY = y + yShift;
+		if (curTex != fh->tex) {
+			curTex = fh->tex;
+			renderer->curLayer->Bind(fh->tex);
+		}
+		auto& glyph = fh->Glyph((char)(unsigned char)cp);
+		x += glyph.spLeft * scale;
+		if (glyph.width) {
+			float w = glyph.width * scale;
+			if (x + w >= 0 && x < renderer->VirtualScreenWidth()) {
+				renderer->curLayer->Quad(
+					glyph.tcLeft, glyph.tcTop, x, cpY,
+					glyph.tcRight, glyph.tcTop, x + w, cpY,
+					glyph.tcRight, glyph.tcBottom, x + w, cpY + height,
+					glyph.tcLeft, glyph.tcBottom, x, cpY + height
+				);
+			}
+			x += w;
+		}
+		x += glyph.spRight * scale;
+		x = std::ceil(x);
+	};
 
 	// Render the string
-	while (*str && *str != '\n') {
-		// Skip unprintable characters
-		if (*str >= fh->numGlyph || *str < 0) {
-			str++;
+	for (auto tail = str; !tail.empty();) {
+		// Draw unprintable characters as tofu placeholders
+		auto ch = tail[0];
+		if (ch >= (unsigned)fh->numGlyph) {
+			auto tofu = BuildTofuString(ch);
+			for (auto ch : tofu) {
+				drawCodepoint(tofuFont.fh, tofuFont.fh->height, 1.0f, tofuFont.yPad, ch);
+			}
+			tail = tail.substr(1);
 			continue;
 		}
 
 		// Check for escape character
-		int escLen = IsColorEscape(str);
+		int escLen = IsColorEscape(tail);
 		if (escLen) {
-			ReadColorEscape(str, col);
+			tail = ReadColorEscape(tail, col);
 			col[3] = 1.0f;
 			renderer->curLayer->Color(col);
-			str+= escLen;
 			continue;
 		}
 
 		// Handle tabs
-		if (*str == '\t') {
+		if (ch == U'\t') {
 			auto& glyph = fh->Glyph(' ');
 			int spWidth = glyph.width + glyph.spLeft + glyph.spRight;
 			x+= (spWidth << 2) * scale;
-			str++;
+			tail = tail.substr(1);
 			continue;
 		}
 
 		// Draw glyph
-		auto& glyph = fh->Glyph(*str++);
-		x+= glyph.spLeft * scale;
-		if (glyph.width) {
-			float w = glyph.width * scale;
-			if (x + w >= 0 && x < renderer->VirtualScreenWidth()) {
-				renderer->curLayer->Quad(
-					glyph.tcLeft, glyph.tcTop, x, y,
-					glyph.tcRight, glyph.tcTop, x + w, y,
-					glyph.tcRight, glyph.tcBottom, x + w, y + height,
-					glyph.tcLeft, glyph.tcBottom, x, y + height
-				);
-			}
-			x+= w;
-		}
-		x+= glyph.spRight * scale;
+		drawCodepoint(fh, height, scale, 0, ch);
+		tail = tail.substr(1);
 	}
 }
 
-void r_font_c::Draw(scp_t pos, int align, int height, col4_t col, const char* str)
+void r_font_c::Draw(scp_t pos, int align, int height, col4_t col, std::u32string_view str)
 {
-	if (*str == 0) {
+	if (str.empty()) {
 		pos[Y]+= height;
 		return;
 	}
@@ -323,18 +433,17 @@ void r_font_c::Draw(scp_t pos, int align, int height, col4_t col, const char* st
 	renderer->curLayer->Color(col);
 
 	// Separate into lines and render them
-	const char* lptr = str;
-	while (*lptr) {
-		if (*lptr != '\n') {
-			DrawTextLine(pos, align, height, col, lptr);
+	for (auto I = str.begin(); I != str.end(); ++I) {
+		auto lineEnd = std::find(I, str.end(), U'\n');
+		if (I != lineEnd) {
+			std::u32string_view line(&*I, std::distance(I, lineEnd));
+			DrawTextLine(pos, align, height, col, line);
 		}
-		pos[Y]+= height;
-		const char* nptr = strchr(lptr, '\n');
-		if (nptr) {
-			lptr = nptr + 1;
-		} else {
+		pos[Y] += height;
+		if (lineEnd == str.end()) {
 			break;
 		}
+		I = lineEnd;
 	}
 }
 
@@ -351,5 +460,6 @@ void r_font_c::VDraw(scp_t pos, int align, int height, col4_t col, const char* f
 	char str[65536];
 	vsnprintf(str, 65535, fmt, va);
 	str[65535] = 0;
-	Draw(pos, align, height, col, str);
+	auto idxStr = IndexUTF8ToUTF32(str);
+	Draw(pos, align, height, col, idxStr.text);
 }
diff --git a/engine/render/r_font.h b/engine/render/r_font.h
index 4caecce..cbae830 100644
--- a/engine/render/r_font.h
+++ b/engine/render/r_font.h
@@ -8,22 +8,36 @@
 // Classes
 // =======
 
+#include <string_view>
+
 // Font
 class r_font_c {
 public:
 	r_font_c(class r_renderer_c* renderer, const char* fontName);
 	~r_font_c();
 
-	int		StringWidth(int height, const char* str);
-	int		StringCursorIndex(int height, const char* str, int curX, int curY);
-	void	Draw(scp_t pos, int align, int height, col4_t col, const char* str);
+	int		StringWidth(int height, std::u32string_view str);
+	int		StringCursorIndex(int height, std::u32string_view str, int curX, int curY);
+	void	Draw(scp_t pos, int align, int height, col4_t col, std::u32string_view str);
 	void	FDraw(scp_t pos, int align, int height, col4_t col, const char* fmt, ...);
 	void	VDraw(scp_t pos, int align, int height, col4_t col, const char* fmt, va_list va);
 
 private:
-	int		StringWidthInternal(struct f_fontHeight_s* fh, const char* str);
-	const char*	StringCursorInternal(struct f_fontHeight_s* fh, const char* str, int curX);
-	void	DrawTextLine(scp_t pos, int align, int height, col4_t col, const char* str);
+	int		StringWidthInternal(struct f_fontHeight_s* fh, std::u32string_view str, int height, float scale);
+	size_t	StringCursorInternal(struct f_fontHeight_s* fh, std::u32string_view str, int height, float scale, int curX);
+	void	DrawTextLine(scp_t pos, int align, int height, col4_t col, std::u32string_view str);
+
+	struct EmbeddedFontSpec {
+		f_fontHeight_s* fh;
+		int yPad;
+	};
+	EmbeddedFontSpec FindSmallerFontHeight(int height, int heightIdx, int sizeReduction);
+	
+	struct FontHeightEntry {
+		f_fontHeight_s* fh;
+		int heightIdx;
+	};
+	FontHeightEntry FindFontHeight(int height);
 
 	class r_renderer_c* renderer = nullptr;
 	int		numFontHeight = 0;
diff --git a/engine/render/r_main.cpp b/engine/render/r_main.cpp
index ba0871b..de1a277 100644
--- a/engine/render/r_main.cpp
+++ b/engine/render/r_main.cpp
@@ -1771,6 +1771,7 @@ void r_renderer_c::DrawImageQuad(r_shaderHnd_c* hnd, glm::vec2 p0, glm::vec2 p1,
 
 void r_renderer_c::DrawString(float x, float y, int align, int height, const col4_t col, int font, const char* str)
 {
+	auto idxStr = IndexUTF8ToUTF32(str);
 	if (font < 0 || font >= F_NUMFONTS) {
 		font = F_FIXED;
 	}
@@ -1779,10 +1780,10 @@ void r_renderer_c::DrawString(float x, float y, int align, int height, const col
 	if (col) {
 		col4_t tcol;
 		Vector4Copy(col, tcol);
-		fonts[font]->Draw(pos, align, height, tcol, str);
+		fonts[font]->Draw(pos, align, height, tcol, idxStr.text);
 	}
 	else {
-		fonts[font]->Draw(pos, align, height, drawColor, str);
+		fonts[font]->Draw(pos, align, height, drawColor, idxStr.text);
 	}
 }
 
@@ -1810,18 +1811,31 @@ void r_renderer_c::DrawStringFormat(float x, float y, int align, int height, con
 
 int	r_renderer_c::DrawStringWidth(int height, int font, const char* str)
 {
+	if (!*str) {
+		return 0;
+	}
+	auto idxStr = IndexUTF8ToUTF32(str);
 	if (font < 0 || font >= F_NUMFONTS) {
 		font = F_FIXED;
 	}
-	return fonts[font]->StringWidth(height, str);
+	return fonts[font]->StringWidth(height, idxStr.text);
 }
 
 int r_renderer_c::DrawStringCursorIndex(int height, int font, const char* str, int curX, int curY)
 {
+	if (!*str) {
+		return 0;
+	}
+	std::string_view narrowView(str);
+	auto idxStr = IndexUTF8ToUTF32(narrowView);
 	if (font < 0 || font >= F_NUMFONTS) {
 		font = F_FIXED;
 	}
-	return fonts[font]->StringCursorIndex(height, str, curX, curY);
+	size_t index = fonts[font]->StringCursorIndex(height, idxStr.text, curX, curY);
+	if (index < idxStr.sourceCodeUnitOffsets.size()) {
+		return (int)idxStr.sourceCodeUnitOffsets[index];
+	}
+	return (int)narrowView.size();
 }
 
 // ==============

From 0efdc2c1d76fdfe9de2d0a0d318d0be1492146e6 Mon Sep 17 00:00:00 2001
From: Lars Viklund <zao@zao.se>
Date: Wed, 3 Apr 2024 02:05:54 +0200
Subject: [PATCH 07/16] feat: implement UTF-8 os.rename and os.remove

The LuaJIT implementations of `os.rename` and `os.remove` are using
narrow functions in the system encoding.

This change redirects the functions to variants that widens UTF-8
strings from Lua on Windows and calls wide CRT versions of the functions
to properly rename and remove files.
---
 ui_api.cpp | 39 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)

diff --git a/ui_api.cpp b/ui_api.cpp
index 44b9adf..1bbf976 100644
--- a/ui_api.cpp
+++ b/ui_api.cpp
@@ -1673,6 +1673,36 @@ static int l_RemoveDir(lua_State* L)
 	}
 }
 
+static int l_RemoveFile(lua_State* L)
+{
+	char const* pathStr = luaL_checkstring(L, 1);
+#ifdef _WIN32
+	wchar_t* widePath = WidenUTF8String(pathStr);
+	int rc = _wremove(widePath);
+	FreeWideString(widePath);
+#else
+	int rc = remove(pathStr);
+#endif
+	return luaL_fileresult(L, rc == 0, pathStr);
+}
+
+static int l_RenameFile(lua_State* L)
+{
+	char const* srcStr = luaL_checkstring(L, 1);
+	char const* dstStr = luaL_checkstring(L, 2);
+#ifdef _WIN32
+	wchar_t* wideSrc = WidenUTF8String(srcStr);
+	wchar_t* wideDst = WidenUTF8String(dstStr);
+	int rc = _wrename(wideSrc, wideDst);
+	FreeWideString(wideSrc);
+	FreeWideString(wideDst);
+#else
+	int rc = rename(srcStr, dstStr);
+#endif
+
+	return luaL_fileresult(L, rc == 0, srcStr);
+}
+
 static int l_SetWorkDir(lua_State* L)
 {
 	ui_main_c* ui = GetUIPtr(L);
@@ -2028,7 +2058,16 @@ int ui_main_c::InitAPI(lua_State* L)
 			lua_pushcfunction(L, l_OpenFile);
 			lua_setfield(L, -2, "open");
 		}
+		lua_pop(L, 1);
+
+		lua_getglobal(L, "os");
+		if (!lua_isnil(L, -1)) {
+			lua_pushcfunction(L, l_RemoveFile);
+			lua_setfield(L, -2, "remove");
 
+			lua_pushcfunction(L, l_RenameFile);
+			lua_setfield(L, -2, "rename");
+		}
 		lua_pop(L, 1);
 	}
 

From 1fe8c770ef70b4ee9fa91e4cf341b82367e7176a Mon Sep 17 00:00:00 2001
From: Lars Viklund <zao@zao.se>
Date: Sun, 7 Apr 2024 20:06:45 +0200
Subject: [PATCH 08/16] fix: harden string conversion against invalid data

---
 engine/common.h          |  1 +
 engine/common/common.cpp | 51 +++++++++++++++++++++++++++++++++++-----
 2 files changed, 46 insertions(+), 6 deletions(-)

diff --git a/engine/common.h b/engine/common.h
index d34dfa2..159a45c 100644
--- a/engine/common.h
+++ b/engine/common.h
@@ -490,6 +490,7 @@ char*	_AllocStringLen(size_t len, const char* file, int line);
 #define AllocStringLen(s) _AllocStringLen(s, __FILE__, __LINE__)
 void	FreeString(const char* str);
 dword	StringHash(const char* str, int mask);
+dword	StringHash(std::string_view str, int mask);
 
 struct IndexedUTF32String {
 	std::u32string text;
diff --git a/engine/common/common.cpp b/engine/common/common.cpp
index b26aa7f..3340bad 100644
--- a/engine/common/common.cpp
+++ b/engine/common/common.cpp
@@ -335,14 +335,39 @@ dword StringHash(const char* str, int mask)
 	return hash & mask;
 }
 
+dword StringHash(std::string_view str, int mask)
+{
+	size_t len = str.length();
+	dword hash = 0;
+	for (size_t i = 0; i < len; i++) {
+		hash += (str[i] * 4999) ^ (((dword)i + 17) * 2003);
+	}
+	return hash & mask;
+}
+
 #ifdef _WIN32
 #include <Windows.h>
 
 static wchar_t* WidenCodepageString(const char* str, UINT codepage)
 {
-	int cch = MultiByteToWideChar(codepage, 0, str, -1, nullptr, 0);
-	wchar_t* wstr = new wchar_t[cch];
-	MultiByteToWideChar(codepage, 0, str, -1, wstr, cch);
+	if (!str) {
+		return nullptr;
+	}
+	// Early-out if empty, avoids ambigious error return from MBTWC.
+	if (!*str) {
+		wchar_t* wstr = new wchar_t[1];
+		*wstr = L'\0';
+		return wstr;
+	}
+	DWORD cb = (DWORD)strlen(str);
+	int cch = MultiByteToWideChar(codepage, MB_ERR_INVALID_CHARS, str, cb, nullptr, 0);
+	if (cch == 0) {
+		// Invalid string or other error.
+		return nullptr;
+	}
+	wchar_t* wstr = new wchar_t[cch + 1]; // sized MBTWC doesn't include terminator.
+	MultiByteToWideChar(codepage, 0, str, cb, wstr, cch);
+	wstr[cch] = '\0';
 	return wstr;
 }
 
@@ -363,9 +388,23 @@ wchar_t* WidenUTF8String(const char* str)
 
 char* NarrowCodepageString(const wchar_t* str, UINT codepage)
 {
-	int cb = WideCharToMultiByte(codepage, 0, str, -1, nullptr, 0, nullptr, nullptr);
-	char* nstr = new char[cb];
-	WideCharToMultiByte(codepage, 0, str, -1, nstr, cb, nullptr, nullptr);
+	if (!str) {
+		return nullptr;
+	}
+	if (!*str) {
+		char* nstr = new char[1];
+		*nstr = '\0';
+		return nstr;
+	}
+	DWORD cch = (DWORD)wcslen(str);
+	int cb = WideCharToMultiByte(codepage, 0, str, cch, nullptr, 0, nullptr, nullptr);
+	if (cb == 0) {
+		// Invalid string or other error.
+		return nullptr;
+	}
+	char* nstr = new char[cb + 1];
+	WideCharToMultiByte(codepage, 0, str, cch, nstr, cb, nullptr, nullptr);
+	nstr[cb] = '\0';
 	return nstr;
 }
 

From 6603d7c2b331e92cc5d7cd1cbf033a98ebe1ce34 Mon Sep 17 00:00:00 2001
From: Lars Viklund <zao@zao.se>
Date: Sun, 7 Apr 2024 21:03:53 +0200
Subject: [PATCH 09/16] feat: make images and file streams use `fs::path`

This change adds some foundational changes to widen the rest of the
runtime to use `std::filesystem::path` and try to detect misuse where
narrow strings are used as-is.

Dependent code in the codebase will fail to build with this change.
---
 engine/common/streams.cpp  | 16 +++++++---
 engine/common/streams.h    | 24 +++++++++++++--
 engine/core/core_image.cpp | 61 +++++++++++++++++++-------------------
 engine/core/core_image.h   | 40 +++++++++++++++++--------
 4 files changed, 92 insertions(+), 49 deletions(-)

diff --git a/engine/common/streams.cpp b/engine/common/streams.cpp
index 6274143..82d8222 100644
--- a/engine/common/streams.cpp
+++ b/engine/common/streams.cpp
@@ -255,10 +255,14 @@ bool fileInputStream_c::Read(void* out, size_t len)
 	return fread(out, len, 1, file) < 1;
 }
 
-bool fileInputStream_c::FileOpen(const char* fileName, bool binary)
+bool fileInputStream_c::FileOpen(std::filesystem::path const& fileName, bool binary)
 {
 	FileClose();
-	file = fopen(fileName, binary? "rb" : "r");
+#ifdef _WIN32
+	file = _wfopen(fileName.c_str(), binary ? L"rb" : L"r");
+#else
+	file = fopen(fileName.c_str(), binary ? "rb" : "r");
+#endif
 	if ( !file ) {
 		return true;
 	}
@@ -277,10 +281,14 @@ bool fileOutputStream_c::Write(const void* in, size_t len)
 	return fwrite(in, len, 1, file) < 1;
 }
 
-bool fileOutputStream_c::FileOpen(const char* fileName, bool binary)
+bool fileOutputStream_c::FileOpen(std::filesystem::path const& fileName, bool binary)
 {
 	FileClose();
-	file = fopen(fileName, binary? "wb" : "w");
+#ifdef _WIN32
+	file = _wfopen(fileName.c_str(), binary ? L"wb" : L"w");
+#else
+	file = fopen(fileName.c_str(), binary ? "wb" : "w");
+#endif
 	if ( !file ) {
 		return true;
 	}
diff --git a/engine/common/streams.h b/engine/common/streams.h
index ab6b130..db1d554 100644
--- a/engine/common/streams.h
+++ b/engine/common/streams.h
@@ -4,6 +4,10 @@
 // Streams Header
 //
 
+#include <filesystem>
+#include <string>
+#include <string_view>
+
 // ==========
 // Base Class
 // ==========
@@ -94,13 +98,29 @@ class fileStreamBase_c: public ioStream_c {
 class fileInputStream_c: public fileStreamBase_c {
 public:
 	bool Read(void* out, size_t len);
-	bool FileOpen(const char* fileName, bool binary);
+
+	// Force compile error on narrow strings to favour `std::filesystem::path`.
+	// These are unfortunately necessary as the path constructor is eager to
+	// interpret narrow strings as the ACP codepage. Prefer using
+	// `std::filesystem::u8path` (C++17) or `std::u8string` (since C++20) in
+	// the calls to these functions.
+	bool FileOpen(char const*, bool) = delete;
+	bool FileOpen(std::string const&, bool) = delete;
+	bool FileOpen(std::string_view*, bool) = delete;
+
+	bool FileOpen(std::filesystem::path const& fileName, bool binary);
 };
 
 class fileOutputStream_c: public fileStreamBase_c {
 public:
 	bool Write(const void* in, size_t len);
-	bool FileOpen(const char* fileName, bool binary);
+
+	// Force compile error like in `fileInputStream_c` on non-path types.
+	bool FileOpen(char const*, bool) = delete;
+	bool FileOpen(std::string const&, bool) = delete;
+	bool FileOpen(std::string_view*, bool) = delete;
+
+	bool FileOpen(std::filesystem::path const& fileName, bool binary);
 	void FilePrintf(const char* fmt, ...);
 	void FileFlush();
 };
diff --git a/engine/core/core_image.cpp b/engine/core/core_image.cpp
index 26a8cde..079d245 100644
--- a/engine/core/core_image.cpp
+++ b/engine/core/core_image.cpp
@@ -79,39 +79,39 @@ void image_c::Free()
 	tex = {};
 }
 
-bool image_c::Load(const char* fileName, std::optional<size_callback_t> sizeCallback)
+bool image_c::Load(std::filesystem::path const& fileName, std::optional<size_callback_t> sizeCallback)
 {
 	return true; // o_O
 }
 
-bool image_c::Save(const char* fileName) 
+bool image_c::Save(std::filesystem::path const& fileName) 
 {
 	return true; // o_O
 }
 
-image_c* image_c::LoaderForFile(IConsole* conHnd, const char* fileName)
+image_c* image_c::LoaderForFile(IConsole* conHnd, std::filesystem::path const& fileName)
 {
+	auto nameU8 = fileName.u8string();
 	fileInputStream_c in;
 	if (in.FileOpen(fileName, true)) {
-		conHnd->Warning("'%s' doesn't exist or cannot be opened", fileName);
+		conHnd->Warning("'%s' doesn't exist or cannot be opened", nameU8.c_str());
 		return NULL;
 	}
 
 	// Detect first by extension, as decompressing could be expensive.
-	auto p = std::filesystem::path(fileName); // NOTE(LV): This should be u8path later.
-	if (p.extension() == ".zst") {
-		auto inner = p.filename();
+	if (fileName.extension() == ".zst") {
+		auto inner = fileName.filename();
 		inner.replace_extension();
 		if (inner.extension() == ".dds")
 			return new dds_c(conHnd);
 	}
-	if (p.extension() == ".dds")
+	if (fileName.extension() == ".dds")
 		return new dds_c(conHnd);
 
-	// Attempt to detect image file type from first 12 bytes of file
+	// Attempt to detect image file type from first 4 bytes of file
 	byte dat[4];
 	if (in.Read(dat, 4)) {
-		conHnd->Warning("'%s': cannot read image file (file is corrupt?)", fileName);
+		conHnd->Warning("'%s': cannot read image file (file is corrupt?)", nameU8.c_str());
 		return NULL;
 	}
 	if (dat[0] == 0xFF && dat[1] == 0xD8) {
@@ -130,7 +130,7 @@ image_c* image_c::LoaderForFile(IConsole* conHnd, const char* fileName)
 		// Detect all valid image types (whether supported or not)
 		return new targa_c(conHnd);
 	}
-	conHnd->Warning("'%s': unsupported image file format", fileName);
+	conHnd->Warning("'%s': unsupported image file format", nameU8.c_str());
 	return NULL;
 }
 
@@ -153,24 +153,24 @@ struct tgaHeader_s {
 };
 #pragma pack(pop)
 
-bool targa_c::Load(const char* fileName, std::optional<size_callback_t> sizeCallback)
+bool targa_c::Load(std::filesystem::path const& fileName, std::optional<size_callback_t> sizeCallback)
 {
-	Free();
-
 	// Open the file
 	fileInputStream_c in;
 	if (in.FileOpen(fileName, true)) {
 		return true;
 	}
 
+	auto nameU8 = fileName.u8string();
+
 	// Read header
 	tgaHeader_s hdr;
 	if (in.TRead(hdr)) {
-		con->Warning("TGA '%s': couldn't read header", fileName);
+		con->Warning("TGA '%s': couldn't read header", nameU8.c_str());
 		return true;
 	}
 	if (hdr.colorMapType) {
-		con->Warning("TGA '%s': color mapped images not supported", fileName);
+		con->Warning("TGA '%s': color mapped images not supported", nameU8.c_str());
 		return true;
 	}
 	in.Seek(hdr.idLen, SEEK_CUR);
@@ -188,7 +188,7 @@ bool targa_c::Load(const char* fileName, std::optional<size_callback_t> sizeCall
 		if (ittable[it_m][0] == (hdr.imgType & 7) && ittable[it_m][1] == hdr.depth) break;
 	}
 	if (it_m == 3) {
-		con->Warning("TGA '%s': unsupported image type (it: %d pd: %d)", fileName, hdr.imgType, hdr.depth);
+		con->Warning("TGA '%s': unsupported image type (it: %d pd: %d)", nameU8.c_str(), hdr.imgType, hdr.depth);
 		return true;
 	}
 
@@ -211,7 +211,7 @@ bool targa_c::Load(const char* fileName, std::optional<size_callback_t> sizeCall
 				in.TRead(rlehdr);
 				int rlen = ((rlehdr & 0x7F) + 1) * comp; 
 				if (x + rlen > rowSize) {
-					con->Warning("TGA '%s': invalid RLE coding (overlong row)", fileName);
+					con->Warning("TGA '%s': invalid RLE coding (overlong row)", nameU8.c_str());
 					delete[] dat;
 					return true;
 				}
@@ -247,7 +247,7 @@ bool targa_c::Load(const char* fileName, std::optional<size_callback_t> sizeCall
 	return !CopyRaw(type, width, height, dat);
 }
 
-bool targa_c::Save(const char* fileName)
+bool targa_c::Save(std::filesystem::path const& fileName)
 {
 	auto format = tex.format();
 	if (is_compressed(format) || !is_unsigned_integer(format))
@@ -276,7 +276,7 @@ bool targa_c::Save(const char* fileName)
 // JPEG Image
 // ==========
 
-bool jpeg_c::Load(const char* fileName, std::optional<size_callback_t> sizeCallback)
+bool jpeg_c::Load(std::filesystem::path const& fileName, std::optional<size_callback_t> sizeCallback)
 {
 	Free();
 
@@ -286,6 +286,8 @@ bool jpeg_c::Load(const char* fileName, std::optional<size_callback_t> sizeCallb
 		return true;
 	}
 
+	auto nameU8 = fileName.u8string();
+
 	std::vector<byte> fileData(in.GetLen());
 	if (in.Read(fileData.data(), fileData.size())) {
 		return true;
@@ -295,7 +297,7 @@ bool jpeg_c::Load(const char* fileName, std::optional<size_callback_t> sizeCallb
 		return true;
 	}
 	if (in_comp != 1 && in_comp != 3) {
-		con->Warning("JPEG '%s': unsupported component count '%d'", fileName, in_comp);
+		con->Warning("JPEG '%s': unsupported component count '%d'", nameU8.c_str(), in_comp);
 		return true;
 	}
 	if (sizeCallback)
@@ -313,7 +315,7 @@ bool jpeg_c::Load(const char* fileName, std::optional<size_callback_t> sizeCallb
 	return !success;
 }
 
-bool jpeg_c::Save(const char* fileName)
+bool jpeg_c::Save(std::filesystem::path const& fileName)
 {
 	// JPEG only supports RGB and grayscale images
 	auto format = tex.format();
@@ -342,7 +344,7 @@ bool jpeg_c::Save(const char* fileName)
 // PNG Image
 // =========
 
-bool png_c::Load(const char* fileName, std::optional<size_callback_t> sizeCallback)
+bool png_c::Load(std::filesystem::path const& fileName, std::optional<size_callback_t> sizeCallback)
 {
 	Free();
 
@@ -380,7 +382,7 @@ bool png_c::Load(const char* fileName, std::optional<size_callback_t> sizeCallba
 	return !success;
 }
 
-bool png_c::Save(const char* fileName)
+bool png_c::Save(std::filesystem::path const& fileName)
 {
 	auto format = tex.format();
 	if (is_compressed(format) || !is_unsigned_integer(format))
@@ -409,7 +411,7 @@ bool png_c::Save(const char* fileName)
 // GIF Image
 // =========
 
-bool gif_c::Load(const char* fileName, std::optional<size_callback_t> sizeCallback)
+bool gif_c::Load(std::filesystem::path const& fileName, std::optional<size_callback_t> sizeCallback)
 {
 	// Open file
 	fileInputStream_c in;
@@ -440,7 +442,7 @@ bool gif_c::Load(const char* fileName, std::optional<size_callback_t> sizeCallba
 	}
 }
 
-bool gif_c::Save(const char* fileName)
+bool gif_c::Save(std::filesystem::path const& fileName)
 {
 	// HELL no.
 	return true;
@@ -450,9 +452,8 @@ bool gif_c::Save(const char* fileName)
 // DDS Image
 // =========
 
-bool dds_c::Load(const char* fileName, std::optional<size_callback_t> sizeCallback)
+bool dds_c::Load(std::filesystem::path const& fileName, std::optional<size_callback_t> sizeCallback)
 {
-	auto p = std::filesystem::path(fileName); // NOTE(LV): This should be u8path later.
 	// Open file
 	fileInputStream_c in;
 	if (in.FileOpen(fileName, true))
@@ -462,7 +463,7 @@ bool dds_c::Load(const char* fileName, std::optional<size_callback_t> sizeCallba
 	if (in.Read(fileData.data(), fileData.size()))
 		return true;
 
-	if (p.extension() == ".zst" || fileData.size() >= 4 && *(uint32_t*)fileData.data() == 0xFD2FB528) {
+	if (fileName.extension() == ".zst" || fileData.size() >= 4 && *(uint32_t*)fileData.data() == 0xFD2FB528) {
 		auto ret = DecompressZstandard(as_bytes(gsl::span(fileData)));
 		if (!ret.has_value())
 			return true;
@@ -476,7 +477,7 @@ bool dds_c::Load(const char* fileName, std::optional<size_callback_t> sizeCallba
 	return false;
 }
 
-bool dds_c::Save(const char* fileName)
+bool dds_c::Save(std::filesystem::path const& fileName)
 {
 	// Nope.
 	return true;
diff --git a/engine/core/core_image.h b/engine/core/core_image.h
index 0158157..ba47cf0 100644
--- a/engine/core/core_image.h
+++ b/engine/core/core_image.h
@@ -54,13 +54,27 @@ class image_c {
 
 	using size_callback_t = std::function<void(int, int)>;
 
-	virtual bool Load(const char* fileName, std::optional<size_callback_t> sizeCallback = {});
-	virtual bool Save(const char* fileName);
+	virtual bool Load(std::filesystem::path const& fileName, std::optional<size_callback_t> sizeCallback = {});
+	virtual bool Save(std::filesystem::path const& fileName);
 
 	bool CopyRaw(int type, dword width, dword height, const byte* dat);
 	void Free();
 
-	static image_c* LoaderForFile(IConsole* conHnd, const char* fileName);
+	static image_c* LoaderForFile(IConsole* conHnd, char const* fileName) = delete;
+	static image_c* LoaderForFile(IConsole* conHnd, std::filesystem::path const& fileName);
+
+private:
+	// Force compile error on narrow strings to favour `std::filesystem::path`.
+	// These are unfortunately necessary as the path constructor is eager to
+	// interpret narrow strings as the ACP codepage. Prefer using
+	// `std::filesystem::u8path` (C++17) or `std::u8string` (since C++20) in
+	// the calls to these functions.
+	virtual bool Load(char const* fileName) { return true; }
+	virtual bool Load(std::string const& fileName) { return true; }
+	virtual bool Load(std::string_view fileName) { return true; }
+	virtual bool Save(char const* fileName) { return true; }
+	virtual bool Save(std::string const& fileName) { return true; }
+	virtual bool Save(std::string_view fileName) { return true; }
 };
 
 // Targa Image
@@ -68,8 +82,8 @@ class targa_c : public image_c {
 public:
 	bool rle;
 	targa_c(IConsole* conHnd) : image_c(conHnd) { rle = true; }
-	bool Load(const char* fileName, std::optional<size_callback_t> sizeCallback = {}) override;
-	bool Save(const char* fileName) override;
+	bool Load(std::filesystem::path const& fileName, std::optional<size_callback_t> sizeCallback = {}) override;
+	bool Save(std::filesystem::path const& fileName) override;
 };
 
 // JPEG Image
@@ -77,30 +91,30 @@ class jpeg_c : public image_c {
 public:
 	int quality;
 	jpeg_c(IConsole* conHnd) : image_c(conHnd) { quality = 80; }
-	bool Load(const char* fileName, std::optional<size_callback_t> sizeCallback = {}) override;
-	bool Save(const char* fileName) override;
+	bool Load(std::filesystem::path const& fileName, std::optional<size_callback_t> sizeCallback = {}) override;
+	bool Save(std::filesystem::path const& fileName) override;
 };
 
 // PNG Image
 class png_c : public image_c {
 public:
 	png_c(IConsole* conHnd) : image_c(conHnd) { }
-	bool Load(const char* fileName, std::optional<size_callback_t> sizeCallback = {}) override;
-	bool Save(const char* fileName) override;
+	bool Load(std::filesystem::path const& fileName, std::optional<size_callback_t> sizeCallback = {}) override;
+	bool Save(std::filesystem::path const& fileName) override;
 };
 
 // GIF Image
 class gif_c : public image_c {
 public:
 	gif_c(IConsole* conHnd) : image_c(conHnd) { }
-	bool Load(const char* fileName, std::optional<size_callback_t> sizeCallback = {}) override;
-	bool Save(const char* fileName) override;
+	bool Load(std::filesystem::path const& fileName, std::optional<size_callback_t> sizeCallback = {}) override;
+	bool Save(std::filesystem::path const& fileName) override;
 };
 
 // DDS Image
 class dds_c : public image_c {
 public:
 	dds_c(IConsole* conHnd) : image_c(conHnd) {}
-	bool Load(const char* fileName, std::optional<size_callback_t> sizeCallback = {}) override;
-	bool Save(const char* fileName) override;
+	bool Load(std::filesystem::path const& fileName, std::optional<size_callback_t> sizeCallback = {}) override;
+	bool Save(std::filesystem::path const& fileName) override;
 };

From aa649035ba3a3d83adda9e92cc49bebfe7939061 Mon Sep 17 00:00:00 2001
From: Lars Viklund <zao@zao.se>
Date: Sun, 7 Apr 2024 21:15:14 +0200
Subject: [PATCH 10/16] feat: canonicalize script paths to Unicode paths

Script paths are now Unicode and canonicalized to remove any extraneous
"./" and "../" segments, resolving the full path, these typically arise
from how the program is lauched during development.

Most API functions that use these are updated to use the new path types.
---
 engine/system/sys_main.h       |  2 +-
 engine/system/win/sys_local.h  |  2 +-
 engine/system/win/sys_main.cpp |  8 ++---
 ui_api.cpp                     | 20 +++++------
 ui_main.cpp                    | 64 ++++++++++++++--------------------
 ui_main.h                      |  8 ++---
 6 files changed, 45 insertions(+), 59 deletions(-)

diff --git a/engine/system/sys_main.h b/engine/system/sys_main.h
index 7e3b94b..30157c1 100644
--- a/engine/system/sys_main.h
+++ b/engine/system/sys_main.h
@@ -72,8 +72,8 @@ class sys_IMain {
 	virtual bool	IsKeyDown(byte key) = 0;
 	virtual void	ClipboardCopy(const char* str) = 0;
 	virtual char*	ClipboardPaste() = 0;
-	virtual bool	SetWorkDir(const char* newCwd = NULL) = 0;
 	virtual void	SpawnProcess(const char* cmdName, const char* argList) = 0;
+	virtual bool	SetWorkDir(std::filesystem::path const& newCwd = {}) = 0;
 	virtual void	OpenURL(const char* url) = 0;
 	virtual void	Error(const char* fmt, ...) = 0;
 	virtual void	Exit(const char* msg = NULL) = 0;
diff --git a/engine/system/win/sys_local.h b/engine/system/win/sys_local.h
index 8e135c8..41a8f0d 100644
--- a/engine/system/win/sys_local.h
+++ b/engine/system/win/sys_local.h
@@ -35,8 +35,8 @@ class sys_main_c: public sys_IMain {
 	bool	IsKeyDown(byte key);
 	void	ClipboardCopy(const char* str);
 	char*	ClipboardPaste();
-	bool	SetWorkDir(const char* newCwd = NULL);
 	void	SpawnProcess(const char* cmdName, const char* argList);
+	bool	SetWorkDir(std::filesystem::path const& newCwd = {});
 	void	OpenURL(const char* url);
 	void	Error(const char* fmt, ...);
 	void	Exit(const char* msg = NULL);
diff --git a/engine/system/win/sys_main.cpp b/engine/system/win/sys_main.cpp
index 1d80da7..3af2e64 100644
--- a/engine/system/win/sys_main.cpp
+++ b/engine/system/win/sys_main.cpp
@@ -380,7 +380,7 @@ char* sys_main_c::ClipboardPaste()
 	return AllocString(glfwGetClipboardString(nullptr));
 }
 
-bool sys_main_c::SetWorkDir(const char* newCwd)
+bool sys_main_c::SetWorkDir(std::filesystem::path const& newCwd)
 {
 #ifdef _WIN32
 	auto changeDir = [](std::filesystem::path const& p) {
@@ -391,10 +391,10 @@ bool sys_main_c::SetWorkDir(const char* newCwd)
 		return _chdir(p.c_str());
 	};
 #endif
-	if (newCwd) {
-		return changeDir(newCwd) != 0;
+	if (newCwd.empty()) {
+		return changeDir(basePath) != 0;
 	} else {
-		return changeDir(basePath.c_str()) != 0;
+		return changeDir(newCwd) != 0;
 	}
 }
 
diff --git a/ui_api.cpp b/ui_api.cpp
index 1bbf976..b9ec61a 100644
--- a/ui_api.cpp
+++ b/ui_api.cpp
@@ -1611,7 +1611,7 @@ static int l_GetTime(lua_State* L)
 static int l_GetScriptPath(lua_State* L)
 {
 	ui_main_c* ui = GetUIPtr(L);
-	lua_pushstring(L, ui->scriptPath);
+	lua_pushstring(L, ui->scriptPath.u8string().c_str());
 	return 1;
 }
 
@@ -1702,27 +1702,25 @@ static int l_RenameFile(lua_State* L)
 
 	return luaL_fileresult(L, rc == 0, srcStr);
 }
-
-static int l_SetWorkDir(lua_State* L)
+SG_LUA_CPP_FUN_BEGIN(SetWorkDir)
 {
 	ui_main_c* ui = GetUIPtr(L);
 	int n = lua_gettop(L);
-	ui->LAssert(L, n >= 1, "Usage: SetWorkDir(path)");
-	ui->LAssert(L, lua_isstring(L, 1), "SetWorkDir() argument 1: expected string, got %s", luaL_typename(L, 1));
-	const char* newWorkDir = lua_tostring(L, 1);
+	ui->LExpect(L, n >= 1, "Usage: SetWorkDir(path)");
+	ui->LExpect(L, lua_isstring(L, 1), "SetWorkDir() argument 1: expected string, got %s", luaL_typename(L, 1));
+	auto newWorkDir = std::filesystem::u8path(lua_tostring(L, 1));
+
 	if (!ui->sys->SetWorkDir(newWorkDir)) {
-		if (ui->scriptWorkDir) {
-			FreeString(ui->scriptWorkDir);
-		}
-		ui->scriptWorkDir = AllocString(newWorkDir);
+		ui->scriptWorkDir = newWorkDir;
 	}
 	return 0;
 }
+SG_LUA_CPP_FUN_END()
 
 static int l_GetWorkDir(lua_State* L)
 {
 	ui_main_c* ui = GetUIPtr(L);
-	lua_pushstring(L, ui->scriptWorkDir);
+	lua_pushstring(L, ui->scriptWorkDir.u8string().c_str());
 	return 1;
 }
 
diff --git a/ui_main.cpp b/ui_main.cpp
index 79a1f05..8dbaec4 100644
--- a/ui_main.cpp
+++ b/ui_main.cpp
@@ -152,7 +152,7 @@ void ui_main_c::PCall(int narg, int nret)
 	inLua = true;
 	int err = lua_pcall(L, narg, nret, 1);
 	inLua = false;
-	sys->SetWorkDir(NULL);
+	sys->SetWorkDir();
 	if (err && !didExit) {
 		DoError("Runtime error in", lua_tostring(L, -1));
 	}
@@ -160,8 +160,9 @@ void ui_main_c::PCall(int narg, int nret)
 
 void ui_main_c::DoError(const char* msg, const char* error)
 {
-	char* errText = AllocStringLen(strlen(msg) + strlen(scriptName) + strlen(error) + 30);
-	sprintf(errText, "--- SCRIPT ERROR ---\n%s '%s':\n%s\n", msg, scriptName, error);
+	auto scriptStr = scriptName.u8string();
+	char* errText = AllocStringLen(strlen(msg) + scriptStr.size() + strlen(error) + 30);
+	sprintf(errText, "--- SCRIPT ERROR ---\n%s '%s':\n%s\n", msg, scriptStr.c_str(), error);
 	sys->Exit(errText);
 	FreeString(errText);
 	didExit = true;
@@ -203,29 +204,19 @@ static int l_panicFunc(lua_State* L)
 void ui_main_c::Init(int argc, char** argv)
 {
 	// Find paths
-	scriptName = AllocString(argv[0]);
-	scriptCfg = AllocStringLen(strlen(scriptName) + 4);
-	strcpy(scriptCfg, scriptName);
-	char* ext = strrchr(scriptCfg, '.');
-	if (ext) {
-		strcpy(ext, ".cfg");
-	} else {
-		strcat(scriptCfg, ".cfg");
-	}
-	char tmpDir[512];
-	strcpy(tmpDir, scriptName);
-	char* ls = strrchr(tmpDir, '\\');
-	if ( !ls ) {
-		ls = strrchr(tmpDir, '/');
-	}
-	if (ls) {
-		*ls = 0;
-		scriptPath = AllocString(tmpDir);
-		scriptWorkDir = AllocString(tmpDir);
-	} else {
-		scriptPath = AllocString(sys->basePath.u8string().c_str());
-		scriptWorkDir = AllocString(sys->basePath.u8string().c_str());
+	scriptName = std::filesystem::u8path(argv[0]);
+	if (scriptName.is_relative()) {
+		scriptName = sys->basePath / scriptName;
 	}
+	scriptName = canonical(scriptName);
+
+	scriptCfg = scriptName;
+	scriptCfg.replace_extension(".cfg");
+
+	auto scriptParent = scriptName.parent_path();
+	scriptPath = scriptParent;
+	scriptWorkDir = scriptParent;
+
 	scriptArgc = argc;
 	scriptArgv = new char*[argc];
 	for (int a = 0; a < argc; a++) {
@@ -236,8 +227,7 @@ void ui_main_c::Init(int argc, char** argv)
 	core->config->LoadConfig("SimpleGraphic/SimpleGraphic.cfg");
 	core->config->LoadConfig("SimpleGraphic/SimpleGraphicAuto.cfg");
 	if (core->config->LoadConfig(scriptCfg)) {
-		FreeString(scriptCfg);
-		scriptCfg = NULL;
+		scriptCfg.clear();
 	}
 
 	// Initialise script
@@ -254,7 +244,7 @@ void ui_main_c::RenderInit(r_featureFlag_e features)
 		return;
 	}
 
-	sys->SetWorkDir(NULL);
+	sys->SetWorkDir();
 
 	sys->con->ExecCommands(true);
 
@@ -276,11 +266,11 @@ void ui_main_c::ScriptInit()
 {
 	sys->con->PrintFunc("UI Init");
 
-	sys->con->Printf("Script: %s\n", scriptName);
-	if (scriptPath) {
-		sys->con->Printf("Script working directory: %s\n", scriptWorkDir);
+	sys->con->Printf("Script: %s\n", scriptName.u8string().c_str());
+	if (!scriptPath.empty()) {
+		sys->con->Printf("Script working directory: %s\n", scriptWorkDir.u8string().c_str());
 	}
-	sys->video->SetTitle(scriptName);
+	sys->video->SetTitle(scriptName.u8string().c_str());
 
 	restartFlag = false;
 	didExit = false;
@@ -322,11 +312,13 @@ void ui_main_c::ScriptInit()
 	}
 	
 	// Load the script file
- 	err = luaL_loadfile(L, scriptName);
+	sys->SetWorkDir(scriptWorkDir);
+ 	err = luaL_loadfile(L, scriptName.filename().u8string().c_str());
 	if (err) {
 		DoError("Error loading", lua_tostring(L, -1));
 		return;
 	}
+	sys->SetWorkDir();
 
 	// Run the script
 	sys->con->Printf("Running script...\n");
@@ -467,16 +459,12 @@ void ui_main_c::Shutdown()
 	}
 
 	// Save config
-	if (scriptCfg) {
+	if (!scriptCfg.empty()) {
 		core->config->SaveConfig(scriptCfg);
 	} else {
 		core->config->SaveConfig("SimpleGraphic/SimpleGraphic.cfg");
 	}
 
-	FreeString(scriptName);
-	FreeString(scriptCfg);
-	FreeString(scriptPath);
-	FreeString(scriptWorkDir);
 	for (int a = 0; a < scriptArgc; a++) {
 		FreeString(scriptArgv[a]);
 	}
diff --git a/ui_main.h b/ui_main.h
index ce8d36e..321513b 100644
--- a/ui_main.h
+++ b/ui_main.h
@@ -36,10 +36,10 @@ class ui_main_c: public ui_IMain {
 
 	std::optional<sol::state> solState;
 	lua_State* L = nullptr;
-	char*	scriptName = nullptr;
-	char*	scriptCfg = nullptr;
-	char*	scriptPath = nullptr;
-	char*	scriptWorkDir = nullptr;
+	std::filesystem::path scriptName;
+	std::filesystem::path scriptCfg;
+	std::filesystem::path scriptPath;
+	std::filesystem::path scriptWorkDir;
 	int		scriptArgc = 0;
 	char**	scriptArgv = nullptr;
 	bool	restartFlag = false;

From 99be82dc4b40f103607acb3ce2e75a709d253652 Mon Sep 17 00:00:00 2001
From: Lars Viklund <zao@zao.se>
Date: Sun, 7 Apr 2024 21:28:34 +0200
Subject: [PATCH 11/16] feat: revise config paths and various utilities

Use modern paths in the configuration system as well as in utilities
like subprocess spawning, cloud provider info and file search.
---
 engine/core/core_config.cpp    | 30 +++++------
 engine/core/core_config.h      |  4 +-
 engine/system/sys_main.h       |  9 ++--
 engine/system/win/sys_local.h  |  2 +-
 engine/system/win/sys_main.cpp | 69 +++++++++++++------------
 ui_api.cpp                     | 94 +++++++++++++++++++---------------
 6 files changed, 109 insertions(+), 99 deletions(-)

diff --git a/engine/core/core_config.cpp b/engine/core/core_config.cpp
index 4618d4f..a073811 100644
--- a/engine/core/core_config.cpp
+++ b/engine/core/core_config.cpp
@@ -18,8 +18,8 @@
 class core_config_c: public core_IConfig, public conPrintHook_c, public conCmdHandler_c {
 public:
 	// Interface
-	bool	LoadConfig(const char* cfgName);
-	bool	SaveConfig(const char* cfgName);
+	bool	LoadConfig(std::filesystem::path const& cfgName);
+	bool	SaveConfig(std::filesystem::path const& cfgName);
 
 	// Encapsulated
 	core_config_c(sys_IMain* sysHnd);
@@ -152,19 +152,17 @@ void core_config_c::C_MemReport(IConsole* conHnd, args_c &args)
 // Config Files
 // ============
 
-bool core_config_c::LoadConfig(const char* cfgName)
+bool core_config_c::LoadConfig(std::filesystem::path const& cfgName)
 {
 	// Make sure it has .cfg extension
-	std::string fileName = cfgName;
-	if (fileName.find('.') == fileName.npos) {
-		fileName += ".cfg";
-	}
+	auto fileName = cfgName;
+	fileName.replace_extension(".cfg");
 
-	sys->con->Print(fmt::format("Executing {}\n", fileName).c_str());
+	sys->con->Print(fmt::format("Executing {}\n", fileName.u8string()).c_str());
 
 	// Read the config file
 	fileInputStream_c f;
-	if (f.FileOpen(fileName.c_str(), true)) {
+	if (f.FileOpen(fileName, true)) {
 		sys->con->Warning("config file not found");
 		return true;
 	}
@@ -198,19 +196,17 @@ bool core_config_c::LoadConfig(const char* cfgName)
 	return false;
 }
 
-bool core_config_c::SaveConfig(const char* cfgName)
+bool core_config_c::SaveConfig(std::filesystem::path const& cfgName)
 {
 	// Make sure it has .cfg extension
-	std::string fileName = cfgName;
-	if (fileName.find('.') == fileName.npos) {
-		fileName += ".cfg";
-	}
+	auto fileName = cfgName;
+	fileName.replace_extension(".cfg");
 
-	sys->con->Print(fmt::format("Saving {}\n", fileName).c_str());
+	sys->con->Print(fmt::format("Saving {}\n", fileName.u8string()).c_str());
 
 	// Open the config file
 	fileOutputStream_c f;
-	if (f.FileOpen(fileName.c_str(), false)) {
+	if (f.FileOpen(fileName, false)) {
 		sys->con->Warning("couldnt write config file");
 		return true;
 	}
@@ -235,7 +231,7 @@ void core_config_c::ConPrintHook(const char* text)
 {
 	if (con_log->intVal) {
 		if (logOpen == false) {
-			logFile.FileOpen(CFG_LOGFILE, false);
+			logFile.FileOpen(std::filesystem::u8path(CFG_LOGFILE), false);
 			logOpen = true;
 			logFile.FilePrintf("Log opened.\n");
 		}
diff --git a/engine/core/core_config.h b/engine/core/core_config.h
index da2eefc..8fcb117 100644
--- a/engine/core/core_config.h
+++ b/engine/core/core_config.h
@@ -14,6 +14,6 @@ class core_IConfig {
 	static core_IConfig* GetHandle(sys_IMain* sysHnd);
 	static void FreeHandle(core_IConfig* hnd);
 	
-	virtual bool	LoadConfig(const char* cfgName) = 0;
-	virtual bool	SaveConfig(const char* cfgName) = 0;
+	virtual bool	LoadConfig(std::filesystem::path const& cfgName) = 0;
+	virtual bool	SaveConfig(std::filesystem::path const& cfgName) = 0;
 };
diff --git a/engine/system/sys_main.h b/engine/system/sys_main.h
index 30157c1..a716871 100644
--- a/engine/system/sys_main.h
+++ b/engine/system/sys_main.h
@@ -5,6 +5,7 @@
 //
 
 #include <filesystem>
+#include <optional>
 #include <string>
 
 // =======
@@ -35,17 +36,17 @@ class thread_c {
 // File finder
 class find_c {
 public:
-	std::string fileName;
+	std::filesystem::path fileName;
 	bool	isDirectory = false;
 	uintmax_t	fileSize = 0;
 	unsigned long long modified = 0;
 
 	find_c();
 	~find_c();
-	bool	FindFirst(const char* fileSpec);
+	bool	FindFirst(std::filesystem::path const&& fileSpec);
 	bool	FindNext();
 private:
-	std::filesystem::path glob;
+	std::optional<std::string> globPattern; // Empty pattern accepts all files like "*" and "*.*"
 	std::filesystem::directory_iterator iter;
 };
 
@@ -72,8 +73,8 @@ class sys_IMain {
 	virtual bool	IsKeyDown(byte key) = 0;
 	virtual void	ClipboardCopy(const char* str) = 0;
 	virtual char*	ClipboardPaste() = 0;
-	virtual void	SpawnProcess(const char* cmdName, const char* argList) = 0;
 	virtual bool	SetWorkDir(std::filesystem::path const& newCwd = {}) = 0;
+	virtual void	SpawnProcess(std::filesystem::path cmdName, const char* argList) = 0;
 	virtual void	OpenURL(const char* url) = 0;
 	virtual void	Error(const char* fmt, ...) = 0;
 	virtual void	Exit(const char* msg = NULL) = 0;
diff --git a/engine/system/win/sys_local.h b/engine/system/win/sys_local.h
index 41a8f0d..ded5b04 100644
--- a/engine/system/win/sys_local.h
+++ b/engine/system/win/sys_local.h
@@ -35,8 +35,8 @@ class sys_main_c: public sys_IMain {
 	bool	IsKeyDown(byte key);
 	void	ClipboardCopy(const char* str);
 	char*	ClipboardPaste();
-	void	SpawnProcess(const char* cmdName, const char* argList);
 	bool	SetWorkDir(std::filesystem::path const& newCwd = {});
+	void	SpawnProcess(std::filesystem::path cmdName, const char* argList);
 	void	OpenURL(const char* url);
 	void	Error(const char* fmt, ...);
 	void	Exit(const char* msg = NULL);
diff --git a/engine/system/win/sys_main.cpp b/engine/system/win/sys_main.cpp
index 3af2e64..14ae434 100644
--- a/engine/system/win/sys_main.cpp
+++ b/engine/system/win/sys_main.cpp
@@ -136,14 +136,14 @@ find_c::find_c()
 find_c::~find_c()
 {}
 
-bool GlobMatch(const std::filesystem::path& glob, const std::filesystem::path& file)
+std::optional<std::string> BuildGlobPattern(std::filesystem::path const& glob)
 {
 	using namespace std::literals::string_view_literals;
-	auto globStr = glob.generic_u8string();
+	auto globStr = glob.u8string();
 
 	// Deal with traditional "everything" wildcards.
 	if (glob == "*" || glob == "*.*") {
-		return true;
+		return {};
 	}
 
 	fmt::memory_buffer buf;
@@ -179,32 +179,34 @@ bool GlobMatch(const std::filesystem::path& glob, const std::filesystem::path& f
 			}
 		}
 	}
+	return to_string(buf);
+}
 
+bool GlobMatch(std::optional<std::string> const& globPattern, std::filesystem::path const& file)
+{
+	if (!globPattern.has_value()) {
+		// Empty pattern is like "*" and "*.*".
+		return true;
+	}
 	// Assume case-insensitive comparisons are desired.
 	RE2::Options reOpts;
 	reOpts.set_case_sensitive(false);
-	RE2 reGlob{to_string(buf), reOpts};
+	RE2 reGlob{globPattern.value(), reOpts};
 
-	auto fileStr = file.generic_u8string();
+	auto fileStr = file.u8string();
 	return RE2::FullMatch(fileStr, reGlob);
 }
 
-bool find_c::FindFirst(const char* fileSpec)
+bool find_c::FindFirst(std::filesystem::path const&& fileSpec)
 {
-#ifdef _WIN32
-	wchar_t* wideSpec = WidenUTF8String(fileSpec);
-	std::filesystem::path p(wideSpec);
-	FreeWideString(wideSpec);
-#else
-	std::filesystem::path p(fileSpec);
-#endif
-	glob = p.filename();
+	auto parent = fileSpec.parent_path();
+	globPattern = BuildGlobPattern(fileSpec.filename());
 
 	std::error_code ec;
-	for (iter = std::filesystem::directory_iterator(p.parent_path(), ec); iter != std::filesystem::directory_iterator{}; ++iter) {
+	for (iter = std::filesystem::directory_iterator(parent, ec); iter != std::filesystem::directory_iterator{}; ++iter) {
 		auto candFilename = iter->path().filename();
-		if (GlobMatch(glob, candFilename)) {
-			fileName = candFilename.u8string();
+		if (GlobMatch(globPattern, candFilename)) {
+			fileName = candFilename;
 			isDirectory = iter->is_directory();
 			fileSize = iter->file_size();
 			auto mod = iter->last_write_time();
@@ -223,8 +225,8 @@ bool find_c::FindNext()
 
 	for (++iter; iter != std::filesystem::directory_iterator{}; ++iter) {
 		auto candFilename = iter->path().filename();
-		if (GlobMatch(glob, candFilename)) {
-			fileName = candFilename.u8string();
+		if (GlobMatch(globPattern, candFilename)) {
+			fileName = candFilename;
 			isDirectory = iter->is_directory();
 			fileSize = iter->file_size();
 			auto mod = iter->last_write_time();
@@ -398,26 +400,27 @@ bool sys_main_c::SetWorkDir(std::filesystem::path const& newCwd)
 	}
 }
 
-void sys_main_c::SpawnProcess(const char* cmdName, const char* argList)
+void sys_main_c::SpawnProcess(std::filesystem::path cmdName, const char* argList)
 {
 #ifdef _WIN32
-	char cmd[512];
-	strcpy(cmd, cmdName);
-	if ( !strchr(cmd, '.') ) {
-		strcat(cmd, ".exe");
+	if (!cmdName.has_extension()) {
+		cmdName.replace_extension(".exe");
 	}
-	SHELLEXECUTEINFOA sinfo;
-	memset(&sinfo, 0, sizeof(SHELLEXECUTEINFOA));
-	sinfo.cbSize       = sizeof(SHELLEXECUTEINFOA);
+	auto fileStr = cmdName.wstring();
+	auto wideArgs = WidenUTF8String(argList);
+	SHELLEXECUTEINFOW sinfo;
+	memset(&sinfo, 0, sizeof(sinfo));
+	sinfo.cbSize       = sizeof(sinfo);
 	sinfo.fMask        = SEE_MASK_NOCLOSEPROCESS;
-	sinfo.lpFile       = cmd;
-	sinfo.lpParameters = argList;
-	sinfo.lpVerb       = "open";
+	sinfo.lpFile       = fileStr.c_str();
+	sinfo.lpParameters = wideArgs;
+	sinfo.lpVerb       = L"open";
 	sinfo.nShow        = SW_SHOWMAXIMIZED;
-	if ( !ShellExecuteExA(&sinfo) ) {
-		sinfo.lpVerb = "runas";
-		ShellExecuteExA(&sinfo);
+	if ( !ShellExecuteExW(&sinfo) ) {
+		sinfo.lpVerb = L"runas";
+		ShellExecuteExW(&sinfo);
 	}
+	FreeWideString(wideArgs);
 #else
 #warning LV: Subprocesses not implemented on this OS.
 	// TODO(LV): Implement subprocesses for other OSes.
diff --git a/ui_api.cpp b/ui_api.cpp
index b9ec61a..2d8b74d 100644
--- a/ui_api.cpp
+++ b/ui_api.cpp
@@ -1199,7 +1199,14 @@ static int l_OpenFile(lua_State* L)
 	return lua_gettop(L) - 2;
 #else
 	wchar_t* widePath = WidenUTF8String(lua_tostring(L, 1));
+	if (!widePath) {
+		return 0;
+	}
 	wchar_t* wideMode = WidenUTF8String(lua_tostring(L, 2));
+	if (!wideMode) {
+		FreeWideString(widePath);
+		return 0;
+	}
 	FILE* fp = _wfopen(widePath, wideMode);
 	FreeWideString(widePath);
 	FreeWideString(wideMode);
@@ -1252,19 +1259,20 @@ struct searchHandle_s {
 	bool	dirOnly;
 };
 
-static int l_NewFileSearch(lua_State* L)
+SG_LUA_CPP_FUN_BEGIN(NewFileSearch)
 {
 	ui_main_c* ui = GetUIPtr(L);
 	int n = lua_gettop(L);
-	ui->LAssert(L, n >= 1, "Usage: NewFileSearch(spec[, findDirectories])");
-	ui->LAssert(L, lua_isstring(L, 1), "NewFileSearch() argument 1: expected string, got %s", luaL_typename(L, 1));
+	ui->LExpect(L, n >= 1, "Usage: NewFileSearch(spec[, findDirectories])");
+	ui->LExpect(L, lua_isstring(L, 1), "NewFileSearch() argument 1: expected string, got %s", luaL_typename(L, 1));
 	find_c* find = new find_c();
-	if (!find->FindFirst(lua_tostring(L, 1))) {
+	auto path = std::filesystem::u8path(lua_tostring(L, 1));
+	if (!find->FindFirst(std::move(path))) {
 		delete find;
 		return 0;
 	}
 	bool dirOnly = lua_toboolean(L, 2) != 0;
-	while (find->isDirectory != dirOnly || find->fileName == "." || find->fileName == "..") {
+	while (find->isDirectory != dirOnly || find->fileName.filename() == "." || find->fileName.filename() == "..") {
 		if (!find->FindNext()) {
 			delete find;
 			return 0;
@@ -1277,6 +1285,7 @@ static int l_NewFileSearch(lua_State* L)
 	lua_setmetatable(L, -2);
 	return 1;
 }
+SG_LUA_CPP_FUN_END()
 
 static searchHandle_s* GetSearchHandle(lua_State* L, ui_main_c* ui, const char* method, bool valid)
 {
@@ -1301,13 +1310,14 @@ static int l_searchHandleNextFile(lua_State* L)
 {
 	ui_main_c* ui = GetUIPtr(L);
 	searchHandle_s* searchHandle = GetSearchHandle(L, ui, "NextFile", true);
+	auto &find = searchHandle->find;
 	do {
-		if (!searchHandle->find->FindNext()) {
-			delete searchHandle->find;
-			searchHandle->find = NULL;
+		if (!find->FindNext()) {
+			delete find;
+			find = NULL;
 			return 0;
 		}
-	} while (searchHandle->find->isDirectory != searchHandle->dirOnly || searchHandle->find->fileName == "." || searchHandle->find->fileName == "..");
+	} while (find->isDirectory != searchHandle->dirOnly || find->fileName.filename() == "." || find->fileName.filename() == "..");
 	lua_pushboolean(L, 1);
 	return 1;
 }
@@ -1316,7 +1326,7 @@ static int l_searchHandleGetFileName(lua_State* L)
 {
 	ui_main_c* ui = GetUIPtr(L);
 	searchHandle_s* searchHandle = GetSearchHandle(L, ui, "GetFileName", true);
-	lua_pushstring(L, searchHandle->find->fileName.c_str());
+	lua_pushstring(L, searchHandle->find->fileName.u8string().c_str());
 	return 1;
 }
 
@@ -1351,9 +1361,9 @@ struct CloudProviderInfo {
 #include <cfapi.h>
 
 static std::string NarrowString(std::wstring_view ws) {
-	auto cb = WideCharToMultiByte(CP_UTF8, 0, ws.data(), ws.size(), nullptr, 0, nullptr, nullptr);
+	auto cb = WideCharToMultiByte(CP_UTF8, 0, ws.data(), (int)ws.size(), nullptr, 0, nullptr, nullptr);
 	std::string ret(cb, '\0');
-	WideCharToMultiByte(CP_UTF8, 0, ws.data(), ws.size(), ret.data(), ret.size(), nullptr, nullptr);
+	WideCharToMultiByte(CP_UTF8, 0, ws.data(), (int)ws.size(), ret.data(), (int)ret.size(), nullptr, nullptr);
 	return ret;
 }
 
@@ -1387,7 +1397,7 @@ static std::optional<CloudProviderInfo> GetCloudProviderInfo(std::filesystem::pa
 	if (!lib.Loaded()) {
 		return {};
 	}
-	hr = lib.CfGetSyncRootInfoByPath(path.generic_wstring().c_str(), CF_SYNC_ROOT_INFO_PROVIDER, buf.data(), buf.size(), &len);
+	hr = lib.CfGetSyncRootInfoByPath(path.generic_wstring().c_str(), CF_SYNC_ROOT_INFO_PROVIDER, buf.data(), (DWORD)buf.size(), &len);
 	if (FAILED(hr) && GetLastError() != ERROR_MORE_DATA) {
 		return {};
 	}
@@ -1415,7 +1425,8 @@ static int l_GetCloudProvider(lua_State* L) {
 	ui->LAssert(L, n >= 1, "Usage: GetCloudProvider(path)");
 	ui->LAssert(L, lua_isstring(L, 1), "GetCloudProvider() argument 1: expected string, got %s", luaL_typename(L, 1));
 
-	auto info = GetCloudProviderInfo(lua_tostring(L, 1));
+	auto path = std::filesystem::u8path(lua_tostring(L, 1));
+	auto info = GetCloudProviderInfo(path);
 	if (info) {
 		lua_pushstring(L, info->name.c_str());
 		lua_pushstring(L, info->version.c_str());
@@ -1677,9 +1688,8 @@ static int l_RemoveFile(lua_State* L)
 {
 	char const* pathStr = luaL_checkstring(L, 1);
 #ifdef _WIN32
-	wchar_t* widePath = WidenUTF8String(pathStr);
-	int rc = _wremove(widePath);
-	FreeWideString(widePath);
+	auto path = std::filesystem::u8path(pathStr);
+	int rc = _wremove(path.c_str());
 #else
 	int rc = remove(pathStr);
 #endif
@@ -1691,15 +1701,12 @@ static int l_RenameFile(lua_State* L)
 	char const* srcStr = luaL_checkstring(L, 1);
 	char const* dstStr = luaL_checkstring(L, 2);
 #ifdef _WIN32
-	wchar_t* wideSrc = WidenUTF8String(srcStr);
-	wchar_t* wideDst = WidenUTF8String(dstStr);
-	int rc = _wrename(wideSrc, wideDst);
-	FreeWideString(wideSrc);
-	FreeWideString(wideDst);
+	auto srcPath = std::filesystem::u8path(srcStr);
+	auto dstPath = std::filesystem::u8path(dstStr);
+	int rc = _wrename(srcPath.c_str(), dstPath.c_str());
 #else
 	int rc = rename(srcStr, dstStr);
 #endif
-
 	return luaL_fileresult(L, rc == 0, srcStr);
 }
 SG_LUA_CPP_FUN_BEGIN(SetWorkDir)
@@ -1787,46 +1794,46 @@ static int l_IsSubScriptRunning(lua_State* L)
 	return 1;
 }
 
-static int l_LoadModule(lua_State* L)
+SG_LUA_CPP_FUN_BEGIN(LoadModule)
 {
 	ui_main_c* ui = GetUIPtr(L);
 	int n = lua_gettop(L);
-	ui->LAssert(L, n >= 1, "Usage: LoadModule(name[, ...])");
-	ui->LAssert(L, lua_isstring(L, 1), "LoadModule() argument 1: expected string, got %s", luaL_typename(L, 1));
+	ui->LExpect(L, n >= 1, "Usage: LoadModule(name[, ...])");
+	ui->LExpect(L, lua_isstring(L, 1), "LoadModule() argument 1: expected string, got %s", luaL_typename(L, 1));
 	const char* modName = lua_tostring(L, 1);
-	char fileName[1024];
-	strcpy(fileName, modName);
-	if (!strchr(fileName, '.')) {
-		strcat(fileName, ".lua");
+	auto fileName = std::filesystem::u8path(modName);
+	if (!fileName.has_extension()) {
+		fileName.replace_extension(".lua");
 	}
+
 	ui->sys->SetWorkDir(ui->scriptPath);
-	int err = luaL_loadfile(L, fileName);
+	int err = luaL_loadfile(L, fileName.u8string().c_str());
 	ui->sys->SetWorkDir(ui->scriptWorkDir);
-	ui->LAssert(L, err == 0, "LoadModule() error loading '%s' (%d):\n%s", fileName, err, lua_tostring(L, -1));
+	ui->LExpect(L, err == 0, "LoadModule() error loading '%s' (%d):\n%s", fileName, err, lua_tostring(L, -1));
 	lua_replace(L, 1);	// Replace module name with module main chunk
 	lua_call(L, n - 1, LUA_MULTRET);
 	return lua_gettop(L);
 }
+SG_LUA_CPP_FUN_END()
 
-static int l_PLoadModule(lua_State* L)
+SG_LUA_CPP_FUN_BEGIN(PLoadModule)
 {
 	ui_main_c* ui = GetUIPtr(L);
 	int n = lua_gettop(L);
-	ui->LAssert(L, n >= 1, "Usage: PLoadModule(name[, ...])");
-	ui->LAssert(L, lua_isstring(L, 1), "PLoadModule() argument 1: expected string, got %s", luaL_typename(L, 1));
+	ui->LExpect(L, n >= 1, "Usage: PLoadModule(name[, ...])");
+	ui->LExpect(L, lua_isstring(L, 1), "PLoadModule() argument 1: expected string, got %s", luaL_typename(L, 1));
 	const char* modName = lua_tostring(L, 1);
-	char* fileName = AllocStringLen(strlen(modName) + 4);
-	strcpy(fileName, modName);
-	if (!strchr(fileName, '.')) {
-		strcat(fileName, ".lua");
+	auto fileName = std::filesystem::u8path(modName);
+	if (!fileName.has_extension()) {
+		fileName.replace_extension(".lua");
 	}
+
 	ui->sys->SetWorkDir(ui->scriptPath);
-	int err = luaL_loadfile(L, fileName);
+	int err = luaL_loadfile(L, fileName.u8string().c_str());
 	ui->sys->SetWorkDir(ui->scriptWorkDir);
 	if (err) {
 		return 1;
 	}
-	FreeString(fileName);
 	lua_replace(L, 1);	// Replace module name with module main chunk
 	lua_getfield(L, LUA_REGISTRYINDEX, "traceback");
 	lua_insert(L, 1); // Insert traceback function at start of stack
@@ -1838,6 +1845,7 @@ static int l_PLoadModule(lua_State* L)
 	lua_replace(L, 1); // Replace traceback function with nil
 	return lua_gettop(L);
 }
+SG_LUA_CPP_FUN_END()
 
 static int l_PCall(lua_State* L)
 {
@@ -1982,7 +1990,9 @@ static int l_SpawnProcess(lua_State* L)
 	int n = lua_gettop(L);
 	ui->LAssert(L, n >= 1, "Usage: SpawnProcess(cmdName[, args])");
 	ui->LAssert(L, lua_isstring(L, 1), "SpawnProcess() argument 1: expected string, got %s", luaL_typename(L, 1));
-	ui->sys->SpawnProcess(lua_tostring(L, 1), lua_tostring(L, 2));
+	auto cmdPath = std::filesystem::u8path(lua_tostring(L, 1));
+	auto args = lua_tostring(L, 2);
+	ui->sys->SpawnProcess(cmdPath, args);
 	return 0;
 }
 

From 7357cb05f5f732f0fa6c17376852fa5246910f7e Mon Sep 17 00:00:00 2001
From: Lars Viklund <zao@zao.se>
Date: Sun, 7 Apr 2024 21:35:25 +0200
Subject: [PATCH 12/16] feat: use paths and string keys in texture library

The texture library needs string keys to identify "shaders", they remain
narrow UTF-8 as all image paths in use are ASCII and while there's now
proper path use for texture loading it doesn't need to permeate quite
that far into that kind of performance-sensitive place.
---
 engine/render.h             |  2 +-
 engine/render/r_main.cpp    | 45 +++++++++++++++++--------------------
 engine/render/r_main.h      |  2 +-
 engine/render/r_texture.cpp | 14 ++++++------
 engine/render/r_texture.h   |  4 ++--
 ui_api.cpp                  | 32 ++++++++++++--------------
 6 files changed, 46 insertions(+), 53 deletions(-)

diff --git a/engine/render.h b/engine/render.h
index d32b9c0..9eb2c5d 100644
--- a/engine/render.h
+++ b/engine/render.h
@@ -77,7 +77,7 @@ class r_IRenderer {
 	virtual void	BeginFrame() = 0;
 	virtual void	EndFrame() = 0;
 	
-	virtual r_shaderHnd_c* RegisterShader(const char* name, int flags) = 0;
+	virtual r_shaderHnd_c* RegisterShader(std::string_view name, int flags) = 0;
 	virtual r_shaderHnd_c* RegisterShaderFromImage(std::unique_ptr<image_c> img, int flags) = 0;
 	virtual void	GetShaderImageSize(r_shaderHnd_c* hnd, int &width, int &height) = 0;
 	virtual void	SetShaderLoadingPriority(r_shaderHnd_c* hnd, int pri) = 0;
diff --git a/engine/render/r_main.cpp b/engine/render/r_main.cpp
index de1a277..e286b03 100644
--- a/engine/render/r_main.cpp
+++ b/engine/render/r_main.cpp
@@ -45,40 +45,39 @@ enum r_takeScreenshot_e {
 class r_shader_c {
 public:
 	r_renderer_c* renderer;
-	char* name;
+	std::string name;
 	dword		nameHash;
 	int			refCount;
 	r_tex_c* tex;
 
-	r_shader_c(r_renderer_c* renderer, const char* shname, int flags);
-	r_shader_c(r_renderer_c* renderer, const char* shname, int flags, std::unique_ptr<image_c> img);
+	r_shader_c(r_renderer_c* renderer, std::string_view shname, int flags);
+	r_shader_c(r_renderer_c* renderer, std::string_view shname, int flags, std::unique_ptr<image_c> img);
 	~r_shader_c();
 };
 
-r_shader_c::r_shader_c(r_renderer_c* renderer, const char* shname, int flags)
+r_shader_c::r_shader_c(r_renderer_c* renderer, std::string_view shname, int flags)
 	: renderer(renderer)
 {
-	name = AllocString(shname);
-	nameHash = StringHash(shname, 0xFFFF);
+	name = shname;
+	nameHash = StringHash(name.c_str(), 0xFFFF);
 	refCount = 0;
 	tex = new r_tex_c(renderer->texMan, name, flags);
 	if (tex->error) {
-		renderer->sys->con->Warning("couldn't load texture '%s'", name);
+		renderer->sys->con->Warning("couldn't load texture '%s'", name.c_str());
 	}
 }
 
-r_shader_c::r_shader_c(r_renderer_c* renderer, const char* shname, int flags, std::unique_ptr<image_c> img)
+r_shader_c::r_shader_c(r_renderer_c* renderer, std::string_view shname, int flags, std::unique_ptr<image_c> img)
 	: renderer(renderer)
 {
-	name = AllocString(shname);
-	nameHash = StringHash(shname, 0xFFFF);
+	name = shname;
+	nameHash = StringHash(name.c_str(), 0xFFFF);
 	refCount = 0;
 	tex = new r_tex_c(renderer->texMan, std::move(img), flags);
 }
 
 r_shader_c::~r_shader_c()
 {
-	FreeString(name);
 	delete tex;
 }
 
@@ -1570,19 +1569,20 @@ void r_renderer_c::PurgeShaders()
 	}
 }
 
-r_shaderHnd_c* r_renderer_c::RegisterShader(const char* shname, int flags)
+r_shaderHnd_c* r_renderer_c::RegisterShader(std::string_view shname, int flags)
 {
-	if (*shname == 0) {
+	if (shname.empty()) {
 		return NULL;
 	}
 
-	dword nameHash = StringHash(shname, 0xFFFF);
+	std::string name(shname);
+	dword nameHash = StringHash(name, 0xFFFF);
 	int newId = -1;
 	for (int s = 0; s < numShader; s++) {
 		if (!shaderList[s]) {
 			newId = s;
 		}
-		else if (shaderList[s]->nameHash == nameHash && _stricmp(shname, shaderList[s]->name) == 0 && shaderList[s]->tex->flags == flags) {
+		else if (shaderList[s]->nameHash == nameHash && _stricmp(name.c_str(), shaderList[s]->name.c_str()) == 0 && shaderList[s]->tex->flags == flags) {
 			// Shader already exists, return a new handle for it
 			// Ensure texture is loaded as soon as possible
 			shaderList[s]->tex->ForceLoad();
@@ -1930,7 +1930,7 @@ void r_renderer_c::DoScreenshot(image_c* i, int type, const char* ext)
 	// Flip and convert the image to RGB
 	int const readSpan = xs * 4;
 	int	const writeSpan = xs * 3;
-	std::vector<byte> ss(writeSize); // This is a raw pointer as ownership is taken by the image object.
+	std::vector<byte> ss(writeSize);
 	byte* p1 = sbuf.data();
 	byte* p2 = ss.data() + writeSize - writeSpan;
 	for (int y = 0; y < ys; ++y, p2 -= writeSpan * 2) {
@@ -1949,26 +1949,23 @@ void r_renderer_c::DoScreenshot(image_c* i, int type, const char* ext)
 
 	time_t curTime;
 	time(&curTime);
-	std::string ssname = fmt::format(CFG_DATAPATH "Screenshots/{:%m%d%y_%H%M%S}.{}",
-		fmt::localtime(curTime), ext);
-	// curTimeSt.tm_mon+1, curTimeSt.tm_mday, curTimeSt.tm_year%100,
-	// curTimeSt.tm_hour, curTimeSt.tm_min, curTimeSt.tm_sec, ext);
+	auto ssPath = std::filesystem::u8path(fmt::format(CFG_DATAPATH "Screenshots/{:%m%d%y_%H%M%S}.{}",
+		fmt::localtime(curTime), ext));
 
 
 	// Make folder if it doesn't exist
 	std::error_code ec;
-	std::filesystem::create_directory(CFG_DATAPATH "Screenshots", ec);
+	std::filesystem::create_directories(ssPath.parent_path(), ec);
 	if (ec) {
 		sys->con->Print("Couldn't create screenshot folder!\n");
 		return;
 	}
 
-	if (i->Save(ssname.c_str())) {
+	if (i->Save(ssPath)) {
 		sys->con->Print("Couldn't write screenshot!\n");
 		return;
 	}
-
-	sys->con->Print(fmt::format("Wrote screenshot to {}\n", ssname).c_str());
+	sys->con->Print(fmt::format("Wrote screenshot to {}\n", ssPath.u8string()).c_str());
 }
 
 r_renderer_c::RenderTarget& r_renderer_c::GetDrawRenderTarget()
diff --git a/engine/render/r_main.h b/engine/render/r_main.h
index a8eadb3..0df5a22 100644
--- a/engine/render/r_main.h
+++ b/engine/render/r_main.h
@@ -73,7 +73,7 @@ class r_renderer_c: public r_IRenderer, public conCmdHandler_c {
 	void	BeginFrame();
 	void	EndFrame();
 	
-	r_shaderHnd_c* RegisterShader(const char* shname, int flags);
+	r_shaderHnd_c* RegisterShader(std::string_view shname, int flags);
 	r_shaderHnd_c* RegisterShaderFromImage(std::unique_ptr<image_c> img, int flags);
 	void	GetShaderImageSize(r_shaderHnd_c* hnd, int &width, int &height);
 	void	SetShaderLoadingPriority(r_shaderHnd_c* hnd, int pri);
diff --git a/engine/render/r_texture.cpp b/engine/render/r_texture.cpp
index 9047da3..f7a511e 100644
--- a/engine/render/r_texture.cpp
+++ b/engine/render/r_texture.cpp
@@ -294,7 +294,7 @@ static void T_ResampleImage(byte* in, dword in_w, dword in_h, int in_comp, byte*
 // OpenGL Texture Class
 // ====================
 
-r_tex_c::r_tex_c(r_ITexManager* manager, const char* fileName, int flags)
+r_tex_c::r_tex_c(r_ITexManager* manager, std::string_view fileName, int flags)
 {
 	Init(manager, fileName, flags);
 
@@ -307,7 +307,7 @@ r_tex_c::r_tex_c(r_ITexManager* manager, const char* fileName, int flags)
 
 r_tex_c::r_tex_c(r_ITexManager* manager, std::unique_ptr<image_c> img, int flags)
 {
-	Init(manager, NULL, flags);
+	Init(manager, {}, flags);
 
 	// Direct upload
 	img = BuildMipSet(std::move(img));
@@ -322,7 +322,7 @@ r_tex_c::~r_tex_c()
 	glDeleteTextures(1, &texId);
 }
 
-void r_tex_c::Init(r_ITexManager* i_manager, const char* i_fileName, int i_flags)
+void r_tex_c::Init(r_ITexManager* i_manager, std::string_view i_fileName, int i_flags)
 {
 	manager = (t_manager_c*)i_manager;
 	renderer = manager->renderer;
@@ -331,7 +331,7 @@ void r_tex_c::Init(r_ITexManager* i_manager, const char* i_fileName, int i_flags
 	loadPri = 0;
 	texId = 0;
 	flags = i_flags;
-	fileName = i_fileName ? i_fileName : "";
+	fileName = i_fileName;
 	fileWidth = 0;
 	fileHeight = 0;
 }
@@ -482,15 +482,15 @@ void r_tex_c::LoadFile()
 	}
 
 	// Try to load image file using appropriate loader
-	std::string loadPath = fileName;
-	img = std::unique_ptr<image_c>(image_c::LoaderForFile(renderer->sys->con, loadPath.c_str()));
+	auto path = std::filesystem::u8path(fileName);
+	img = std::unique_ptr<image_c>(image_c::LoaderForFile(renderer->sys->con, path));
 	if (img) {
 		auto sizeCallback = [this](int width, int height) {
 			this->fileWidth = width;
 			this->fileHeight = height;
 			this->status = SIZE_KNOWN;
 		};
-		error = img->Load(loadPath.c_str(), sizeCallback);
+		error = img->Load(path, sizeCallback);
 		if ( !error ) {
 			stackLayers = img->tex.layers();
 			const bool is_async = !!(flags & TF_ASYNC);
diff --git a/engine/render/r_texture.h b/engine/render/r_texture.h
index 6645992..c20c8bf 100644
--- a/engine/render/r_texture.h
+++ b/engine/render/r_texture.h
@@ -39,7 +39,7 @@ class r_tex_c {
 	GLenum target{};
 	size_t stackLayers = 1;
 
-	r_tex_c(class r_ITexManager* manager, const char* fileName, int flags);
+	r_tex_c(class r_ITexManager* manager, std::string_view fileName, int flags);
 	r_tex_c(class r_ITexManager* manager, std::unique_ptr<image_c> img, int flags);
 	~r_tex_c();
 
@@ -57,7 +57,7 @@ class r_tex_c {
 private:
 	class t_manager_c* manager;
 	class r_renderer_c* renderer;
-	void	Init(class r_ITexManager* manager, const char* fileName, int flags);
+	void	Init(class r_ITexManager* manager, std::string_view fileName, int flags);
 	void	Upload(image_c& img, int flags);
 	std::unique_ptr<image_c> BuildMipSet(std::unique_ptr<image_c> img);
 };
diff --git a/ui_api.cpp b/ui_api.cpp
index 2d8b74d..f770ea3 100644
--- a/ui_api.cpp
+++ b/ui_api.cpp
@@ -299,15 +299,14 @@ SG_LUA_CPP_FUN_BEGIN(NewArtHandle)
 
 	int n = lua_gettop(L);
 	ui->LExpect(L, n >= 1, "Usage: NewArtHandle(fileName)");
-	std::filesystem::path filePath = reader.ArgToString(1);
+	std::filesystem::path filePath = std::filesystem::u8path(reader.ArgToString(1));
 	if (filePath.is_relative())
 		filePath = ui->scriptWorkDir / filePath;
-	std::string fileName = filePath.generic_string();
-	std::unique_ptr<image_c> img(image_c::LoaderForFile(ui->sys->con, fileName.c_str()));
+	std::unique_ptr<image_c> img(image_c::LoaderForFile(ui->sys->con, filePath));
 	if (!img)
 		return 0;
 
-	if (img->Load(fileName.c_str()))
+	if (img->Load(filePath))
 		return 0;
 
 	const auto format = img->tex.format();
@@ -398,13 +397,9 @@ SG_LUA_CPP_FUN_BEGIN(imgHandleLoad)
 	int n = lua_gettop(L);
 	ui->LExpect(L, n >= 1, "Usage: imgHandle:Load(fileName[, flag1[, flag2...]])");
 	ui->LExpect(L, lua_isstring(L, 1), "imgHandle:Load() argument 1: expected string, got %s", luaL_typename(L, 1));
-	const char* fileName = lua_tostring(L, 1);
-	char fullFileName[512];
-	if (strchr(fileName, ':') || !ui->scriptWorkDir) {
-		strcpy(fullFileName, fileName);
-	}
-	else {
-		sprintf(fullFileName, "%s/%s", ui->scriptWorkDir, fileName);
+	auto fileName = std::filesystem::u8path(lua_tostring(L, 1));
+	if (!fileName.is_absolute() && !ui->scriptWorkDir.empty()) {
+		fileName = ui->scriptWorkDir / fileName;
 	}
 	delete imgHandle->hnd;
 	int flags = TF_NOMIPMAP;
@@ -412,24 +407,25 @@ SG_LUA_CPP_FUN_BEGIN(imgHandleLoad)
 		if (!lua_isstring(L, f)) {
 			continue;
 		}
-		const char* flag = lua_tostring(L, f);
-		if (!strcmp(flag, "ASYNC")) {
+		std::string flag = lua_tostring(L, f);
+		if (flag == "ASYNC") {
 			flags |= TF_ASYNC;
 		}
-		else if (!strcmp(flag, "CLAMP")) {
+		else if (flag == "CLAMP") {
 			flags |= TF_CLAMP;
 		}
-		else if (!strcmp(flag, "MIPMAP")) {
+		else if (flag == "MIPMAP") {
 			flags &= ~TF_NOMIPMAP;
 		}
-		else if (!strcmp(flag, "NEAREST")) {
+		else if (flag == "NEAREST") {
 			flags |= TF_NEAREST;
 		}
 		else {
-			ui->LExpect(L, 0, "imgHandle:Load(): unrecognised flag '%s'", flag);
+			ui->LExpect(L, 0, "imgHandle:Load(): unrecognised flag '%s'", flag.c_str());
 		}
 	}
-	imgHandle->hnd = ui->renderer->RegisterShader(fullFileName, flags);
+	// TODO(LV): should we use u8path throughout here, to support any callers that use paths outside of working directory?
+	imgHandle->hnd = ui->renderer->RegisterShader(fileName.u8string(), flags);
 	return 0;
 }
 SG_LUA_CPP_FUN_END()

From 3499b3d7ef3fc2e7b6effcc8451c2eaf757ff7ab Mon Sep 17 00:00:00 2001
From: Lars Viklund <zao@zao.se>
Date: Tue, 9 Apr 2024 00:55:04 +0200
Subject: [PATCH 13/16] feat: remove UTF-8 impls for Lua io/os functions

As these functions can be fixed inside of the LuaJIT interpreter itself,
we don't need these hooked into the main interpreter anymore.

A positive side effect of this is that subscripts now also can use UTF-8
in paths as the changes are universal across all LuaJIT interpreters.

The fallback regular Lua interpreter in `Update.exe` is still narrow ACP
and care needs to be made when writing update op-files and modifying
`UpdateApply.lua`
---
 ui_api.cpp | 126 -----------------------------------------------------
 1 file changed, 126 deletions(-)

diff --git a/ui_api.cpp b/ui_api.cpp
index f770ea3..15f0751 100644
--- a/ui_api.cpp
+++ b/ui_api.cpp
@@ -1170,82 +1170,6 @@ static void stackDump(lua_State* L)
 }
 #endif
 
-/* io:open replacement that opens a file with UTF-8 paths instead of the user codepage.
-*  In order to reduce implementation effort this function opens a well-known readable file
-*  with regular io:open and swaps out the internal FILE* pointer to a file opened via the
-*  Unicode-aware _wfopen on Windows.
-* 
-*  The resulting file object has all the functionality of the standard library file
-*  object due to actually being such an object.
-*/
-static int l_OpenFile(lua_State* L)
-{
-	ui_main_c* ui = GetUIPtr(L);
-	
-	int n = lua_gettop(L);
-	ui->LAssert(L, n == 2, "Usage: OpenFile(path, mode)");
-	ui->LAssert(L, lua_isstring(L, 1), "OpenFile() argument 1: expected string, got %s", luaL_typename(L, 1));
-	ui->LAssert(L, lua_isstring(L, 2), "OpenFile() argument 2: expected string, got %s", luaL_typename(L, 2));
-
-#ifndef _WIN32
-	lua_rawgeti(L, LUA_REGISTRYINDEX, ui->ioOpenf);
-	lua_pushstring(L, lua_tostring(L, 1));
-	lua_pushstring(L, lua_tostring(L, 2));
-	lua_call(L, 2, 2);
-	return lua_gettop(L) - 2;
-#else
-	wchar_t* widePath = WidenUTF8String(lua_tostring(L, 1));
-	if (!widePath) {
-		return 0;
-	}
-	wchar_t* wideMode = WidenUTF8String(lua_tostring(L, 2));
-	if (!wideMode) {
-		FreeWideString(widePath);
-		return 0;
-	}
-	FILE* fp = _wfopen(widePath, wideMode);
-	FreeWideString(widePath);
-	FreeWideString(wideMode);
-	if (!fp) {
-		return 0;
-	}
-
-	static const std::string placeholder = [] {
-		char buf[MAX_PATH]{};
-		HMODULE mod = LoadLibraryA("ntdll.dll");
-		GetModuleFileNameA(mod, buf, sizeof(buf));
-		FreeLibrary(mod);
-		return std::string(buf);
-	}();
-
-	{
-		lua_rawgeti(L, LUA_REGISTRYINDEX, ui->ioOpenf);
-		lua_pushstring(L, placeholder.c_str());
-		lua_pushstring(L, "r");
-		lua_call(L, 2, 2);
-
-
-		if (lua_isnil(L, -2)) {
-			fclose(fp);
-			ui->LAssert(L, !lua_isnil(L, -2), "OpenFile(): failed to open placeholder path %s: %s", placeholder.c_str(), luaL_checkstring(L, -1));
-		}
-	}
-
-	lua_pop(L, 1);
-
-	struct luajitInternalFileHandlePart {
-		FILE* f;
-	};
-
-	luajitInternalFileHandlePart* ljData = (luajitInternalFileHandlePart*)luaL_checkudata(L, -1, "FILE*");
-
-	fclose(ljData->f);
-	ljData->f = fp;
-
-	return 1;
-#endif
-}
-
 // ==============
 // Search Handles
 // ==============
@@ -1680,31 +1604,6 @@ static int l_RemoveDir(lua_State* L)
 	}
 }
 
-static int l_RemoveFile(lua_State* L)
-{
-	char const* pathStr = luaL_checkstring(L, 1);
-#ifdef _WIN32
-	auto path = std::filesystem::u8path(pathStr);
-	int rc = _wremove(path.c_str());
-#else
-	int rc = remove(pathStr);
-#endif
-	return luaL_fileresult(L, rc == 0, pathStr);
-}
-
-static int l_RenameFile(lua_State* L)
-{
-	char const* srcStr = luaL_checkstring(L, 1);
-	char const* dstStr = luaL_checkstring(L, 2);
-#ifdef _WIN32
-	auto srcPath = std::filesystem::u8path(srcStr);
-	auto dstPath = std::filesystem::u8path(dstStr);
-	int rc = _wrename(srcPath.c_str(), dstPath.c_str());
-#else
-	int rc = rename(srcStr, dstStr);
-#endif
-	return luaL_fileresult(L, rc == 0, srcStr);
-}
 SG_LUA_CPP_FUN_BEGIN(SetWorkDir)
 {
 	ui_main_c* ui = GetUIPtr(L);
@@ -2053,28 +1952,6 @@ int ui_main_c::InitAPI(lua_State* L)
 	sol::state_view lua(L);
 	luaL_openlibs(L);
 
-	{
-		ui_main_c* ui = GetUIPtr(L);
-		lua_getglobal(L, "io");
-		if (!lua_isnil(L, -1)) {
-			lua_getfield(L, -1, "open");
-			ui->ioOpenf = luaL_ref(L, LUA_REGISTRYINDEX);
-			lua_pushcfunction(L, l_OpenFile);
-			lua_setfield(L, -2, "open");
-		}
-		lua_pop(L, 1);
-
-		lua_getglobal(L, "os");
-		if (!lua_isnil(L, -1)) {
-			lua_pushcfunction(L, l_RemoveFile);
-			lua_setfield(L, -2, "remove");
-
-			lua_pushcfunction(L, l_RenameFile);
-			lua_setfield(L, -2, "rename");
-		}
-		lua_pop(L, 1);
-	}
-
 	// Add "lua/" subdir for non-JIT Lua
 	{
 		lua_getglobal(L, "package");
@@ -2182,9 +2059,6 @@ int ui_main_c::InitAPI(lua_State* L)
 	ADDFUNC(GetAsyncCount);
 	ADDFUNC(RenderInit);
 
-	// Wide file I/O
-	ADDFUNC(OpenFile);
-
 	// Search handles
 	lua_newtable(L);	// Search handle metatable
 	lua_pushvalue(L, -1);	// Push search handle metatable

From 9afb76f94552d7c1fb9cf25144520a577d7e8070 Mon Sep 17 00:00:00 2001
From: Lars Viklund <zao@zao.se>
Date: Tue, 9 Apr 2024 01:23:03 +0200
Subject: [PATCH 14/16] feat: patch LuaJIT to use UTF-8 internally

This patch to the vendored LuaJIT build changes much of the internal
machinery to use UTF-8 for paths, using W-style Windows API functions
and _w-type CRT functions with UTF-16LE wchar_t transcoding internally
to UTF-8 at the edge.

This also patches several functions exposed to Lua like `os.getenv`,
`os.remove`, `os.rename`, `os.system`, `os.tmpnam` as well as `io.open`.
---
 vcpkg-configuration.json                      |   2 +-
 .../ports/luajit/2023-04-16/msvcbuild.patch   |  31 --
 ...-do-not-set-macosx-deployment-target.patch |   0
 .../005-do-not-pass-ld-e-macosx.patch         |   0
 .../Makefile.nmake                            |   0
 .../{2023-04-16 => 2023-04-16_1}/configure    |   0
 .../luajit.pc.win.in                          |   0
 .../ports/luajit/2023-04-16_1/msvcbuild.patch |  62 +++
 .../luajit/2023-04-16_1/pob-wide-crt.patch    | 432 ++++++++++++++++++
 .../portfile.cmake                            |   1 +
 .../{2023-04-16 => 2023-04-16_1}/vcpkg.json   |   2 +-
 vcpkg-ports/versions/baseline.json            |   4 +-
 vcpkg-ports/versions/l-/luajit.json           |   4 +-
 13 files changed, 501 insertions(+), 37 deletions(-)
 delete mode 100644 vcpkg-ports/ports/luajit/2023-04-16/msvcbuild.patch
 rename vcpkg-ports/ports/luajit/{2023-04-16 => 2023-04-16_1}/003-do-not-set-macosx-deployment-target.patch (100%)
 rename vcpkg-ports/ports/luajit/{2023-04-16 => 2023-04-16_1}/005-do-not-pass-ld-e-macosx.patch (100%)
 rename vcpkg-ports/ports/luajit/{2023-04-16 => 2023-04-16_1}/Makefile.nmake (100%)
 rename vcpkg-ports/ports/luajit/{2023-04-16 => 2023-04-16_1}/configure (100%)
 rename vcpkg-ports/ports/luajit/{2023-04-16 => 2023-04-16_1}/luajit.pc.win.in (100%)
 create mode 100644 vcpkg-ports/ports/luajit/2023-04-16_1/msvcbuild.patch
 create mode 100644 vcpkg-ports/ports/luajit/2023-04-16_1/pob-wide-crt.patch
 rename vcpkg-ports/ports/luajit/{2023-04-16 => 2023-04-16_1}/portfile.cmake (99%)
 rename vcpkg-ports/ports/luajit/{2023-04-16 => 2023-04-16_1}/vcpkg.json (98%)

diff --git a/vcpkg-configuration.json b/vcpkg-configuration.json
index 8765a52..f8054ee 100644
--- a/vcpkg-configuration.json
+++ b/vcpkg-configuration.json
@@ -7,7 +7,7 @@
     {
       "kind": "filesystem",
       "path": "vcpkg-ports",
-      "baseline": "2024-03-26",
+      "baseline": "2025-01-19",
       "packages": ["glfw3", "luajit"]
     }
   ]
diff --git a/vcpkg-ports/ports/luajit/2023-04-16/msvcbuild.patch b/vcpkg-ports/ports/luajit/2023-04-16/msvcbuild.patch
deleted file mode 100644
index 4537578..0000000
--- a/vcpkg-ports/ports/luajit/2023-04-16/msvcbuild.patch
+++ /dev/null
@@ -1,31 +0,0 @@
-diff --git a/src/msvcbuild.bat b/src/msvcbuild.bat
-index 045965f8..8b0a4ace 100644
---- a/src/msvcbuild.bat
-+++ b/src/msvcbuild.bat
-@@ -69,10 +69,9 @@ buildvm -m folddef -o lj_folddef.h lj_opt_fold.c
- @set BUILDTYPE=debug
- @set LJCOMPILE=%LJCOMPILE% /Zi %DEBUGCFLAGS%
- :NODEBUG
--@set LJLINK=%LJLINK% /%BUILDTYPE%
- @if "%1"=="amalg" goto :AMALGDLL
- @if "%1"=="static" goto :STATIC
--%LJCOMPILE% /MD /DLUA_BUILD_AS_DLL lj_*.c lib_*.c
-+%LJCOMPILE% /MD /DLUA_BUILD_AS_DLL lj_*.c lib_*.c /Fdlua51.pdb
- @if errorlevel 1 goto :BAD
- %LJLINK% /DLL /out:%LJDLLNAME% lj_*.obj lib_*.obj
- @if errorlevel 1 goto :BAD
-@@ -92,7 +91,7 @@ buildvm -m folddef -o lj_folddef.h lj_opt_fold.c
- if exist %LJDLLNAME%.manifest^
-   %LJMT% -manifest %LJDLLNAME%.manifest -outputresource:%LJDLLNAME%;2
- 
--%LJCOMPILE% luajit.c
-+%LJCOMPILE% luajit.c /Fdluajit.pdb
- @if errorlevel 1 goto :BAD
- %LJLINK% /out:luajit.exe luajit.obj %LJLIBNAME%
- @if errorlevel 1 goto :BAD
-@@ -114,4 +113,5 @@ if exist luajit.exe.manifest^
- @goto :END
- :FAIL
- @echo You must open a "Visual Studio Command Prompt" to run this script
-+exit 1
- :END
diff --git a/vcpkg-ports/ports/luajit/2023-04-16/003-do-not-set-macosx-deployment-target.patch b/vcpkg-ports/ports/luajit/2023-04-16_1/003-do-not-set-macosx-deployment-target.patch
similarity index 100%
rename from vcpkg-ports/ports/luajit/2023-04-16/003-do-not-set-macosx-deployment-target.patch
rename to vcpkg-ports/ports/luajit/2023-04-16_1/003-do-not-set-macosx-deployment-target.patch
diff --git a/vcpkg-ports/ports/luajit/2023-04-16/005-do-not-pass-ld-e-macosx.patch b/vcpkg-ports/ports/luajit/2023-04-16_1/005-do-not-pass-ld-e-macosx.patch
similarity index 100%
rename from vcpkg-ports/ports/luajit/2023-04-16/005-do-not-pass-ld-e-macosx.patch
rename to vcpkg-ports/ports/luajit/2023-04-16_1/005-do-not-pass-ld-e-macosx.patch
diff --git a/vcpkg-ports/ports/luajit/2023-04-16/Makefile.nmake b/vcpkg-ports/ports/luajit/2023-04-16_1/Makefile.nmake
similarity index 100%
rename from vcpkg-ports/ports/luajit/2023-04-16/Makefile.nmake
rename to vcpkg-ports/ports/luajit/2023-04-16_1/Makefile.nmake
diff --git a/vcpkg-ports/ports/luajit/2023-04-16/configure b/vcpkg-ports/ports/luajit/2023-04-16_1/configure
similarity index 100%
rename from vcpkg-ports/ports/luajit/2023-04-16/configure
rename to vcpkg-ports/ports/luajit/2023-04-16_1/configure
diff --git a/vcpkg-ports/ports/luajit/2023-04-16/luajit.pc.win.in b/vcpkg-ports/ports/luajit/2023-04-16_1/luajit.pc.win.in
similarity index 100%
rename from vcpkg-ports/ports/luajit/2023-04-16/luajit.pc.win.in
rename to vcpkg-ports/ports/luajit/2023-04-16_1/luajit.pc.win.in
diff --git a/vcpkg-ports/ports/luajit/2023-04-16_1/msvcbuild.patch b/vcpkg-ports/ports/luajit/2023-04-16_1/msvcbuild.patch
new file mode 100644
index 0000000..1e733dc
--- /dev/null
+++ b/vcpkg-ports/ports/luajit/2023-04-16_1/msvcbuild.patch
@@ -0,0 +1,62 @@
+From 63a98f283b12265b722b4a7e0cd87d0e527a38f9 Mon Sep 17 00:00:00 2001
+From: Lars Viklund <zao@zao.se>
+Date: Mon, 8 Apr 2024 01:03:43 +0200
+Subject: [PATCH] feat: output PDBs for the library and apply LTCG
+
+---
+ src/msvcbuild.bat | 16 ++++++++--------
+ 1 file changed, 8 insertions(+), 8 deletions(-)
+
+diff --git a/src/msvcbuild.bat b/src/msvcbuild.bat
+index 045965f8..ac85da80 100644
+--- a/src/msvcbuild.bat
++++ b/src/msvcbuild.bat
+@@ -13,9 +13,9 @@
+ 
+ @setlocal
+ @rem Add more debug flags here, e.g. DEBUGCFLAGS=/DLUA_USE_APICHECK
+-@set DEBUGCFLAGS=
+-@set LJCOMPILE=cl /nologo /c /O2 /W3 /D_CRT_SECURE_NO_DEPRECATE /D_CRT_STDIO_INLINE=__declspec(dllexport)__inline
+-@set LJLINK=link /nologo
++@set DEBUGCFLAGS=/Od
++@set LJCOMPILE=cl /nologo /c /O2 /W3 /D_CRT_SECURE_NO_DEPRECATE /D_CRT_STDIO_INLINE=__declspec(dllexport)__inline /Z7 /GL
++@set LJLINK=link /nologo /LTCG
+ @set LJMT=mt /nologo
+ @set LJLIB=lib /nologo /nodefaultlib
+ @set DASMDIR=..\dynasm
+@@ -67,14 +67,13 @@ buildvm -m folddef -o lj_folddef.h lj_opt_fold.c
+ @if "%1" neq "debug" goto :NODEBUG
+ @shift
+ @set BUILDTYPE=debug
+-@set LJCOMPILE=%LJCOMPILE% /Zi %DEBUGCFLAGS%
++@set LJCOMPILE=%LJCOMPILE% /Z7 %DEBUGCFLAGS%
+ :NODEBUG
+-@set LJLINK=%LJLINK% /%BUILDTYPE%
+ @if "%1"=="amalg" goto :AMALGDLL
+ @if "%1"=="static" goto :STATIC
+-%LJCOMPILE% /MD /DLUA_BUILD_AS_DLL lj_*.c lib_*.c
++%LJCOMPILE% /MD /DLUA_BUILD_AS_DLL lj_*.c lib_*.c /Fdlua51.pdb
+ @if errorlevel 1 goto :BAD
+-%LJLINK% /DLL /out:%LJDLLNAME% lj_*.obj lib_*.obj
++%LJLINK% /DLL /out:%LJDLLNAME% lj_*.obj lib_*.obj /DEBUG /OPT:ICF /OPT:REF
+ @if errorlevel 1 goto :BAD
+ @goto :MTDLL
+ :STATIC
+@@ -92,7 +91,7 @@ buildvm -m folddef -o lj_folddef.h lj_opt_fold.c
+ if exist %LJDLLNAME%.manifest^
+   %LJMT% -manifest %LJDLLNAME%.manifest -outputresource:%LJDLLNAME%;2
+ 
+-%LJCOMPILE% luajit.c
++%LJCOMPILE% luajit.c /Fdluajit.pdb
+ @if errorlevel 1 goto :BAD
+ %LJLINK% /out:luajit.exe luajit.obj %LJLIBNAME%
+ @if errorlevel 1 goto :BAD
+@@ -114,4 +113,5 @@ if exist luajit.exe.manifest^
+ @goto :END
+ :FAIL
+ @echo You must open a "Visual Studio Command Prompt" to run this script
++exit 1
+ :END
+-- 
+2.42.0.windows.2
+
diff --git a/vcpkg-ports/ports/luajit/2023-04-16_1/pob-wide-crt.patch b/vcpkg-ports/ports/luajit/2023-04-16_1/pob-wide-crt.patch
new file mode 100644
index 0000000..4f5b82d
--- /dev/null
+++ b/vcpkg-ports/ports/luajit/2023-04-16_1/pob-wide-crt.patch
@@ -0,0 +1,432 @@
+From b72444ea45ca0fcb1a55dc3e9f9976f596cd57ed Mon Sep 17 00:00:00 2001
+From: Lars Viklund <zao@zao.se>
+Date: Mon, 8 Apr 2024 01:06:14 +0200
+Subject: [PATCH] feat: use wide CRT/API functions on Windows
+
+As functions like `fopen` and `LoadLibraryA` use the ACP on Windows and
+we would like to use UTF-8 like on other platform, transcode paths on
+demand to UTF16-LE and use MSVC CRT and W-type Windows API functions.
+---
+ src/Makefile      |   2 +-
+ src/lib_io.c      |   5 +-
+ src/lib_os.c      |  13 +++--
+ src/lib_package.c |  33 +++++++----
+ src/lj_clib.c     |   9 ++-
+ src/lj_load.c     |   3 +-
+ src/lj_utf8win.c  | 137 ++++++++++++++++++++++++++++++++++++++++++++++
+ src/lj_utf8win.h  |  28 ++++++++++
+ 8 files changed, 208 insertions(+), 22 deletions(-)
+ create mode 100644 src/lj_utf8win.c
+ create mode 100644 src/lj_utf8win.h
+
+diff --git a/src/Makefile b/src/Makefile
+index c4d0b14d..002cb149 100644
+--- a/src/Makefile
++++ b/src/Makefile
+@@ -471,7 +471,7 @@ LJCORE_O= lj_gc.o lj_err.o lj_char.o lj_bc.o lj_obj.o \
+ 	  lj_mcode.o lj_snap.o lj_record.o lj_crecord.o lj_ffrecord.o \
+ 	  lj_asm.o lj_trace.o lj_gdbjit.o \
+ 	  lj_ctype.o lj_cdata.o lj_cconv.o lj_ccall.o lj_ccallback.o \
+-	  lj_carith.o lj_clib.o lj_cparse.o \
++	  lj_carith.o lj_clib.o lj_cparse.o lj_utf8win.o \
+ 	  lj_lib.o lj_alloc.o lib_aux.o \
+ 	  $(LJLIB_O) lib_init.o
+ 
+diff --git a/src/lib_io.c b/src/lib_io.c
+index d5786e5d..31a470da 100644
+--- a/src/lib_io.c
++++ b/src/lib_io.c
+@@ -23,6 +23,7 @@
+ #include "lj_state.h"
+ #include "lj_ff.h"
+ #include "lj_lib.h"
++#include "lj_utf8win.h"
+ 
+ /* Userdata payload for I/O file. */
+ typedef struct IOFileUD {
+@@ -82,7 +83,7 @@ static IOFileUD *io_file_open(lua_State *L, const char *mode)
+ {
+   const char *fname = strdata(lj_lib_checkstr(L, 1));
+   IOFileUD *iof = io_file_new(L);
+-  iof->fp = fopen(fname, mode);
++  iof->fp = _lua_fopen(fname, mode);
+   if (iof->fp == NULL)
+     luaL_argerror(L, 1, lj_str_pushf(L, "%s: %s", fname, strerror(errno)));
+   return iof;
+@@ -413,7 +414,7 @@ LJLIB_CF(io_open)
+   GCstr *s = lj_lib_optstr(L, 2);
+   const char *mode = s ? strdata(s) : "r";
+   IOFileUD *iof = io_file_new(L);
+-  iof->fp = fopen(fname, mode);
++  iof->fp = _lua_fopen(fname, mode);
+   return iof->fp != NULL ? 1 : luaL_fileresult(L, 0, fname);
+ }
+ 
+diff --git a/src/lib_os.c b/src/lib_os.c
+index 7ad7dfaf..f8ac64a3 100644
+--- a/src/lib_os.c
++++ b/src/lib_os.c
+@@ -19,6 +19,7 @@
+ #include "lj_obj.h"
+ #include "lj_err.h"
+ #include "lj_lib.h"
++#include "lj_utf8win.h"
+ 
+ #if LJ_TARGET_POSIX
+ #include <unistd.h>
+@@ -46,7 +47,7 @@ LJLIB_CF(os_execute)
+ #endif
+ #else
+   const char *cmd = luaL_optstring(L, 1, NULL);
+-  int stat = system(cmd);
++  int stat = _lua_system(cmd);
+ #if LJ_52
+   if (cmd)
+     return luaL_execresult(L, stat);
+@@ -61,14 +62,14 @@ LJLIB_CF(os_execute)
+ LJLIB_CF(os_remove)
+ {
+   const char *filename = luaL_checkstring(L, 1);
+-  return luaL_fileresult(L, remove(filename) == 0, filename);
++  return luaL_fileresult(L, _lua_remove(filename) == 0, filename);
+ }
+ 
+ LJLIB_CF(os_rename)
+ {
+   const char *fromname = luaL_checkstring(L, 1);
+   const char *toname = luaL_checkstring(L, 2);
+-  return luaL_fileresult(L, rename(fromname, toname) == 0, fromname);
++  return luaL_fileresult(L, _lua_rename(fromname, toname) == 0, fromname);
+ }
+ 
+ LJLIB_CF(os_tmpname)
+@@ -88,7 +89,7 @@ LJLIB_CF(os_tmpname)
+     lj_err_caller(L, LJ_ERR_OSUNIQF);
+ #else
+   char buf[L_tmpnam];
+-  if (tmpnam(buf) == NULL)
++  if (_lua_tmpnam(buf) == NULL)
+     lj_err_caller(L, LJ_ERR_OSUNIQF);
+ #endif
+   lua_pushstring(L, buf);
+@@ -101,7 +102,9 @@ LJLIB_CF(os_getenv)
+ #if LJ_TARGET_CONSOLE
+   lua_pushnil(L);
+ #else
+-  lua_pushstring(L, getenv(luaL_checkstring(L, 1)));  /* if NULL push nil */
++  char const* val = _lua_getenvcopy(luaL_checkstring(L, 1));
++  lua_pushstring(L, val); /* if NULL push nil */
++  _lua_getenvfree(val);
+ #endif
+   return 1;
+ }
+diff --git a/src/lib_package.c b/src/lib_package.c
+index d2ef474f..681e7258 100644
+--- a/src/lib_package.c
++++ b/src/lib_package.c
+@@ -16,6 +16,7 @@
+ #include "lj_obj.h"
+ #include "lj_err.h"
+ #include "lj_lib.h"
++#include "lj_utf8win.h"
+ 
+ /* ------------------------------------------------------------------------ */
+ 
+@@ -80,15 +81,18 @@ BOOL WINAPI GetModuleHandleExA(DWORD, LPCSTR, HMODULE*);
+ 
+ static void setprogdir(lua_State *L)
+ {
+-  char buff[MAX_PATH + 1];
+-  char *lb;
+-  DWORD nsize = sizeof(buff);
+-  DWORD n = GetModuleFileNameA(NULL, buff, nsize);
+-  if (n == 0 || n == nsize || (lb = strrchr(buff, '\\')) == NULL) {
++  enum { N = 1<<16 };
++  wchar_t buff[N];
++  wchar_t *lb;
++  DWORD nsize = N;
++  DWORD n = GetModuleFileNameW(NULL, buff, nsize);
++  if (n == 0 || n == nsize || (lb = wcsrchr(buff, L'\\')) == NULL) {
+     luaL_error(L, "unable to get ModuleFileName");
+   } else {
+-    *lb = '\0';
+-    luaL_gsub(L, lua_tostring(L, -1), LUA_EXECDIR, buff);
++    *lb = L'\0';
++    char nbuff[N];
++    _lua_narrowtobuffer(buff, nbuff, N);
++    luaL_gsub(L, lua_tostring(L, -1), LUA_EXECDIR, nbuff);
+     lua_remove(L, -2);  /* remove original string */
+   }
+ }
+@@ -111,7 +115,9 @@ static void ll_unloadlib(void *lib)
+ 
+ static void *ll_load(lua_State *L, const char *path, int gl)
+ {
+-  HINSTANCE lib = LoadLibraryA(path);
++  wchar_t pathBuf[1<<16];
++  _lua_widentobuffer(path, pathBuf, _countof(pathBuf));
++  HINSTANCE lib = LoadLibraryW(pathBuf);
+   if (lib == NULL) pusherror(L);
+   UNUSED(gl);
+   return lib;
+@@ -267,7 +273,7 @@ static int lj_cf_package_unloadlib(lua_State *L)
+ 
+ static int readable(const char *filename)
+ {
+-  FILE *f = fopen(filename, "r");  /* try to open file */
++  FILE *f = _lua_fopen(filename, "r");  /* try to open file */
+   if (f == NULL) return 0;  /* open failed */
+   fclose(f);
+   return 1;
+@@ -535,17 +541,20 @@ static void setpath(lua_State *L, const char *fieldname, const char *envname,
+ #if LJ_TARGET_CONSOLE
+   const char *path = NULL;
+   UNUSED(envname);
++  return;
+ #else
+-  const char *path = getenv(envname);
++  char const *path = _lua_getenvcopy(envname);
+ #endif
+   if (path == NULL || noenv) {
+     lua_pushstring(L, def);
+   } else {
+-    path = luaL_gsub(L, path, LUA_PATHSEP LUA_PATHSEP,
++    char const* lpath = luaL_gsub(L, path, LUA_PATHSEP LUA_PATHSEP,
+ 			      LUA_PATHSEP AUXMARK LUA_PATHSEP);
+-    luaL_gsub(L, path, AUXMARK, def);
++
++    luaL_gsub(L, lpath, AUXMARK, def);
+     lua_remove(L, -2);
+   }
++  _lua_getenvfree(path);
+   setprogdir(L);
+   lua_setfield(L, -2, fieldname);
+ }
+diff --git a/src/lj_clib.c b/src/lj_clib.c
+index ab2db33a..d1d3c816 100644
+--- a/src/lj_clib.c
++++ b/src/lj_clib.c
+@@ -16,6 +16,7 @@
+ #include "lj_cconv.h"
+ #include "lj_cdata.h"
+ #include "lj_clib.h"
++#include "lj_utf8win.h"
+ 
+ /* -- OS-specific functions ----------------------------------------------- */
+ 
+@@ -200,7 +201,13 @@ static const char *clib_extname(lua_State *L, const char *name)
+ static void *clib_loadlib(lua_State *L, const char *name, int global)
+ {
+   DWORD oldwerr = GetLastError();
+-  void *h = (void *)LoadLibraryA(clib_extname(L, name));
++
++  enum {NameSize = 1<<16};
++  wchar_t namebuf[NameSize];
++  void *h = NULL;
++  if (_lua_widentobuffer(clib_extname(L, name), namebuf, NameSize)) {
++    h = (void *)LoadLibraryW(namebuf);
++  }
+   if (!h) clib_error(L, "cannot load module " LUA_QS ": %s", name);
+   SetLastError(oldwerr);
+   UNUSED(global);
+diff --git a/src/lj_load.c b/src/lj_load.c
+index dbd36ac7..8beeb452 100644
+--- a/src/lj_load.c
++++ b/src/lj_load.c
+@@ -22,6 +22,7 @@
+ #include "lj_lex.h"
+ #include "lj_bcdump.h"
+ #include "lj_parse.h"
++#include "lj_utf8win.h"
+ 
+ /* -- Load Lua source code and bytecode ----------------------------------- */
+ 
+@@ -88,7 +89,7 @@ LUALIB_API int luaL_loadfilex(lua_State *L, const char *filename,
+   int status;
+   const char *chunkname;
+   if (filename) {
+-    ctx.fp = fopen(filename, "rb");
++    ctx.fp = _lua_fopen(filename, "rb");
+     if (ctx.fp == NULL) {
+       lua_pushfstring(L, "cannot open %s: %s", filename, strerror(errno));
+       return LUA_ERRFILE;
+diff --git a/src/lj_utf8win.c b/src/lj_utf8win.c
+new file mode 100644
+index 00000000..96ee913a
+--- /dev/null
++++ b/src/lj_utf8win.c
+@@ -0,0 +1,137 @@
++#include "lj_utf8win.h"
++
++#ifdef _WIN32
++#include <Windows.h>
++#include <stdbool.h>
++#include <stdio.h>
++#include <wchar.h>
++
++enum { PathBufSize = 1<<16, ModeBufSize = 5 };
++
++bool _lua_narrowtobuffer(wchar_t const* srcPtr, char* dstBuf, int dstSize)
++{
++  int cchSrc = (int)wcslen(srcPtr);
++  if (!cchSrc) {
++    dstBuf[0] = '\0';
++    return true;
++  }
++  int cbDst = WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS, srcPtr, cchSrc, dstBuf, dstSize - 1, NULL, NULL);
++  if (!cbDst || cbDst >= dstSize) {
++    return false;
++  }
++  dstBuf[cbDst] = '\0';
++  return true;
++}
++
++bool _lua_widentobuffer(char const* srcPtr, wchar_t* dstBuf, int dstSize)
++{
++  int cbSrc = (int)strlen(srcPtr);
++  if (!cbSrc) {
++    dstBuf[0] = L'\0';
++    return true;
++  }
++  int cchDst = MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, srcPtr, cbSrc, dstBuf, dstSize - 1);
++  if (!cchDst || cchDst >= dstSize ) {
++    return false;
++  }
++  dstBuf[cchDst] = L'\0';
++  return true;
++}
++
++FILE* _lua_fopen(char const* path, char const* mode)
++{
++  if (!path || !mode) {
++    return NULL;
++  }
++  wchar_t pathBuf[PathBufSize];
++  wchar_t modeBuf[ModeBufSize];
++  if (!_lua_widentobuffer(path, pathBuf, PathBufSize) ||
++      !_lua_widentobuffer(mode, modeBuf, ModeBufSize))
++  {
++    return NULL;
++  }
++  FILE* fp = _wfopen(pathBuf, modeBuf);
++  return fp;
++}
++
++char const* _lua_getenvcopy(char const* name)
++{
++  if (!name) {
++    return NULL;
++  }
++  wchar_t nameBuf[PathBufSize];
++  if (!_lua_widentobuffer(name, nameBuf, PathBufSize)) {
++    return NULL;
++  }
++  wchar_t* var = _wgetenv(nameBuf);
++  if (!var) {
++    return NULL;
++  }
++  char varBuf[PathBufSize];
++  if (!_lua_narrowtobuffer(var, varBuf, PathBufSize)) {
++    return NULL;
++  }
++  size_t cbVar = strlen(varBuf) + 1;
++  char* ret = calloc(1, cbVar);
++  memcpy(ret, varBuf, cbVar);
++  return ret;
++}
++
++void _lua_getenvfree(char const* var)
++{
++  if (var) {
++    free((char*)var);
++  }
++}
++
++int _lua_remove(char const* path)
++{
++  if (!path) {
++    return -1;
++  }
++  wchar_t pathBuf[PathBufSize];
++  if (!_lua_widentobuffer(path, pathBuf, PathBufSize)) {
++        return -1;
++  }
++  return _wremove(pathBuf);
++}
++
++int _lua_rename(char const* oldpath, char const* newpath)
++{
++  if (!oldpath || !newpath) {
++    return -1;
++  }
++  wchar_t oldBuf[PathBufSize];
++  wchar_t newBuf[PathBufSize];
++  if (!_lua_widentobuffer(oldpath, oldBuf, PathBufSize) ||
++      !_lua_widentobuffer(newpath, newBuf, PathBufSize))
++  {
++    return -1;
++  }
++  return _wrename(oldBuf, newBuf);
++}
++
++int _lua_system(char const* cmd)
++{
++  if (!cmd) {
++    return -1;
++  }
++  wchar_t cmdBuf[PathBufSize];
++  if (!_lua_widentobuffer(cmd, cmdBuf, PathBufSize)) {
++    return -1;
++  }
++  return _wsystem(cmdBuf);
++}
++
++char* _lua_tmpnam(char* s)
++{
++  wchar_t tmpBuf[L_tmpnam];
++  _wtmpnam(tmpBuf);
++
++  if (!_lua_narrowtobuffer(tmpBuf, s, L_tmpnam)) {
++    return NULL;
++  }
++  return s;
++}
++
++#endif
+diff --git a/src/lj_utf8win.h b/src/lj_utf8win.h
+new file mode 100644
+index 00000000..72e9ccae
+--- /dev/null
++++ b/src/lj_utf8win.h
+@@ -0,0 +1,28 @@
++#ifndef LJ_UTF8_WIN_H
++#define LJ_UTF8_WIN_H
++
++#include <stdbool.h>
++#include <stddef.h>
++#include <stdio.h>
++
++#ifdef _WIN32
++bool _lua_narrowtobuffer(wchar_t const* srcPtr, char* dstBuf, int dstSize);
++bool _lua_widentobuffer(char const* srcPtr, wchar_t* dstBuf, int dstSize);
++FILE* _lua_fopen(char const* path, char const* mode);
++char const* _lua_getenvcopy(char const* name);
++void _lua_getenvfree(char const* name);
++int _lua_remove(char const* path);
++int _lua_rename(char const* oldpath, char const* newpath);
++int _lua_system(char const* cmd);
++char* _lua_tmpnam();
++#else
++#define _lua_fopen(path, mode) fopen(path, mode)
++#define _lua_getenvcopy(name) strdup(getenv(name))
++#define _lua_getenvfree(name) free(name)
++#define _lua_remove(path) remove(path)
++#define _lua_rename(oldpath, newpath) rename(oldpath, newpath)
++#define _lua_system(cmd) system(cmd)
++#define _lua_tmpnam(s) tmpnam(s)
++#endif
++
++#endif
+-- 
+2.42.0.windows.2
+
diff --git a/vcpkg-ports/ports/luajit/2023-04-16/portfile.cmake b/vcpkg-ports/ports/luajit/2023-04-16_1/portfile.cmake
similarity index 99%
rename from vcpkg-ports/ports/luajit/2023-04-16/portfile.cmake
rename to vcpkg-ports/ports/luajit/2023-04-16_1/portfile.cmake
index cbb91f3..5392eec 100644
--- a/vcpkg-ports/ports/luajit/2023-04-16/portfile.cmake
+++ b/vcpkg-ports/ports/luajit/2023-04-16_1/portfile.cmake
@@ -12,6 +12,7 @@ vcpkg_from_github(
     PATCHES
         msvcbuild.patch
         003-do-not-set-macosx-deployment-target.patch
+        pob-wide-crt.patch
         ${extra_patches}
 )
 
diff --git a/vcpkg-ports/ports/luajit/2023-04-16/vcpkg.json b/vcpkg-ports/ports/luajit/2023-04-16_1/vcpkg.json
similarity index 98%
rename from vcpkg-ports/ports/luajit/2023-04-16/vcpkg.json
rename to vcpkg-ports/ports/luajit/2023-04-16_1/vcpkg.json
index 7170aa2..51df242 100644
--- a/vcpkg-ports/ports/luajit/2023-04-16/vcpkg.json
+++ b/vcpkg-ports/ports/luajit/2023-04-16_1/vcpkg.json
@@ -1,7 +1,7 @@
 {
   "name": "luajit",
   "version-date": "2023-04-16",
-  "port-version": 0,
+  "port-version": 1,
   "description": "LuaJIT is a Just-In-Time (JIT) compiler for the Lua programming language.",
   "homepage": "https://github.com/LuaJIT/LuaJIT",
   "license": "MIT",
diff --git a/vcpkg-ports/versions/baseline.json b/vcpkg-ports/versions/baseline.json
index 07af5ad..c95d67c 100644
--- a/vcpkg-ports/versions/baseline.json
+++ b/vcpkg-ports/versions/baseline.json
@@ -2,8 +2,8 @@
     "2022-08-14": {
         "luajit": { "baseline": "2022-08-11", "port-version": 1 }
     },
-    "2024-03-26": {
+    "2025-01-19": {
         "glfw3": { "baseline": "3.4", "port-version": 1 },
-        "luajit": { "baseline": "2023-04-16", "port-version": 0 }
+        "luajit": { "baseline": "2023-04-16", "port-version": 1 }
     }
 }
\ No newline at end of file
diff --git a/vcpkg-ports/versions/l-/luajit.json b/vcpkg-ports/versions/l-/luajit.json
index fc43c9f..1f587eb 100644
--- a/vcpkg-ports/versions/l-/luajit.json
+++ b/vcpkg-ports/versions/l-/luajit.json
@@ -2,8 +2,8 @@
     "versions": [
       {
         "version-date": "2023-04-16",
-        "port-version": 0,
-        "path": "$/ports/luajit/2023-04-16"
+        "port-version": 1,
+        "path": "$/ports/luajit/2023-04-16_1"
       },
       {
         "version-date": "2022-08-11",

From 222574d7b39a5bb9423ce399f231b46a1bfa2556 Mon Sep 17 00:00:00 2001
From: Lars Viklund <zao@zao.se>
Date: Tue, 9 Apr 2024 17:12:08 +0200
Subject: [PATCH 15/16] fix: expose all paths to Lua with forward slashes

For `fs::path` the `u8string()` function yields paths in the native OS
format with backward slashes on Windows. For consistency with existing
forward slash use in PoB Lua code they're now instead exposed as
`generic_u8string()` which has consistent separators across platforms.
---
 engine/core/core_config.cpp    |  4 ++--
 engine/core/core_image.cpp     |  6 +++---
 engine/render/r_main.cpp       |  3 +--
 engine/system/win/sys_main.cpp |  4 ++--
 ui_api.cpp                     | 19 ++++++++++---------
 ui_main.cpp                    | 10 +++++-----
 6 files changed, 23 insertions(+), 23 deletions(-)

diff --git a/engine/core/core_config.cpp b/engine/core/core_config.cpp
index a073811..d5f4135 100644
--- a/engine/core/core_config.cpp
+++ b/engine/core/core_config.cpp
@@ -158,7 +158,7 @@ bool core_config_c::LoadConfig(std::filesystem::path const& cfgName)
 	auto fileName = cfgName;
 	fileName.replace_extension(".cfg");
 
-	sys->con->Print(fmt::format("Executing {}\n", fileName.u8string()).c_str());
+	sys->con->Print(fmt::format("Executing {}\n", fileName.generic_u8string()).c_str());
 
 	// Read the config file
 	fileInputStream_c f;
@@ -202,7 +202,7 @@ bool core_config_c::SaveConfig(std::filesystem::path const& cfgName)
 	auto fileName = cfgName;
 	fileName.replace_extension(".cfg");
 
-	sys->con->Print(fmt::format("Saving {}\n", fileName.u8string()).c_str());
+	sys->con->Print(fmt::format("Saving {}\n", fileName.generic_u8string()).c_str());
 
 	// Open the config file
 	fileOutputStream_c f;
diff --git a/engine/core/core_image.cpp b/engine/core/core_image.cpp
index 079d245..54d62c8 100644
--- a/engine/core/core_image.cpp
+++ b/engine/core/core_image.cpp
@@ -91,7 +91,7 @@ bool image_c::Save(std::filesystem::path const& fileName)
 
 image_c* image_c::LoaderForFile(IConsole* conHnd, std::filesystem::path const& fileName)
 {
-	auto nameU8 = fileName.u8string();
+	auto nameU8 = fileName.generic_u8string();
 	fileInputStream_c in;
 	if (in.FileOpen(fileName, true)) {
 		conHnd->Warning("'%s' doesn't exist or cannot be opened", nameU8.c_str());
@@ -161,7 +161,7 @@ bool targa_c::Load(std::filesystem::path const& fileName, std::optional<size_cal
 		return true;
 	}
 
-	auto nameU8 = fileName.u8string();
+	auto nameU8 = fileName.generic_u8string();
 
 	// Read header
 	tgaHeader_s hdr;
@@ -286,7 +286,7 @@ bool jpeg_c::Load(std::filesystem::path const& fileName, std::optional<size_call
 		return true;
 	}
 
-	auto nameU8 = fileName.u8string();
+	auto nameU8 = fileName.generic_u8string();
 
 	std::vector<byte> fileData(in.GetLen());
 	if (in.Read(fileData.data(), fileData.size())) {
diff --git a/engine/render/r_main.cpp b/engine/render/r_main.cpp
index e286b03..5fee07d 100644
--- a/engine/render/r_main.cpp
+++ b/engine/render/r_main.cpp
@@ -1952,7 +1952,6 @@ void r_renderer_c::DoScreenshot(image_c* i, int type, const char* ext)
 	auto ssPath = std::filesystem::u8path(fmt::format(CFG_DATAPATH "Screenshots/{:%m%d%y_%H%M%S}.{}",
 		fmt::localtime(curTime), ext));
 
-
 	// Make folder if it doesn't exist
 	std::error_code ec;
 	std::filesystem::create_directories(ssPath.parent_path(), ec);
@@ -1965,7 +1964,7 @@ void r_renderer_c::DoScreenshot(image_c* i, int type, const char* ext)
 		sys->con->Print("Couldn't write screenshot!\n");
 		return;
 	}
-	sys->con->Print(fmt::format("Wrote screenshot to {}\n", ssPath.u8string()).c_str());
+	sys->con->Print(fmt::format("Wrote screenshot to {}\n", ssPath.generic_u8string()).c_str());
 }
 
 r_renderer_c::RenderTarget& r_renderer_c::GetDrawRenderTarget()
diff --git a/engine/system/win/sys_main.cpp b/engine/system/win/sys_main.cpp
index 14ae434..82dc631 100644
--- a/engine/system/win/sys_main.cpp
+++ b/engine/system/win/sys_main.cpp
@@ -139,7 +139,7 @@ find_c::~find_c()
 std::optional<std::string> BuildGlobPattern(std::filesystem::path const& glob)
 {
 	using namespace std::literals::string_view_literals;
-	auto globStr = glob.u8string();
+	auto globStr = glob.generic_u8string();
 
 	// Deal with traditional "everything" wildcards.
 	if (glob == "*" || glob == "*.*") {
@@ -193,7 +193,7 @@ bool GlobMatch(std::optional<std::string> const& globPattern, std::filesystem::p
 	reOpts.set_case_sensitive(false);
 	RE2 reGlob{globPattern.value(), reOpts};
 
-	auto fileStr = file.u8string();
+	auto fileStr = file.generic_u8string();
 	return RE2::FullMatch(fileStr, reGlob);
 }
 
diff --git a/ui_api.cpp b/ui_api.cpp
index 15f0751..0622ba8 100644
--- a/ui_api.cpp
+++ b/ui_api.cpp
@@ -425,7 +425,7 @@ SG_LUA_CPP_FUN_BEGIN(imgHandleLoad)
 		}
 	}
 	// TODO(LV): should we use u8path throughout here, to support any callers that use paths outside of working directory?
-	imgHandle->hnd = ui->renderer->RegisterShader(fileName.u8string(), flags);
+	imgHandle->hnd = ui->renderer->RegisterShader(fileName.generic_u8string(), flags);
 	return 0;
 }
 SG_LUA_CPP_FUN_END()
@@ -1246,7 +1246,7 @@ static int l_searchHandleGetFileName(lua_State* L)
 {
 	ui_main_c* ui = GetUIPtr(L);
 	searchHandle_s* searchHandle = GetSearchHandle(L, ui, "GetFileName", true);
-	lua_pushstring(L, searchHandle->find->fileName.u8string().c_str());
+	lua_pushstring(L, searchHandle->find->fileName.generic_u8string().c_str());
 	return 1;
 }
 
@@ -1542,14 +1542,14 @@ static int l_GetTime(lua_State* L)
 static int l_GetScriptPath(lua_State* L)
 {
 	ui_main_c* ui = GetUIPtr(L);
-	lua_pushstring(L, ui->scriptPath.u8string().c_str());
+	lua_pushstring(L, ui->scriptPath.generic_u8string().c_str());
 	return 1;
 }
 
 static int l_GetRuntimePath(lua_State* L)
 {
 	ui_main_c* ui = GetUIPtr(L);
-	lua_pushstring(L, ui->sys->basePath.u8string().c_str());
+	lua_pushstring(L, ui->sys->basePath.generic_u8string().c_str());
 	return 1;
 }
 
@@ -1558,7 +1558,7 @@ static int l_GetUserPath(lua_State* L)
 	ui_main_c* ui = GetUIPtr(L);
 	auto& userPath = ui->sys->userPath;
 	if (userPath) {
-		lua_pushstring(L, userPath->u8string().c_str());
+		lua_pushstring(L, userPath->generic_u8string().c_str());
 		return 1;
 	}
 	return 0;
@@ -1622,7 +1622,7 @@ SG_LUA_CPP_FUN_END()
 static int l_GetWorkDir(lua_State* L)
 {
 	ui_main_c* ui = GetUIPtr(L);
-	lua_pushstring(L, ui->scriptWorkDir.u8string().c_str());
+	lua_pushstring(L, ui->scriptWorkDir.generic_u8string().c_str());
 	return 1;
 }
 
@@ -1702,9 +1702,10 @@ SG_LUA_CPP_FUN_BEGIN(LoadModule)
 	}
 
 	ui->sys->SetWorkDir(ui->scriptPath);
-	int err = luaL_loadfile(L, fileName.u8string().c_str());
+	auto fileStr = fileName.generic_u8string();
+	int err = luaL_loadfile(L, fileStr.c_str());
 	ui->sys->SetWorkDir(ui->scriptWorkDir);
-	ui->LExpect(L, err == 0, "LoadModule() error loading '%s' (%d):\n%s", fileName, err, lua_tostring(L, -1));
+	ui->LExpect(L, err == 0, "LoadModule() error loading '%s' (%d):\n%s", fileStr.c_str(), err, lua_tostring(L, -1));
 	lua_replace(L, 1);	// Replace module name with module main chunk
 	lua_call(L, n - 1, LUA_MULTRET);
 	return lua_gettop(L);
@@ -1724,7 +1725,7 @@ SG_LUA_CPP_FUN_BEGIN(PLoadModule)
 	}
 
 	ui->sys->SetWorkDir(ui->scriptPath);
-	int err = luaL_loadfile(L, fileName.u8string().c_str());
+	int err = luaL_loadfile(L, fileName.generic_u8string().c_str());
 	ui->sys->SetWorkDir(ui->scriptWorkDir);
 	if (err) {
 		return 1;
diff --git a/ui_main.cpp b/ui_main.cpp
index 8dbaec4..1c0281d 100644
--- a/ui_main.cpp
+++ b/ui_main.cpp
@@ -160,7 +160,7 @@ void ui_main_c::PCall(int narg, int nret)
 
 void ui_main_c::DoError(const char* msg, const char* error)
 {
-	auto scriptStr = scriptName.u8string();
+	auto scriptStr = scriptName.generic_u8string();
 	char* errText = AllocStringLen(strlen(msg) + scriptStr.size() + strlen(error) + 30);
 	sprintf(errText, "--- SCRIPT ERROR ---\n%s '%s':\n%s\n", msg, scriptStr.c_str(), error);
 	sys->Exit(errText);
@@ -266,11 +266,11 @@ void ui_main_c::ScriptInit()
 {
 	sys->con->PrintFunc("UI Init");
 
-	sys->con->Printf("Script: %s\n", scriptName.u8string().c_str());
+	sys->con->Printf("Script: %s\n", scriptName.generic_u8string().c_str());
 	if (!scriptPath.empty()) {
-		sys->con->Printf("Script working directory: %s\n", scriptWorkDir.u8string().c_str());
+		sys->con->Printf("Script working directory: %s\n", scriptWorkDir.generic_u8string().c_str());
 	}
-	sys->video->SetTitle(scriptName.u8string().c_str());
+	sys->video->SetTitle(scriptName.generic_u8string().c_str());
 
 	restartFlag = false;
 	didExit = false;
@@ -313,7 +313,7 @@ void ui_main_c::ScriptInit()
 	
 	// Load the script file
 	sys->SetWorkDir(scriptWorkDir);
- 	err = luaL_loadfile(L, scriptName.filename().u8string().c_str());
+ 	err = luaL_loadfile(L, scriptName.filename().generic_u8string().c_str());
 	if (err) {
 		DoError("Error loading", lua_tostring(L, -1));
 		return;

From d8f5c4d468ecf1f035d9cdd042eeb75f5a5bef74 Mon Sep 17 00:00:00 2001
From: Lars Viklund <zao@zao.se>
Date: Thu, 11 Apr 2024 20:23:40 +0200
Subject: [PATCH 16/16] feat: make file search patterns understand Unicode

File search patterns previously split up and escaped individual UTF-8
code units instead of matching on the actual codepoint.

This change transcodes to UTF-32 codepoints emits non-special codepoints
as curly hex escapes like `\x{10FFFF}`.
---
 engine/system/win/sys_main.cpp | 35 ++++++++++++++++++++--------------
 1 file changed, 21 insertions(+), 14 deletions(-)

diff --git a/engine/system/win/sys_main.cpp b/engine/system/win/sys_main.cpp
index 82dc631..e2a9992 100644
--- a/engine/system/win/sys_main.cpp
+++ b/engine/system/win/sys_main.cpp
@@ -140,42 +140,49 @@ std::optional<std::string> BuildGlobPattern(std::filesystem::path const& glob)
 {
 	using namespace std::literals::string_view_literals;
 	auto globStr = glob.generic_u8string();
+	auto globView = std::string_view(globStr);
 
 	// Deal with traditional "everything" wildcards.
-	if (glob == "*" || glob == "*.*") {
+	if (globView == "*" || globView == "*.*") {
 		return {};
 	}
 
+	auto u32Str = IndexUTF8ToUTF32(globStr);
+	auto& offsets = u32Str.sourceCodeUnitOffsets;
+
 	fmt::memory_buffer buf;
+	buf.reserve(globStr.size() * 3); // Decent estimate of final pattern size.
 
 	// If no wildcards are present, test file path verbatim.
 	// We use a regex rather than string comparisons to make it case-insensitive.
-	if (globStr.find_first_of("?*") == std::string::npos) {
-		buf.reserve(globStr.size() * 3); // Decent estimate of final pattern size.
-
-		for (char ch : globStr) {
-			fmt::format_to(fmt::appender(buf), "[{}]", ch);
+	if (u32Str.text.find_first_of(U"?*") == std::u32string::npos) {
+		for (size_t offIdx = 0; offIdx < offsets.size(); ++offIdx) {
+			int byteOffset = offsets[offIdx];
+			int nextOffset = (offIdx + 1 < offsets.size()) ? offsets[offIdx + 1] : globStr.size();
+			fmt::format_to(fmt::appender(buf), "[{}]", globView.substr(byteOffset, nextOffset - byteOffset));
 		}
 	}
 	else {
 		// Otherwise build a regular expression from the glob and use that to match files.
 		auto it = fmt::appender(buf);
-		for (char ch : globStr) {
-			if (ch == '*') {
+		for (size_t offIdx = 0; offIdx < offsets.size(); ++offIdx) {
+			char32_t ch = u32Str.text[offIdx];
+			if (ch == U'*') {
 				it = fmt::format_to(it, ".*");
 			}
-			else if (ch == '?') {
+			else if (ch == U'?') {
 				*it++ = '.';
 			}
-			else if ("+[]{}+()|"sv.find(ch) != std::string_view::npos) {
+			else if (U".+[]{}+()|"sv.find(ch) != std::u32string::npos) {
 				// Escape metacharacters
-				it = fmt::format_to(it, "\\{}", ch);
+				it = fmt::format_to(it, "\\{}", (char)ch);
 			}
-			else if (std::isalnum((unsigned char)ch)) {
-				*it++ = ch;
+			else if (ch < 0x80 && std::isalnum((unsigned char)ch)) {
+				*it++ = (char)ch;
 			}
 			else {
-				it = fmt::format_to(it, "[{}]", ch);
+				// Emit as \x{10FFFF}.
+				it = fmt::format_to(it, "\\x{{{:X}}}", (uint32_t)ch);
 			}
 		}
 	}