Skip to content

Commit 8015a57

Browse files
gropaulNiclasHaderer
gropaul
and
NiclasHaderer
authored
New Result Serialization (#26)
* added result serializer * fixed some bugs for exotic types * use internal function for type names * added missing cmath include * Added e2e tests * Cleanup * Moved up clang-format and tidy config so IDE and make format picks up on the right formatting rules * Moved most logic into new compact serializer * Added some docs for contributing * Ignore pycache * Cleanup --------- Co-authored-by: Niclas Haderer <[email protected]>
1 parent bd381c1 commit 8015a57

18 files changed

+784
-125
lines changed

Diff for: ‎.clang-format

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
duckdb/.clang-format

Diff for: ‎.clang-tidy

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
duckdb/.clang-tidy

Diff for: ‎.gitignore

+2
Original file line numberDiff line numberDiff line change
@@ -6,3 +6,5 @@ duckdb_unittest_tempdir/
66
testext
77
test/python/__pycache__/
88
.Rhistory
9+
__pycache__
10+
venv

Diff for: ‎CMakeLists.txt

+11-14
Original file line numberDiff line numberDiff line change
@@ -6,24 +6,20 @@ set(EXTENSION_NAME ${TARGET_NAME}_extension)
66
set(LOADABLE_EXTENSION_NAME ${TARGET_NAME}_loadable_extension)
77

88
project(${TARGET_NAME})
9-
include_directories(
10-
src/include
11-
${CMAKE_CURRENT_BINARY_DIR}
12-
duckdb/third_party/httplib
13-
duckdb/parquet/include
14-
)
9+
include_directories(src/include ${CMAKE_CURRENT_BINARY_DIR}
10+
duckdb/third_party/httplib duckdb/parquet/include)
1511

1612
# Embed ./src/assets/index.html as a C++ header
1713
add_custom_command(
1814
OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/playground.hpp
19-
COMMAND ${CMAKE_COMMAND} -P ${PROJECT_SOURCE_DIR}/embed.cmake ${PROJECT_SOURCE_DIR}/src/assets/index.html ${CMAKE_CURRENT_BINARY_DIR}/playground.hpp playgroundContent
20-
DEPENDS ${PROJECT_SOURCE_DIR}/src/assets/index.html
21-
)
15+
COMMAND
16+
${CMAKE_COMMAND} -P ${PROJECT_SOURCE_DIR}/embed.cmake
17+
${PROJECT_SOURCE_DIR}/src/assets/index.html
18+
${CMAKE_CURRENT_BINARY_DIR}/playground.hpp playgroundContent
19+
DEPENDS ${PROJECT_SOURCE_DIR}/src/assets/index.html)
2220

23-
set(EXTENSION_SOURCES
24-
src/httpserver_extension.cpp
25-
${CMAKE_CURRENT_BINARY_DIR}/playground.hpp
26-
)
21+
set(EXTENSION_SOURCES src/httpserver_extension.cpp src/result_serializer.cpp
22+
${CMAKE_CURRENT_BINARY_DIR}/playground.hpp)
2723

2824
if(MINGW)
2925
set(OPENSSL_USE_STATIC_LIBS TRUE)
@@ -36,7 +32,8 @@ build_static_extension(${TARGET_NAME} ${EXTENSION_SOURCES})
3632
build_loadable_extension(${TARGET_NAME} " " ${EXTENSION_SOURCES})
3733

3834
include_directories(${OPENSSL_INCLUDE_DIR})
39-
target_link_libraries(${LOADABLE_EXTENSION_NAME} duckdb_mbedtls ${OPENSSL_LIBRARIES})
35+
target_link_libraries(${LOADABLE_EXTENSION_NAME} duckdb_mbedtls
36+
${OPENSSL_LIBRARIES})
4037
target_link_libraries(${EXTENSION_NAME} duckdb_mbedtls ${OPENSSL_LIBRARIES})
4138

4239
if(MINGW)

Diff for: ‎docs/README.md

+37
Original file line numberDiff line numberDiff line change
@@ -203,6 +203,43 @@ Check out this flocking macro from fellow _Italo-Amsterdammer_ @carlopi @ DuckDB
203203
204204
<br>
205205
206+
## Development
207+
208+
### Cloning the Repository
209+
210+
Clone the repository and all its submodules
211+
212+
```bash
213+
git clone <your-fork-url>
214+
git submodule update --init --recursive
215+
```
216+
217+
### Setting up CLion
218+
**Opening project:**
219+
Configuring CLion with the extension template requires a little work. Firstly, make sure that the DuckDB submodule is available.
220+
Then make sure to open `./duckdb/CMakeLists.txt` (so not the top level `CMakeLists.txt` file from this repo) as a project in CLion.
221+
Now to fix your project path go to `tools->CMake->Change Project Root`([docs](https://www.jetbrains.com/help/clion/change-project-root-directory.html)) to set the project root to the root dir of this repo.
222+
223+
**Debugging:**
224+
To set up debugging in CLion, there are two simple steps required. Firstly, in `CLion -> Settings / Preferences -> Build, Execution, Deploy -> CMake` you will need to add the desired builds (e.g. Debug, Release, RelDebug, etc). There's different ways to configure this, but the easiest is to leave all empty, except the `build path`, which needs to be set to `../build/{build type}`. Now on a clean repository you will first need to run `make {build type}` to initialize the CMake build directory. After running make, you will be able to (re)build from CLion by using the build target we just created. If you use the CLion editor, you can create a CLion CMake profiles matching the CMake variables that are described in the makefile, and then you don't need to invoke the Makefile.
225+
226+
The second step is to configure the unittest runner as a run/debug configuration. To do this, go to `Run -> Edit Configurations` and click `+ -> Cmake Application`. The target and executable should be `unittest`. This will run all the DuckDB tests. To specify only running the extension specific tests, add `--test-dir ../../.. [sql]` to the `Program Arguments`. Note that it is recommended to use the `unittest` executable for testing/development within CLion. The actual DuckDB CLI currently does not reliably work as a run target in CLion.
227+
228+
229+
### Testing
230+
231+
To run the E2E test install all packages necessary:
232+
233+
```bash
234+
pip install -r requirements.txt
235+
```
236+
237+
Then run the test suite:
238+
239+
```bash
240+
pytest pytest test_http_api
241+
```
242+
206243
##### :black_joker: Disclaimers
207244
208245
[^1]: DuckDB ® is a trademark of DuckDB Foundation. All rights reserved by their respective owners. [^1]

Diff for: ‎requirements.txt

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
httpx==0.28.1
2+
pytest==8.3.4

Diff for: ‎src/httpserver_extension.cpp

+17-110
Original file line numberDiff line numberDiff line change
@@ -1,32 +1,30 @@
11
#define DUCKDB_EXTENSION_MAIN
2+
#define CPPHTTPLIB_OPENSSL_SUPPORT
3+
4+
#include <chrono>
5+
#include <cstdlib>
6+
#include <thread>
27
#include "httpserver_extension.hpp"
8+
#include "query_stats.hpp"
39
#include "duckdb.hpp"
410
#include "duckdb/common/exception.hpp"
5-
#include "duckdb/common/string_util.hpp"
611
#include "duckdb/function/scalar_function.hpp"
712
#include "duckdb/main/extension_util.hpp"
8-
#include "duckdb/common/atomic.hpp"
9-
#include "duckdb/common/exception/http_exception.hpp"
1013
#include "duckdb/common/allocator.hpp"
11-
#include <chrono>
12-
#include <thread>
13-
#include <memory>
14-
#include <cstdlib>
15-
16-
#ifndef _WIN32
17-
#include <syslog.h>
18-
#endif
19-
20-
#define CPPHTTPLIB_OPENSSL_SUPPORT
14+
#include "result_serializer.hpp"
15+
#include "result_serializer_compact_json.hpp"
2116
#include "httplib.hpp"
2217
#include "yyjson.hpp"
23-
2418
#include "playground.hpp"
2519

26-
using namespace duckdb_yyjson; // NOLINT
20+
#ifndef _WIN32
21+
#include <syslog.h>
22+
#endif
2723

2824
namespace duckdb {
2925

26+
using namespace duckdb_yyjson; // NOLINT(*-build-using-namespace)
27+
3028
struct HttpServerState {
3129
std::unique_ptr<duckdb_httplib_openssl::Server> server;
3230
std::unique_ptr<std::thread> server_thread;
@@ -40,98 +38,6 @@ struct HttpServerState {
4038

4139
static HttpServerState global_state;
4240

43-
std::string GetColumnType(MaterializedQueryResult &result, idx_t column) {
44-
if (result.RowCount() == 0) {
45-
return "String";
46-
}
47-
switch (result.types[column].id()) {
48-
case LogicalTypeId::FLOAT:
49-
return "Float";
50-
case LogicalTypeId::DOUBLE:
51-
return "Double";
52-
case LogicalTypeId::INTEGER:
53-
return "Int32";
54-
case LogicalTypeId::BIGINT:
55-
return "Int64";
56-
case LogicalTypeId::UINTEGER:
57-
return "UInt32";
58-
case LogicalTypeId::UBIGINT:
59-
return "UInt64";
60-
case LogicalTypeId::VARCHAR:
61-
return "String";
62-
case LogicalTypeId::TIME:
63-
return "DateTime";
64-
case LogicalTypeId::DATE:
65-
return "Date";
66-
case LogicalTypeId::TIMESTAMP:
67-
return "DateTime";
68-
case LogicalTypeId::BOOLEAN:
69-
return "Int8";
70-
default:
71-
return "String";
72-
}
73-
return "String";
74-
}
75-
76-
struct ReqStats {
77-
float elapsed_sec;
78-
int64_t read_bytes;
79-
int64_t read_rows;
80-
};
81-
82-
// Convert the query result to JSON format
83-
static std::string ConvertResultToJSON(MaterializedQueryResult &result, ReqStats &req_stats) {
84-
auto doc = yyjson_mut_doc_new(nullptr);
85-
auto root = yyjson_mut_obj(doc);
86-
yyjson_mut_doc_set_root(doc, root);
87-
// Add meta information
88-
auto meta_array = yyjson_mut_arr(doc);
89-
for (idx_t col = 0; col < result.ColumnCount(); ++col) {
90-
auto column_obj = yyjson_mut_obj(doc);
91-
yyjson_mut_obj_add_str(doc, column_obj, "name", result.ColumnName(col).c_str());
92-
yyjson_mut_arr_append(meta_array, column_obj);
93-
std::string tp(GetColumnType(result, col));
94-
yyjson_mut_obj_add_strcpy(doc, column_obj, "type", tp.c_str());
95-
}
96-
yyjson_mut_obj_add_val(doc, root, "meta", meta_array);
97-
98-
// Add data
99-
auto data_array = yyjson_mut_arr(doc);
100-
for (idx_t row = 0; row < result.RowCount(); ++row) {
101-
auto row_array = yyjson_mut_arr(doc);
102-
for (idx_t col = 0; col < result.ColumnCount(); ++col) {
103-
Value value = result.GetValue(col, row);
104-
if (value.IsNull()) {
105-
yyjson_mut_arr_append(row_array, yyjson_mut_null(doc));
106-
} else {
107-
std::string value_str = value.ToString();
108-
yyjson_mut_arr_append(row_array, yyjson_mut_strncpy(doc, value_str.c_str(), value_str.length()));
109-
}
110-
}
111-
yyjson_mut_arr_append(data_array, row_array);
112-
}
113-
yyjson_mut_obj_add_val(doc, root, "data", data_array);
114-
115-
// Add row count
116-
yyjson_mut_obj_add_int(doc, root, "rows", result.RowCount());
117-
//"statistics":{"elapsed":0.00031403,"rows_read":1,"bytes_read":0}}
118-
auto stat_obj = yyjson_mut_obj_add_obj(doc, root, "statistics");
119-
yyjson_mut_obj_add_real(doc, stat_obj, "elapsed", req_stats.elapsed_sec);
120-
yyjson_mut_obj_add_int(doc, stat_obj, "rows_read", req_stats.read_rows);
121-
yyjson_mut_obj_add_int(doc, stat_obj, "bytes_read", req_stats.read_bytes);
122-
// Write to string
123-
auto data = yyjson_mut_write(doc, 0, nullptr);
124-
if (!data) {
125-
yyjson_mut_doc_free(doc);
126-
throw InternalException("Failed to render the result as JSON, yyjson failed");
127-
}
128-
129-
std::string json_output(data);
130-
free(data);
131-
yyjson_mut_doc_free(doc);
132-
return json_output;
133-
}
134-
13541
// New: Base64 decoding function
13642
std::string base64_decode(const std::string &in) {
13743
std::string out;
@@ -300,7 +206,8 @@ void HandleHttpRequest(const duckdb_httplib_openssl::Request& req, duckdb_httpli
300206
std::string json_output = ConvertResultToNDJSON(*result);
301207
res.set_content(json_output, "application/x-ndjson");
302208
} else if (format == "JSONCompact") {
303-
std::string json_output = ConvertResultToJSON(*result, stats);
209+
ResultSerializerCompactJson serializer;
210+
std::string json_output = serializer.Serialize(*result, stats);
304211
res.set_content(json_output, "application/json");
305212
} else {
306213
// Default to NDJSON for DuckDB's own queries
@@ -325,9 +232,9 @@ void HttpServerStart(DatabaseInstance& db, string_t host, int32_t port, string_t
325232
global_state.is_running = true;
326233
global_state.auth_token = auth.GetString();
327234

328-
// Custom basepath, defaults to root /
235+
// Custom basepath, defaults to root /
329236
const char* base_path_env = std::getenv("DUCKDB_HTTPSERVER_BASEPATH");
330-
std::string base_path = "/";
237+
std::string base_path = "/";
331238

332239
if (base_path_env && base_path_env[0] == '/' && strlen(base_path_env) > 1) {
333240
base_path = std::string(base_path_env);

Diff for: ‎src/include/httpserver_extension.hpp

-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
#pragma once
22

33
#include "duckdb.hpp"
4-
#include "duckdb/common/file_system.hpp"
54

65
namespace duckdb {
76

Diff for: ‎src/include/query_stats.hpp

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
#pragma once
2+
#include <cstdint>
3+
4+
namespace duckdb {
5+
6+
struct ReqStats {
7+
float elapsed_sec;
8+
uint64_t read_bytes;
9+
uint64_t read_rows;
10+
};
11+
12+
} // namespace duckdb

Diff for: ‎src/include/result_serializer.hpp

+46
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
#pragma once
2+
3+
#include "duckdb/main/query_result.hpp"
4+
#include "yyjson.hpp"
5+
6+
namespace duckdb {
7+
using namespace duckdb_yyjson; // NOLINT(*-build-using-namespace)
8+
9+
class ResultSerializer {
10+
public:
11+
explicit ResultSerializer(const bool _set_invalid_values_to_null = false)
12+
: set_invalid_values_to_null(_set_invalid_values_to_null) {
13+
doc = yyjson_mut_doc_new(nullptr);
14+
}
15+
16+
virtual ~ResultSerializer() {
17+
yyjson_mut_doc_free(doc);
18+
}
19+
20+
std::string YY_ToString() {
21+
auto data = yyjson_mut_write(doc, 0, nullptr);
22+
if (!data) {
23+
throw SerializationException("Could not render yyjson document");
24+
}
25+
std::string json_output(data);
26+
free(data);
27+
return json_output;
28+
}
29+
30+
protected:
31+
void SerializeInternal(QueryResult &query_result, yyjson_mut_val *append_root, bool values_as_array);
32+
33+
void SerializeChunk(const DataChunk &chunk, vector<string> &names, vector<LogicalType> &types,
34+
yyjson_mut_val *append_root, bool values_as_array);
35+
36+
yyjson_mut_val *SerializeRowAsArray(const DataChunk &chunk, idx_t row_idx, vector<LogicalType> &types);
37+
38+
yyjson_mut_val *SerializeRowAsObject(const DataChunk &chunk, idx_t row_idx, vector<string> &names,
39+
vector<LogicalType> &types);
40+
41+
void SerializeValue(yyjson_mut_val *parent, const Value &value, optional_ptr<string> name, const LogicalType &type);
42+
43+
yyjson_mut_doc *doc;
44+
bool set_invalid_values_to_null;
45+
};
46+
} // namespace duckdb

0 commit comments

Comments
 (0)