Skip to content

Commit d4c774f

Browse files
auxtenauxten
authored
Merge pull request #18 from agoncear-mwb/feature/use-purego
Implement chdb v3.0.0 with pureGo instead of cGo
2 parents 24d9727 + 6a4de22 commit d4c774f

26 files changed

+1016
-827
lines changed

Diff for: ‎.github/workflows/chdb.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ jobs:
3232
run: ./chdb-go "SELECT 12345"
3333

3434
build_mac:
35-
runs-on: macos-12
35+
runs-on: macos-15
3636
steps:
3737
- uses: actions/checkout@v3
3838
- name: Fetch library

Diff for: ‎README.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -67,9 +67,9 @@ func main() {
6767
fmt.Println(result)
6868

6969
tmp_path := filepath.Join(os.TempDir(), "chdb_test")
70-
defer os.RemoveAll(tmp_path)
7170
// Stateful Query (persistent)
7271
session, _ := chdb.NewSession(tmp_path)
72+
// session cleanup will also delete the folder
7373
defer session.Cleanup()
7474

7575
_, err = session.Query("CREATE DATABASE IF NOT EXISTS testdb; " +

Diff for: ‎chdb-purego/binding.go

+62
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
package chdbpurego
2+
3+
import (
4+
"os"
5+
"os/exec"
6+
7+
"github.com/ebitengine/purego"
8+
)
9+
10+
func findLibrary() string {
11+
// Env var
12+
if envPath := os.Getenv("CHDB_LIB_PATH"); envPath != "" {
13+
return envPath
14+
}
15+
16+
// ldconfig with Linux
17+
if path, err := exec.LookPath("libchdb.so"); err == nil {
18+
return path
19+
}
20+
21+
// default path
22+
commonPaths := []string{
23+
"/usr/local/lib/libchdb.so",
24+
"/opt/homebrew/lib/libchdb.dylib",
25+
}
26+
27+
for _, p := range commonPaths {
28+
if _, err := os.Stat(p); err == nil {
29+
return p
30+
}
31+
}
32+
33+
//should be an error ?
34+
return "libchdb.so"
35+
}
36+
37+
var (
38+
queryStable func(argc int, argv []string) *local_result
39+
freeResult func(result *local_result)
40+
queryStableV2 func(argc int, argv []string) *local_result_v2
41+
freeResultV2 func(result *local_result_v2)
42+
connectChdb func(argc int, argv []string) **chdb_conn
43+
closeConn func(conn **chdb_conn)
44+
queryConn func(conn *chdb_conn, query string, format string) *local_result_v2
45+
)
46+
47+
func init() {
48+
path := findLibrary()
49+
libchdb, err := purego.Dlopen(path, purego.RTLD_NOW|purego.RTLD_GLOBAL)
50+
if err != nil {
51+
panic(err)
52+
}
53+
purego.RegisterLibFunc(&queryStable, libchdb, "query_stable")
54+
purego.RegisterLibFunc(&freeResult, libchdb, "free_result")
55+
purego.RegisterLibFunc(&queryStableV2, libchdb, "query_stable_v2")
56+
57+
purego.RegisterLibFunc(&freeResultV2, libchdb, "free_result_v2")
58+
purego.RegisterLibFunc(&connectChdb, libchdb, "connect_chdb")
59+
purego.RegisterLibFunc(&closeConn, libchdb, "close_conn")
60+
purego.RegisterLibFunc(&queryConn, libchdb, "query_conn")
61+
62+
}

Diff for: ‎chdb-purego/chdb.go

+178
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,178 @@
1+
package chdbpurego
2+
3+
import (
4+
"errors"
5+
"fmt"
6+
"unsafe"
7+
)
8+
9+
type result struct {
10+
localResv2 *local_result_v2
11+
}
12+
13+
func newChdbResult(cRes *local_result_v2) ChdbResult {
14+
res := &result{
15+
localResv2: cRes,
16+
}
17+
// runtime.SetFinalizer(res, res.Free)
18+
return res
19+
20+
}
21+
22+
// Buf implements ChdbResult.
23+
func (c *result) Buf() []byte {
24+
if c.localResv2 != nil {
25+
if c.localResv2.buf != nil && c.localResv2.len > 0 {
26+
return unsafe.Slice(c.localResv2.buf, c.localResv2.len)
27+
}
28+
}
29+
return nil
30+
}
31+
32+
// BytesRead implements ChdbResult.
33+
func (c *result) BytesRead() uint64 {
34+
if c.localResv2 != nil {
35+
return c.localResv2.bytes_read
36+
}
37+
return 0
38+
}
39+
40+
// Elapsed implements ChdbResult.
41+
func (c *result) Elapsed() float64 {
42+
if c.localResv2 != nil {
43+
return c.localResv2.elapsed
44+
}
45+
return 0
46+
}
47+
48+
// Error implements ChdbResult.
49+
func (c *result) Error() error {
50+
if c.localResv2 != nil {
51+
if c.localResv2.error_message != nil {
52+
return errors.New(ptrToGoString(c.localResv2.error_message))
53+
}
54+
}
55+
return nil
56+
}
57+
58+
// Free implements ChdbResult.
59+
func (c *result) Free() {
60+
if c.localResv2 != nil {
61+
freeResultV2(c.localResv2)
62+
c.localResv2 = nil
63+
}
64+
65+
}
66+
67+
// Len implements ChdbResult.
68+
func (c *result) Len() int {
69+
if c.localResv2 != nil {
70+
return int(c.localResv2.len)
71+
}
72+
return 0
73+
}
74+
75+
// RowsRead implements ChdbResult.
76+
func (c *result) RowsRead() uint64 {
77+
if c.localResv2 != nil {
78+
return c.localResv2.rows_read
79+
}
80+
return 0
81+
}
82+
83+
// String implements ChdbResult.
84+
func (c *result) String() string {
85+
ret := c.Buf()
86+
if ret == nil {
87+
return ""
88+
}
89+
return string(ret)
90+
}
91+
92+
type connection struct {
93+
conn **chdb_conn
94+
}
95+
96+
func newChdbConn(conn **chdb_conn) ChdbConn {
97+
c := &connection{
98+
conn: conn,
99+
}
100+
// runtime.SetFinalizer(c, c.Close)
101+
return c
102+
}
103+
104+
// Close implements ChdbConn.
105+
func (c *connection) Close() {
106+
if c.conn != nil {
107+
closeConn(c.conn)
108+
}
109+
}
110+
111+
// Query implements ChdbConn.
112+
func (c *connection) Query(queryStr string, formatStr string) (result ChdbResult, err error) {
113+
114+
if c.conn == nil {
115+
return nil, fmt.Errorf("invalid connection")
116+
}
117+
118+
rawConn := *c.conn
119+
120+
res := queryConn(rawConn, queryStr, formatStr)
121+
if res == nil {
122+
// According to the C ABI of chDB v1.2.0, the C function query_stable_v2
123+
// returns nil if the query returns no data. This is not an error. We
124+
// will change this behavior in the future.
125+
return newChdbResult(res), nil
126+
}
127+
if res.error_message != nil {
128+
return nil, errors.New(ptrToGoString(res.error_message))
129+
}
130+
131+
return newChdbResult(res), nil
132+
}
133+
134+
func (c *connection) Ready() bool {
135+
if c.conn != nil {
136+
deref := *c.conn
137+
if deref != nil {
138+
return deref.connected
139+
}
140+
}
141+
return false
142+
}
143+
144+
// Session will keep the state of query.
145+
// If path is None, it will create a temporary directory and use it as the database path
146+
// and the temporary directory will be removed when the session is closed.
147+
// You can also pass in a path to create a database at that path where will keep your data.
148+
//
149+
// You can also use a connection string to pass in the path and other parameters.
150+
// Examples:
151+
// - ":memory:" (for in-memory database)
152+
// - "test.db" (for relative path)
153+
// - "file:test.db" (same as above)
154+
// - "/path/to/test.db" (for absolute path)
155+
// - "file:/path/to/test.db" (same as above)
156+
// - "file:test.db?param1=value1&param2=value2" (for relative path with query params)
157+
// - "file::memory:?verbose&log-level=test" (for in-memory database with query params)
158+
// - "///path/to/test.db?param1=value1&param2=value2" (for absolute path)
159+
//
160+
// Connection string args handling:
161+
//
162+
// Connection string can contain query params like "file:test.db?param1=value1&param2=value2"
163+
// "param1=value1" will be passed to ClickHouse engine as start up args.
164+
//
165+
// For more details, see `clickhouse local --help --verbose`
166+
// Some special args handling:
167+
// - "mode=ro" would be "--readonly=1" for clickhouse (read-only mode)
168+
//
169+
// Important:
170+
// - There can be only one session at a time. If you want to create a new session, you need to close the existing one.
171+
// - Creating a new session will close the existing one.
172+
func NewConnection(argc int, argv []string) (ChdbConn, error) {
173+
conn := connectChdb(argc, argv)
174+
if conn == nil {
175+
return nil, fmt.Errorf("could not create a chdb connection")
176+
}
177+
return newChdbConn(conn), nil
178+
}

Diff for: ‎chdb-purego/helpers.go

+21
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
package chdbpurego
2+
3+
import (
4+
"unsafe"
5+
)
6+
7+
func ptrToGoString(ptr *byte) string {
8+
if ptr == nil {
9+
return ""
10+
}
11+
12+
var length int
13+
for {
14+
if *(*byte)(unsafe.Pointer(uintptr(unsafe.Pointer(ptr)) + uintptr(length))) == 0 {
15+
break
16+
}
17+
length++
18+
}
19+
20+
return string(unsafe.Slice(ptr, length))
21+
}

Diff for: ‎chdb-purego/types.go

+60
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
package chdbpurego
2+
3+
import "unsafe"
4+
5+
// old local result struct. for reference:
6+
// https://github.com/chdb-io/chdb/blob/main/programs/local/chdb.h#L29
7+
type local_result struct {
8+
buf *byte
9+
len uintptr
10+
_vec unsafe.Pointer
11+
elapsed float64
12+
rows_read uint64
13+
bytes_read uint64
14+
}
15+
16+
// new local result struct. for reference: https://github.com/chdb-io/chdb/blob/main/programs/local/chdb.h#L40
17+
type local_result_v2 struct {
18+
buf *byte
19+
len uintptr
20+
_vec unsafe.Pointer
21+
elapsed float64
22+
rows_read uint64
23+
bytes_read uint64
24+
error_message *byte
25+
}
26+
27+
// clickhouse background server connection.for reference: https://github.com/chdb-io/chdb/blob/main/programs/local/chdb.h#L82
28+
type chdb_conn struct {
29+
server unsafe.Pointer
30+
connected bool
31+
queue unsafe.Pointer
32+
}
33+
34+
type ChdbResult interface {
35+
// Raw bytes result buffer, used for reading the result of clickhouse query
36+
Buf() []byte
37+
// String rapresentation of the the buffer
38+
String() string
39+
// Lenght in bytes of the buffer
40+
Len() int
41+
// Number of seconds elapsed for the query execution
42+
Elapsed() float64
43+
// Amount of rows returned by the query
44+
RowsRead() uint64
45+
// Amount of bytes returned by the query
46+
BytesRead() uint64
47+
// If the query had any error during execution, here you can retrieve the cause.
48+
Error() error
49+
// Free the query result and all the allocated memory
50+
Free()
51+
}
52+
53+
type ChdbConn interface {
54+
//Query executes the given queryStr in the underlying clickhouse connection, and output the result in the given formatStr
55+
Query(queryStr string, formatStr string) (result ChdbResult, err error)
56+
//Ready returns a boolean indicating if the connections is successfully established.
57+
Ready() bool
58+
//Close the connection and free the underlying allocated memory
59+
Close()
60+
}

0 commit comments

Comments
 (0)