Skip to content

Commit 355c1fb

Browse files
author
lexasub
committed
ggml : draft commit, replace reallocation of vector for set_tensor by reserve inital vector and use it
1 parent 7a2c913 commit 355c1fb

File tree

1 file changed

+7
-5
lines changed

1 file changed

+7
-5
lines changed

ggml/src/ggml-rpc/ggml-rpc.cpp

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -481,15 +481,16 @@ static void ggml_backend_rpc_buffer_init_tensor(ggml_backend_buffer_t buffer, gg
481481
}
482482

483483
static void ggml_backend_rpc_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) {
484+
static std::vector<uint8_t> input{};
484485
ggml_backend_rpc_buffer_context * ctx = (ggml_backend_rpc_buffer_context *)buffer->context;
485486
// input serialization format: | rpc_tensor | offset (8 bytes) | data (size bytes) |
486487
size_t input_size = sizeof(rpc_tensor) + sizeof(uint64_t) + size;
487-
std::vector<uint8_t> input(input_size, 0);
488+
input.resize(input_size);
488489
rpc_tensor rpc_tensor = serialize_tensor(tensor);
489490
memcpy(input.data(), &rpc_tensor, sizeof(rpc_tensor));
490491
memcpy(input.data() + sizeof(rpc_tensor), &offset, sizeof(offset));
491492
memcpy(input.data() + sizeof(rpc_tensor) + sizeof(offset), data, size);
492-
bool status = send_rpc_cmd(ctx->sock, RPC_CMD_SET_TENSOR, input.data(), input.size(), nullptr, 0);
493+
bool status = send_rpc_cmd(ctx->sock, RPC_CMD_SET_TENSOR, input.data(), input_size, nullptr, 0);
493494
GGML_ASSERT(status);
494495
}
495496

@@ -1148,6 +1149,8 @@ rpc_server::~rpc_server() {
11481149
}
11491150

11501151
static void rpc_serve_client(ggml_backend_t backend, sockfd_t sockfd, size_t free_mem, size_t total_mem) {
1152+
std::vector<uint8_t> set_tensor_vec;
1153+
set_tensor_vec.reserve(100);
11511154
rpc_server server(backend);
11521155
while (true) {
11531156
uint8_t cmd;
@@ -1247,11 +1250,10 @@ static void rpc_serve_client(ggml_backend_t backend, sockfd_t sockfd, size_t fre
12471250
break;
12481251
}
12491252
case RPC_CMD_SET_TENSOR: {
1250-
std::vector<uint8_t> input;
1251-
if (!recv_msg(sockfd, input)) {
1253+
if (!recv_msg(sockfd, set_tensor_vec)) {
12521254
return;
12531255
}
1254-
if (!server.set_tensor(input)) {
1256+
if (!server.set_tensor(set_tensor_vec)) {
12551257
return;
12561258
}
12571259
if (!send_msg(sockfd, nullptr, 0)) {

0 commit comments

Comments
 (0)