Skip to content

Commit 1d65126

Browse files
turuslanturuslan
authored
Fix graphsync encoding (#161)
Signed-off-by: turuslan <[email protected]>
1 parent b0bde74 commit 1d65126

File tree

14 files changed

+172
-93
lines changed

14 files changed

+172
-93
lines changed

‎core/adt/address_key.cpp

+2-3
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
*/
55

66
#include "adt/address_key.hpp"
7+
#include "common/span.hpp"
78

89
#include "primitives/address/address_codec.hpp"
910

@@ -15,8 +16,6 @@ namespace fc::adt {
1516

1617
outcome::result<AddressKeyer::Key> AddressKeyer::decode(
1718
const std::string &key) {
18-
// NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast)
19-
return primitives::address::decode(gsl::make_span(
20-
reinterpret_cast<const uint8_t *>(key.data()), key.size()));
19+
return primitives::address::decode(common::span::cbytes(key));
2120
}
2221
} // namespace fc::adt

‎core/adt/uvarint_key.cpp

+2-4
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
*/
55

66
#include "adt/uvarint_key.hpp"
7+
#include "common/span.hpp"
78

89
#include <libp2p/multi/uvarint.hpp>
910

@@ -26,10 +27,7 @@ namespace fc::adt {
2627

2728
outcome::result<UvarintKeyer::Key> UvarintKeyer::decode(
2829
const std::string &key) {
29-
auto maybe = UVarint::create(gsl::make_span(
30-
// NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast)
31-
reinterpret_cast<const uint8_t *>(key.data()),
32-
key.size()));
30+
auto maybe = UVarint::create(common::span::cbytes(key));
3331
if (!maybe) {
3432
return UvarintKeyError::DECODE_ERROR;
3533
}

‎core/codec/uvarint.hpp

+39
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
/**
2+
* Copyright Soramitsu Co., Ltd. All Rights Reserved.
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
#ifndef CPP_FILECOIN_CODEC_UVARINT_HPP
7+
#define CPP_FILECOIN_CODEC_UVARINT_HPP
8+
9+
#include <gsl/span>
10+
#include <libp2p/multi/uvarint.hpp>
11+
12+
#include "common/outcome.hpp"
13+
14+
namespace fc::codec::uvarint {
15+
using Input = gsl::span<const uint8_t>;
16+
17+
template <auto error, typename T = uint64_t>
18+
outcome::result<T> read(Input &input) {
19+
auto value = libp2p::multi::UVarint::create(input);
20+
if (!value) {
21+
return error;
22+
}
23+
input = input.subspan(value->size());
24+
return static_cast<T>(value->toUInt64());
25+
}
26+
27+
template <auto error_length, auto error_data>
28+
outcome::result<Input> readBytes(Input &input) {
29+
OUTCOME_TRY(size, read<error_length>(input));
30+
if (input.size() < static_cast<ptrdiff_t>(size)) {
31+
return error_data;
32+
}
33+
auto result = input.subspan(0, size);
34+
input = input.subspan(size);
35+
return result;
36+
}
37+
} // namespace fc::codec::uvarint
38+
39+
#endif // CPP_FILECOIN_CODEC_UVARINT_HPP

‎core/common/span.hpp

+31
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
/**
2+
* Copyright Soramitsu Co., Ltd. All Rights Reserved.
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
#ifndef CPP_FILECOIN_CORE_COMMON_SPAN_HPP
7+
#define CPP_FILECOIN_CORE_COMMON_SPAN_HPP
8+
9+
#include <gsl/span>
10+
11+
namespace fc::common::span {
12+
template <typename To, typename From>
13+
constexpr auto cast(gsl::span<From> span) {
14+
// NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast)
15+
return gsl::make_span(reinterpret_cast<To *>(span.data()), span.size());
16+
}
17+
18+
constexpr auto cbytes(gsl::span<const char> span) {
19+
return cast<const uint8_t>(span);
20+
}
21+
22+
constexpr auto cstring(gsl::span<const uint8_t> span) {
23+
return cast<const char>(span);
24+
}
25+
26+
constexpr auto string(gsl::span<uint8_t> span) {
27+
return cast<char>(span);
28+
}
29+
} // namespace fc::common::span
30+
31+
#endif // CPP_FILECOIN_CORE_COMMON_SPAN_HPP

‎core/primitives/cid/cid.cpp

+57-10
Original file line numberDiff line numberDiff line change
@@ -7,22 +7,23 @@
77

88
#include <libp2p/multi/content_identifier_codec.hpp>
99

10+
#include "codec/uvarint.hpp"
1011
#include "crypto/blake2/blake2b160.hpp"
1112

13+
using Multicodec = libp2p::multi::MulticodecType::Code;
14+
using libp2p::multi::HashType;
15+
using libp2p::multi::Multihash;
16+
1217
namespace fc {
13-
CID::CID()
14-
: ContentIdentifier(
15-
{}, {}, libp2p::multi::Multihash::create({}, {}).value()) {}
18+
CID::CID() : ContentIdentifier({}, {}, Multihash::create({}, {}).value()) {}
1619

1720
CID::CID(const ContentIdentifier &cid) : ContentIdentifier(cid) {}
1821

1922
CID::CID(ContentIdentifier &&cid) noexcept
2023
: ContentIdentifier(
2124
cid.version, cid.content_type, std::move(cid.content_address)) {}
2225

23-
CID::CID(Version version,
24-
libp2p::multi::MulticodecType::Code content_type,
25-
libp2p::multi::Multihash content_address)
26+
CID::CID(Version version, Multicodec content_type, Multihash content_address)
2627
: ContentIdentifier(version, content_type, std::move(content_address)) {}
2728

2829
CID &CID::operator=(CID &&cid) noexcept {
@@ -67,14 +68,60 @@ namespace fc {
6768
OUTCOME_TRY(cid, libp2p::multi::ContentIdentifierCodec::decode(input));
6869
return CID{std::move(cid)};
6970
}
71+
72+
outcome::result<CID> CID::read(gsl::span<const uint8_t> &input, bool prefix) {
73+
using Error = libp2p::multi::ContentIdentifierCodec::DecodeError;
74+
CID cid;
75+
if (input.size() >= 2 && HashType{input[0]} == HashType::sha256
76+
&& input[1] == 32) {
77+
cid.version = Version::V0;
78+
cid.content_type = Multicodec::DAG_PB;
79+
if (!prefix && input.size() < 34) {
80+
return Multihash::Error::INCONSISTENT_LENGTH;
81+
}
82+
} else {
83+
OUTCOME_TRY(version,
84+
codec::uvarint::read<Error::EMPTY_VERSION, Version>(input));
85+
if (version <= Version::V0) {
86+
return Error::MALFORMED_VERSION;
87+
}
88+
if (version != Version::V1) {
89+
return Error::RESERVED_VERSION;
90+
}
91+
cid.version = version;
92+
OUTCOME_TRY(
93+
codec,
94+
codec::uvarint::read<Error::EMPTY_MULTICODEC, Multicodec>(input));
95+
cid.content_type = codec;
96+
}
97+
98+
OUTCOME_TRY(
99+
hash_type,
100+
codec::uvarint::read<Multihash::Error::ZERO_INPUT_LENGTH, HashType>(
101+
input));
102+
OUTCOME_TRY(hash_size,
103+
codec::uvarint::read<Multihash::Error::INPUT_TOO_SHORT>(input));
104+
gsl::span<const uint8_t> hash_span;
105+
if (prefix) {
106+
static const uint8_t empty[Multihash::kMaxHashLength]{};
107+
hash_span = gsl::make_span(empty, hash_size);
108+
} else {
109+
if (input.size() < static_cast<ptrdiff_t>(hash_size)) {
110+
return Multihash::Error::INCONSISTENT_LENGTH;
111+
}
112+
hash_span = input.subspan(0, hash_size);
113+
input = input.subspan(hash_size);
114+
}
115+
OUTCOME_TRY(hash, Multihash::create(hash_type, hash_span));
116+
cid.content_address = std::move(hash);
117+
return cid;
118+
}
70119
} // namespace fc
71120

72121
namespace fc::common {
73122
outcome::result<CID> getCidOf(gsl::span<const uint8_t> bytes) {
74123
auto hash_raw = crypto::blake2b::blake2b_256(bytes);
75-
OUTCOME_TRY(hash,
76-
libp2p::multi::Multihash::create(
77-
libp2p::multi::HashType::blake2b_256, hash_raw));
78-
return CID(CID::Version::V1, libp2p::multi::MulticodecType::DAG_CBOR, hash);
124+
OUTCOME_TRY(hash, Multihash::create(HashType::blake2b_256, hash_raw));
125+
return CID(CID::Version::V1, Multicodec::DAG_CBOR, hash);
79126
}
80127
} // namespace fc::common

‎core/primitives/cid/cid.hpp

+3
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,9 @@ namespace fc {
6060
static outcome::result<CID> fromString(const std::string &str);
6161

6262
static outcome::result<CID> fromBytes(gsl::span<const uint8_t> input);
63+
64+
static outcome::result<CID> read(gsl::span<const uint8_t> &input,
65+
bool prefix = false);
6366
};
6467
} // namespace fc
6568

‎core/storage/car/car.cpp

+8-42
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55

66
#include "storage/car/car.hpp"
77

8-
#include <libp2p/multi/uvarint.hpp>
8+
#include "codec/uvarint.hpp"
99

1010
OUTCOME_CPP_DEFINE_CATEGORY(fc::storage::car, CarError, e) {
1111
using E = fc::storage::car::CarError;
@@ -16,50 +16,16 @@ OUTCOME_CPP_DEFINE_CATEGORY(fc::storage::car, CarError, e) {
1616
}
1717

1818
namespace fc::storage::car {
19-
using libp2p::multi::UVarint;
20-
21-
outcome::result<uint64_t> readUvarint(Input &input) {
22-
auto value = UVarint::create(input);
23-
if (!value) {
24-
return CarError::DECODE_ERROR;
25-
}
26-
input = input.subspan(UVarint::calculateSize(input));
27-
return value->toUInt64();
28-
}
29-
30-
outcome::result<Input> readUvarintBytes(Input &input) {
31-
OUTCOME_TRY(size, readUvarint(input));
32-
if (input.size() < static_cast<ptrdiff_t>(size)) {
33-
return CarError::DECODE_ERROR;
34-
}
35-
auto result = input.subspan(0, size);
36-
input = input.subspan(size);
37-
return result;
38-
}
39-
40-
outcome::result<CID> readCid(Input &input) {
41-
auto input2 = input;
42-
if (input.size() >= 2 && input[0] == 12 && input[1] == 32) {
43-
if (input.size() < 34) {
44-
return CarError::DECODE_ERROR;
45-
}
46-
} else {
47-
OUTCOME_TRY(readUvarint(input));
48-
OUTCOME_TRY(readUvarint(input));
49-
}
50-
OUTCOME_TRY(readUvarint(input));
51-
OUTCOME_TRY(readUvarintBytes(input));
52-
OUTCOME_TRY(
53-
cid, CID::fromBytes(input2.subspan(0, input2.size() - input.size())));
54-
return std::move(cid);
55-
}
56-
5719
outcome::result<std::vector<CID>> loadCar(Ipld &store, Input input) {
58-
OUTCOME_TRY(header_bytes, readUvarintBytes(input));
20+
OUTCOME_TRY(header_bytes,
21+
codec::uvarint::readBytes<CarError::DECODE_ERROR,
22+
CarError::DECODE_ERROR>(input));
5923
OUTCOME_TRY(header, codec::cbor::decode<CarHeader>(header_bytes));
6024
while (!input.empty()) {
61-
OUTCOME_TRY(node, readUvarintBytes(input));
62-
OUTCOME_TRY(cid, readCid(node));
25+
OUTCOME_TRY(node,
26+
codec::uvarint::readBytes<CarError::DECODE_ERROR,
27+
CarError::DECODE_ERROR>(input));
28+
OUTCOME_TRY(cid, CID::read(node));
6329
OUTCOME_TRY(store.set(cid, common::Buffer{node}));
6430
}
6531
return std::move(header.roots);

‎core/storage/ipfs/graphsync/impl/CMakeLists.txt

+1
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ target_link_libraries(graphsync
3333
cid
3434
buffer
3535
cbor
36+
filecoin_hasher
3637
logger
3738
graphsync_proto
3839
)

‎core/storage/ipfs/graphsync/impl/graphsync_impl.cpp

+6
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@
1111
#include "network/network.hpp"
1212

1313
namespace fc::storage::ipfs::graphsync {
14+
/// Selector that matches current node
15+
common::Buffer kSelectorMatcher{0xa1, 0x61, 0x2e, 0xa0};
1416

1517
GraphsyncImpl::GraphsyncImpl(
1618
std::shared_ptr<libp2p::Host> host,
@@ -65,6 +67,10 @@ namespace fc::storage::ipfs::graphsync {
6567
return local_requests_->newRejectedRequest(std::move(callback));
6668
}
6769

70+
if (selector.empty()) {
71+
selector = kSelectorMatcher;
72+
}
73+
6874
auto newRequest = local_requests_->newRequest(
6975
root_cid, selector, extensions, std::move(callback));
7076

‎core/storage/ipfs/graphsync/impl/network/marshalling/message_parser.cpp

+14-25
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@
66
#include "message_parser.hpp"
77

88
#include "codec/cbor/cbor_decode_stream.hpp"
9+
#include "common/span.hpp"
10+
#include "crypto/hasher/hasher.hpp"
911

1012
#include "protobuf/message.pb.h"
1113

@@ -14,14 +16,8 @@ namespace fc::storage::ipfs::graphsync {
1416
using codec::cbor::CborDecodeStream;
1517

1618
// Dummy protobuf helper, string->bytes
17-
common::Buffer fromString(const std::string &src) {
18-
common::Buffer dst;
19-
if (!src.empty()) {
20-
auto b = (const uint8_t *)src.data();
21-
auto e = b + src.size();
22-
dst.putBytes(b, e);
23-
}
24-
return dst;
19+
inline auto fromString(const std::string &src) {
20+
return common::Buffer{common::span::cbytes(src)};
2521
}
2622

2723
// Checks status code received from wire
@@ -51,20 +47,6 @@ namespace fc::storage::ipfs::graphsync {
5147
return Error::MESSAGE_PARSE_ERROR;
5248
}
5349

54-
// Decodes CID from CBOR source
55-
outcome::result<CID> decodeCid(const std::string &s) {
56-
CID cid;
57-
try {
58-
auto data = (const uint8_t *)s.data(); // NOLINT
59-
CborDecodeStream decoder(gsl::span<const uint8_t>(data, s.size()));
60-
decoder >> cid;
61-
} catch (const std::exception &e) {
62-
logger()->warn("{}: {}", __FUNCTION__, e.what());
63-
return Error::MESSAGE_PARSE_ERROR;
64-
}
65-
return cid;
66-
}
67-
6850
// Extracts requests from protobuf message
6951
outcome::result<void> parseRequests(pb::Message &pb_msg, Message &msg) {
7052
auto sz = pb_msg.requests_size();
@@ -76,7 +58,7 @@ namespace fc::storage::ipfs::graphsync {
7658
if (src.cancel()) {
7759
dst.cancel = true;
7860
} else {
79-
OUTCOME_TRY(cid, decodeCid(src.root()));
61+
OUTCOME_TRY(cid, CID::fromBytes(common::span::cbytes(src.root())));
8062
dst.root_cid = std::move(cid);
8163
dst.selector = fromString(src.selector());
8264
dst.priority = src.priority();
@@ -119,8 +101,15 @@ namespace fc::storage::ipfs::graphsync {
119101
msg.data.reserve(sz);
120102

121103
for (auto &src : pb_msg.data()) {
122-
OUTCOME_TRY(cid, decodeCid(src.prefix()));
123-
msg.data.emplace_back(std::move(cid), fromString(src.data()));
104+
auto data = fromString(src.data());
105+
auto prefix_reader = common::span::cbytes(src.prefix());
106+
OUTCOME_TRY(cid, CID::read(prefix_reader, true));
107+
if (!prefix_reader.empty()) {
108+
return Error::MESSAGE_PARSE_ERROR;
109+
}
110+
cid.content_address =
111+
crypto::Hasher::calculate(cid.content_address.getType(), data);
112+
msg.data.emplace_back(std::move(cid), data);
124113
}
125114
}
126115
return outcome::success();

‎core/storage/ipfs/graphsync/impl/network/marshalling/request_builder.cpp

+1-3
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,7 @@ namespace fc::storage::ipfs::graphsync {
2222
auto *dst = pb_msg_->add_requests();
2323
dst->set_id(request_id);
2424

25-
CborEncodeStream encoder;
26-
encoder << root_cid;
27-
auto d = encoder.data();
25+
OUTCOME_EXCEPT(d, root_cid.toBytes());
2826

2927
dst->set_root(d.data(), d.size());
3028
if (!selector.empty()) {

0 commit comments

Comments
 (0)