Feature/fip 0027 market label (#670)

* FIP-0027 Label type for utf-8 encoding in markets

Signed-off-by: Alexey-N-Chernyshov <[email protected]>

Author: Alexey-N-Chernyshov

cmake/dependencies.cmake

@@ -89,6 +89,9 @@ find_package(libarchive CONFIG REQUIRED)
 hunter_add_package(prometheus-cpp)
 find_package(prometheus-cpp CONFIG REQUIRED)
 
+hunter_add_package(utf8)
+find_package(utf8 CONFIG REQUIRED)
+
 # Add filecoin_ffi target if building without git submodules
 if (NOT BUILD_INTERNAL_DEPS)
   find_package(filecoin_ffi REQUIRED)

core/api/full_node/make.cpp

@@ -37,7 +37,6 @@
 #include "vm/actor/builtin/v5/miner/monies.hpp"
 #include "vm/message/impl/message_signer_impl.hpp"
 #include "vm/message/message.hpp"
-#include "vm/runtime/impl/tipset_randomness.hpp"
 #include "vm/runtime/make_vm.hpp"
 #include "vm/state/impl/state_tree_impl.hpp"
 #include "vm/toolchain/toolchain.hpp"

core/api/rpc/json.hpp

@@ -26,6 +26,7 @@
 #include "primitives/cid/cid_of_cbor.hpp"
 #include "primitives/json_types.hpp"
 #include "vm/actor/builtin/types/market/deal.hpp"
+#include "vm/actor/builtin/types/market/deal_proposal.hpp"
 #include "vm/actor/builtin/types/miner/miner_info.hpp"
 
 namespace fc::common {
@@ -976,6 +977,35 @@ namespace fc::vm::actor::builtin::types {
       Get(j, "SlashEpoch", v.slash_epoch);
     }
 
+    JSON_ENCODE(Universal<DealProposal>) {
+      Value j{rapidjson::kObjectType};
+      Set(j, "PieceCID", v->piece_cid, allocator);
+      Set(j, "PieceSize", v->piece_size, allocator);
+      Set(j, "VerifiedDeal", v->verified, allocator);
+      Set(j, "Client", v->client, allocator);
+      Set(j, "Provider", v->provider, allocator);
+      Set(j, "StartEpoch", v->start_epoch, allocator);
+      Set(j, "EndEpoch", v->end_epoch, allocator);
+      Set(j, "StoragePricePerEpoch", v->storage_price_per_epoch, allocator);
+      Set(j, "ProviderCollateral", v->provider_collateral, allocator);
+      Set(j, "ClientCollateral", v->client_collateral, allocator);
+      return j;
+    }
+
+    JSON_DECODE(Universal<DealProposal>) {
+      v.actor_version = ActorVersion::kVersion0;
+      Get(j, "PieceCID", v->piece_cid);
+      Get(j, "PieceSize", v->piece_size);
+      Get(j, "VerifiedDeal", v->verified);
+      Get(j, "Client", v->client);
+      Get(j, "Provider", v->provider);
+      Get(j, "StartEpoch", v->start_epoch);
+      Get(j, "EndEpoch", v->end_epoch);
+      Get(j, "StoragePricePerEpoch", v->storage_price_per_epoch);
+      Get(j, "ProviderCollateral", v->provider_collateral);
+      Get(j, "ClientCollateral", v->client_collateral);
+    }
+
     JSON_ENCODE(DealProposal) {
       Value j{rapidjson::kObjectType};
       Set(j, "PieceCID", v.piece_cid, allocator);

core/codec/cbor/streams_annotation.hpp

@@ -145,9 +145,10 @@
                 _CBOR_TUPLE_1)  \
   (op, __VA_ARGS__)
 
-#define CBOR_ENCODE_TUPLE(T, ...)                        \
-  CBOR_ENCODE(T, t) {                                    \
-    return s << (s.list() _CBOR_TUPLE(<<, __VA_ARGS__)); \
+#define CBOR_ENCODE_TUPLE(T, ...)                 \
+  CBOR_ENCODE(T, t) {                             \
+    auto l{s.list()};                             \
+    return s << (l _CBOR_TUPLE(<<, __VA_ARGS__)); \
   }
 
 #define CBOR_TUPLE(T, ...)          \

core/markets/storage/client/CMakeLists.txt

@@ -17,6 +17,7 @@ target_link_libraries(storage_market_client
     logger
     piece
     signature
+    market_types
     message
     tipset
     memory_indexed_car

core/markets/storage/client/impl/storage_market_client_impl.cpp

@@ -132,7 +132,7 @@ namespace fc::markets::storage::client {
     OUTCOME_EXCEPT(bytes, codec::cbor::encode(deal->proposal_cid));
     OUTCOME_CB(
         auto signature,
-        api_->WalletSign(deal->client_deal_proposal.proposal.client, bytes));
+        api_->WalletSign(deal->client_deal_proposal.proposal->client, bytes));
     DealStatusRequestV1_1_0 req{{deal->proposal_cid, signature}};
     status_streams_->open({
         deal->miner,
@@ -192,7 +192,7 @@ namespace fc::markets::storage::client {
     OUTCOME_TRY(all_deals, api_->StateMarketDeals(chain_head->key));
     std::vector<StorageDeal> client_deals;
     for (const auto &deal : all_deals) {
-      if (deal.second.proposal.client == address) {
+      if (deal.second.proposal->client == address) {
         client_deals.emplace_back(deal.second);
       }
     }
@@ -282,19 +282,20 @@ namespace fc::markets::storage::client {
       provider_collateral = bigdiv(bounds.min * 12, 10);
     }
 
-    DealProposal deal_proposal{
-        .piece_cid = comm_p,
-        .piece_size = piece_size.padded(),
-        .verified = verified_deal,
-        .client = client_address,
-        .provider = provider_info.address,
-        .label = {},
-        .start_epoch = start_epoch,
-        .end_epoch = end_epoch,
-        .storage_price_per_epoch = price,
-        .provider_collateral = provider_collateral,
-        .client_collateral = 0,
-    };
+    // TODO (a.chernyshov) change to v8 and label_v8
+    auto deal_proposal = UniversalDealProposal(ActorVersion::kVersion0);
+    deal_proposal->piece_cid = comm_p;
+    deal_proposal->piece_size = piece_size.padded();
+    deal_proposal->verified = verified_deal;
+    deal_proposal->client = client_address;
+    deal_proposal->provider = provider_info.address;
+    deal_proposal->label_v0 = {};
+    deal_proposal->start_epoch = start_epoch;
+    deal_proposal->end_epoch = end_epoch;
+    deal_proposal->storage_price_per_epoch = price;
+    deal_proposal->provider_collateral = provider_collateral;
+    deal_proposal->client_collateral = 0;
+
     OUTCOME_TRY(signed_proposal, signProposal(client_address, deal_proposal));
     auto proposal_cid{signed_proposal.cid()};
 
@@ -366,7 +367,7 @@ namespace fc::markets::storage::client {
   }
 
   outcome::result<ClientDealProposal> StorageMarketClientImpl::signProposal(
-      const Address &address, const DealProposal &proposal) const {
+      const Address &address, const UniversalDealProposal &proposal) const {
     OUTCOME_TRY(chain_head, api_->ChainHead());
     OUTCOME_TRY(key_address, api_->StateAccountKey(address, chain_head->key));
     OUTCOME_TRY(proposal_bytes, codec::cbor::encode(proposal));
@@ -380,9 +381,9 @@ namespace fc::markets::storage::client {
     OUTCOME_TRY(
         maybe_cid,
         api_->MarketReserveFunds(
-            deal->client_deal_proposal.proposal.client,
-            deal->client_deal_proposal.proposal.client,
-            deal->client_deal_proposal.proposal.clientBalanceRequirement()));
+            deal->client_deal_proposal.proposal->client,
+            deal->client_deal_proposal.proposal->client,
+            deal->client_deal_proposal.proposal->clientBalanceRequirement()));
     return std::move(maybe_cid);
   }
 
@@ -411,7 +412,7 @@ namespace fc::markets::storage::client {
     OUTCOME_TRY(publish_message, api_->ChainGetMessage(deal->publish_message));
     OUTCOME_TRY(
         miner_info,
-        api_->StateMinerInfo(deal->client_deal_proposal.proposal.provider,
+        api_->StateMinerInfo(deal->client_deal_proposal.proposal->provider,
                              msg_state.tipset));
     OUTCOME_TRY(from_id_address,
                 api_->StateLookupID(publish_message.from, msg_state.tipset));
@@ -645,7 +646,7 @@ namespace fc::markets::storage::client {
       StorageDealStatus from,
       StorageDealStatus to) {
     chain_events_->onDealSectorCommitted(
-        deal->client_deal_proposal.proposal.provider,
+        deal->client_deal_proposal.proposal->provider,
         deal->deal_id,
         [self{shared_from_this()}, deal](auto _r) {
           SELF_FSM_HALT_ON_ERROR(_r, "onDealSectorCommitted error", deal);

core/markets/storage/client/impl/storage_market_client_impl.hpp

@@ -20,6 +20,8 @@
 #include "markets/storage/client/client_events.hpp"
 #include "markets/storage/client/import_manager/import_manager.hpp"
 #include "markets/storage/client/storage_market_client.hpp"
+#include "vm/actor/builtin/types/market/deal_proposal.hpp"
+#include "vm/actor/builtin/types/universal/universal.hpp"
 
 namespace fc::markets::storage::client {
   using api::FullNodeApi;
@@ -37,6 +39,9 @@ namespace fc::markets::storage::client {
   using Datastore = fc::storage::face::PersistentMap<Bytes, Bytes>;
   using data_transfer::DataTransfer;
   using libp2p::connection::StreamOpenQueue;
+  using vm::actor::builtin::types::Universal;
+  using UniversalDealProposal =
+      Universal<vm::actor::builtin::types::market::DealProposal>;
 
   class StorageMarketClientImpl
       : public StorageMarketClient,
@@ -97,7 +102,7 @@ namespace fc::markets::storage::client {
         const DataRef &data_ref) const;
 
     outcome::result<ClientDealProposal> signProposal(
-        const Address &address, const DealProposal &proposal) const;
+        const Address &address, const UniversalDealProposal &proposal) const;
 
     /**
      * Ensure client has enough funds. In case of lack of funds add funds

core/markets/storage/mk_protocol.hpp

@@ -16,6 +16,8 @@
 #include "primitives/types.hpp"
 #include "storage/filestore/path.hpp"
 #include "vm/actor/builtin/types/market/deal.hpp"
+#include "vm/actor/builtin/types/market/deal_proposal.hpp"
+#include "vm/actor/builtin/types/universal/universal.hpp"
 
 namespace fc::markets::storage {
   using codec::cbor::CborDecodeStream;
@@ -27,6 +29,7 @@ namespace fc::markets::storage {
   using primitives::TokenAmount;
   using primitives::address::Address;
   using primitives::piece::UnpaddedPieceSize;
+  using vm::actor::builtin::types::Universal;
   using vm::actor::builtin::types::market::ClientDealProposal;
   using vm::actor::builtin::types::market::DealProposal;
   using vm::actor::builtin::types::market::DealState;
@@ -171,7 +174,7 @@ namespace fc::markets::storage {
    * StorageDeal is a local combination of a proposal and a current deal state
    */
   struct StorageDeal {
-    DealProposal proposal;
+    Universal<DealProposal> proposal;
     DealState state;
   };
 

core/markets/storage/provider/impl/provider_impl.cpp

@@ -183,15 +183,15 @@ namespace fc::markets::storage::provider {
 
     auto unpadded{proofs::padPiece(car_path)};
     if (unpadded.padded()
-        != deal_context->deal->client_deal_proposal.proposal.piece_size) {
+        != deal_context->deal->client_deal_proposal.proposal->piece_size) {
       return StorageMarketProviderError::kPieceCIDDoesNotMatch;
     }
     OUTCOME_TRY(registered_proof, api_->GetProofType(miner_actor_address_, {}));
     OUTCOME_TRY(piece_commitment,
                 piece_io_->generatePieceCommitment(registered_proof, car_path));
 
     if (piece_commitment.first
-        != deal_context->deal->client_deal_proposal.proposal.piece_cid) {
+        != deal_context->deal->client_deal_proposal.proposal->piece_cid) {
       return StorageMarketProviderError::kPieceCIDDoesNotMatch;
     }
     deal_context->deal->piece_path = car_path.string();
@@ -216,7 +216,7 @@ namespace fc::markets::storage::provider {
 
     // verify client's signature
     OUTCOME_TRY(bytes, request.getDigest());
-    const auto &client_address = deal.client_deal_proposal.proposal.client;
+    const auto &client_address = deal.client_deal_proposal.proposal->client;
     OUTCOME_TRY(verified,
                 api_->WalletVerify(client_address, bytes, request.signature));
     if (!verified) {
@@ -267,23 +267,23 @@ namespace fc::markets::storage::provider {
     OUTCOME_TRY(proposal_bytes, codec::cbor::encode(proposal));
     OUTCOME_TRY(
         verified,
-        api_->WalletVerify(proposal.client,
+        api_->WalletVerify(proposal->client,
                            proposal_bytes,
                            deal->client_deal_proposal.client_signature));
     if (!verified) {
       deal->message = "Deal proposal verification failed, wrong signature";
       return false;
     }
 
-    if (proposal.provider != miner_actor_address_) {
+    if (proposal->provider != miner_actor_address_) {
       deal->message =
           "Deal proposal verification failed, incorrect provider for deal";
       return false;
     }
 
     OUTCOME_TRY(chain_head, api_->ChainHead());
     if (chain_head->epoch()
-        > proposal.start_epoch - kDefaultDealAcceptanceBuffer) {
+        > proposal->start_epoch - kDefaultDealAcceptanceBuffer) {
       deal->message =
           "Deal proposal verification failed, deal start epoch is too soon "
           "or deal already expired";
@@ -292,43 +292,43 @@ namespace fc::markets::storage::provider {
 
     OUTCOME_TRY(ask, stored_ask_->getAsk(miner_actor_address_));
     auto min_price = bigdiv(
-        ask.ask.price * static_cast<uint64_t>(proposal.piece_size), 1 << 30);
-    if (proposal.storage_price_per_epoch < min_price) {
+        ask.ask.price * static_cast<uint64_t>(proposal->piece_size), 1 << 30);
+    if (proposal->storage_price_per_epoch < min_price) {
       std::stringstream ss;
       ss << "Deal proposal verification failed, storage price per epoch less "
             "than asking price: "
-         << proposal.storage_price_per_epoch << " < " << min_price;
+         << proposal->storage_price_per_epoch << " < " << min_price;
       deal->message = ss.str();
       return false;
     }
 
-    if (proposal.piece_size < ask.ask.min_piece_size) {
+    if (proposal->piece_size < ask.ask.min_piece_size) {
       std::stringstream ss;
       ss << "Deal proposal verification failed, piece size less than minimum "
             "required size: "
-         << proposal.piece_size << " < " << ask.ask.min_piece_size;
+         << proposal->piece_size << " < " << ask.ask.min_piece_size;
       deal->message = ss.str();
       return false;
     }
-    if (proposal.piece_size > ask.ask.max_piece_size) {
+    if (proposal->piece_size > ask.ask.max_piece_size) {
       std::stringstream ss;
       ss << "Deal proposal verification failed, piece size more than maximum "
             "allowed size: "
-         << proposal.piece_size << " > " << ask.ask.max_piece_size;
+         << proposal->piece_size << " > " << ask.ask.max_piece_size;
       deal->message = ss.str();
       return false;
     }
 
     // This doesn't guarantee that the client won't withdraw / lock those
     // funds but it's a decent first filter
     OUTCOME_TRY(client_balance,
-                api_->StateMarketBalance(proposal.client, chain_head->key));
+                api_->StateMarketBalance(proposal->client, chain_head->key));
     TokenAmount available = client_balance.escrow - client_balance.locked;
-    if (available < proposal.getTotalStorageFee()) {
+    if (available < proposal->getTotalStorageFee()) {
       std::stringstream ss;
       ss << "Deal proposal verification failed, client market available "
             "balance too small: "
-         << available << " < " << proposal.getTotalStorageFee();
+         << available << " < " << proposal->getTotalStorageFee();
       deal->message = ss.str();
       return false;
     }
@@ -342,11 +342,11 @@ namespace fc::markets::storage::provider {
     OUTCOME_TRY(chain_head, api_->ChainHead());
     auto proposal = deal->client_deal_proposal.proposal;
     OUTCOME_TRY(worker_info,
-                api_->StateMinerInfo(proposal.provider, chain_head->key));
+                api_->StateMinerInfo(proposal->provider, chain_head->key));
     OUTCOME_TRY(maybe_cid,
                 api_->MarketReserveFunds(worker_info.worker,
-                                         proposal.provider,
-                                         proposal.provider_collateral));
+                                         proposal->provider,
+                                         proposal->provider_collateral));
     return std::move(maybe_cid);
   }
 
@@ -355,7 +355,7 @@ namespace fc::markets::storage::provider {
     OUTCOME_TRY(chain_head, api_->ChainHead());
     OUTCOME_TRY(
         worker_info,
-        api_->StateMinerInfo(deal->client_deal_proposal.proposal.provider,
+        api_->StateMinerInfo(deal->client_deal_proposal.proposal->provider,
                              chain_head->key));
     market::PublishStorageDeals::Params params{{deal->client_deal_proposal}};
     OUTCOME_TRY(encoded_params, codec::cbor::encode(params));
@@ -441,9 +441,9 @@ namespace fc::markets::storage::provider {
       locations[deal->ref.root] = {};
     }
     OUTCOME_TRY(piece_storage_->addPayloadLocations(
-        deal->client_deal_proposal.proposal.piece_cid, locations));
+        deal->client_deal_proposal.proposal->piece_cid, locations));
     OUTCOME_TRY(piece_storage_->addDealForPiece(
-        deal->client_deal_proposal.proposal.piece_cid,
+        deal->client_deal_proposal.proposal->piece_cid,
         DealInfo{.deal_id = deal->deal_id,
                  .sector_id = piece_location.sector,
                  .offset = piece_location.offset,
@@ -673,12 +673,12 @@ namespace fc::markets::storage::provider {
   FSM_HANDLE_DEFINITION(StorageProviderImpl::onProviderEventDealPublished) {
     auto &proposal{deal_context->deal->client_deal_proposal.proposal};
     auto maybe_piece_location = sector_blocks_->addPiece(
-        proposal.piece_size.unpadded(),
+        proposal->piece_size.unpadded(),
         deal_context->deal->piece_path,
         mining::types::DealInfo{deal_context->deal->publish_cid,
                                 deal_context->deal->deal_id,
                                 proposal,
-                                {proposal.start_epoch, proposal.end_epoch},
+                                {proposal->start_epoch, proposal->end_epoch},
                                 deal_context->deal->is_fast_retrieval});
     FSM_HALT_ON_ERROR(
         maybe_piece_location, "Unable to locate piece", deal_context);
@@ -689,7 +689,7 @@ namespace fc::markets::storage::provider {
 
   FSM_HANDLE_DEFINITION(StorageProviderImpl::onProviderEventDealHandedOff) {
     chain_events_->onDealSectorCommitted(
-        deal_context->deal->client_deal_proposal.proposal.provider,
+        deal_context->deal->client_deal_proposal.proposal->provider,
         deal_context->deal->deal_id,
         [=](auto _r) {
           FSM_HALT_ON_ERROR(_r, "onDealSectorCommitted error", deal_context);