Skip to content

Commit

Permalink
Merge pull request #93 from datachainlab/improve-proto-compiler
Browse files Browse the repository at this point in the history 
Refactor proto-compiler

Signed-off-by: Jun Kimura <[email protected]>
  • Loading branch information
@bluele
bluele authored Dec 7, 2023
2 parents 6ea8c35 + 3d9a6cd commit 6eeedc7
Show file tree
Hide file tree
Showing 7 changed files with 221 additions and 259 deletions.
1 change: 0 additions & 1 deletion proto-compiler/Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 0 additions & 1 deletion proto-compiler/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ version = "0.1.0"
edition = "2021"

[dependencies]
tempfile = "3"
walkdir = "2.3"
argh = "0.1.3"
tonic-build = "0.8"
2 changes: 1 addition & 1 deletion proto-compiler/src/cmd/clone.rs
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
// TODO implement this
// TODO implement this
246 changes: 14 additions & 232 deletions proto-compiler/src/cmd/compile.rs
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,10 @@
use std::fs::remove_dir_all;
use std::fs::{copy, create_dir_all};
use std::fs::{create_dir_all, remove_dir_all};
use std::path::{Path, PathBuf};
use std::process;

use tempfile::TempDir;
use argh::FromArgs;
use walkdir::WalkDir;

use argh::FromArgs;
#[derive(Debug, FromArgs)]
#[argh(subcommand, name = "compile")]
/// Compile
Expand All @@ -26,16 +24,14 @@ pub struct CompileCmd {

impl CompileCmd {
pub fn run(&self) {
let tmp_ibc = TempDir::new().unwrap();
Self::compile_ibc_protos(&self.ibc, tmp_ibc.as_ref());

let tmp_lcp = TempDir::new().unwrap();
Self::compile_lcp_protos(&self.ibc, tmp_lcp.as_ref(), &self.descriptor);

Self::copy_generated_files(tmp_lcp.as_ref(), tmp_ibc.as_ref(), &self.out);
Self::compile_protos(&self.ibc, self.out.as_ref(), &self.descriptor);
}

fn compile_lcp_protos(ibc_dir: &Path, out_dir: &Path, descriptor_path: &Path) {
fn compile_protos(ibc_dir: &Path, out_dir: &Path, descriptor_path: &Path) {
// Remove old compiled files
remove_dir_all(&out_dir).unwrap_or_default();
create_dir_all(&out_dir).unwrap();

println!(
"[info ] Compiling LCP .proto files to Rust into '{}'...",
out_dir.display()
Expand All @@ -45,90 +41,15 @@ impl CompileCmd {

// Paths
let proto_paths = [
format!(
"{}/../proto/definitions",
root
),
];

let proto_includes_paths = [
format!("{}/../proto/definitions", root),
format!("{}/proto", ibc_dir.display()),
format!("{}/third_party/proto", ibc_dir.display()),
];

// List available proto files
let mut protos: Vec<PathBuf> = vec![];
for proto_path in &proto_paths {
println!("Looking for proto files in {:?}", proto_path);
protos.append(
&mut WalkDir::new(proto_path)
.into_iter()
.filter_map(|e| e.ok())
.filter(|e| {
e.file_type().is_file()
&& e.path().extension().is_some()
&& e.path().extension().unwrap() == "proto"
})
.map(|e| e.into_path())
.collect(),
);
}

println!("Found the following protos:");
// Show which protos will be compiled
for proto in &protos {
println!("\t-> {:?}", proto);
}
println!("[info ] Compiling..");

// List available paths for dependencies
let includes: Vec<PathBuf> = proto_includes_paths.iter().map(PathBuf::from).collect();
let attrs_serde = r#"#[derive(::serde::Serialize, ::serde::Deserialize)]"#;
let compilation = tonic_build::configure()
.build_client(true)
.compile_well_known_types(true)
.client_mod_attribute(".", r#"#[cfg(feature = "client")]"#)
.server_mod_attribute(".", r#"#[cfg(feature = "server")]"#)
.build_server(true)
.out_dir(out_dir)
.extern_path(".tendermint", "::tendermint_proto")
.type_attribute(".cosmos.upgrade.v1beta1", attrs_serde)
.type_attribute(".cosmos.base.v1beta1", attrs_serde)
.type_attribute(".cosmos.base.query.v1beta1", attrs_serde)
.type_attribute(".cosmos.bank.v1beta1", attrs_serde)
.type_attribute(".lcp.service.enclave.v1", attrs_serde)
.type_attribute(".lcp.service.elc.v1", attrs_serde)
.file_descriptor_set_path(descriptor_path)
.compile(&protos, &includes);

match compilation {
Ok(_) => {
println!("Successfully compiled proto files");
}
Err(e) => {
println!("Failed to compile:{:?}", e.to_string());
process::exit(1);
}
}
}

fn compile_ibc_protos(ibc_dir: &Path, out_dir: &Path) {
println!(
"[info ] Compiling IBC .proto files to Rust into '{}'...",
out_dir.display()
);

// Paths
let proto_paths = [
// ibc-go proto files
format!(
"{}/proto/ibc/core/client/v1/client.proto",
ibc_dir.display()
),
];

let proto_includes_paths = [
format!("{}/../proto/definitions", root),
format!("{}/proto", ibc_dir.display()),
format!("{}/third_party/proto", ibc_dir.display()),
];
Expand Down Expand Up @@ -160,85 +81,28 @@ impl CompileCmd {

// List available paths for dependencies
let includes: Vec<PathBuf> = proto_includes_paths.iter().map(PathBuf::from).collect();

let attrs_serde = r#"#[derive(::serde::Serialize, ::serde::Deserialize)]"#;
let attrs_jsonschema =
r#"#[cfg_attr(feature = "json-schema", derive(::schemars::JsonSchema))]"#;
let attrs_ord = "#[derive(Eq, PartialOrd, Ord)]";
let attrs_eq = "#[derive(Eq)]";
let attrs_serde_default = r#"#[serde(default)]"#;
let attrs_serde_base64 = r#"#[serde(with = "crate::base64")]"#;
let attrs_jsonschema_str =
r#"#[cfg_attr(feature = "json-schema", schemars(with = "String"))]"#;

let compilation = tonic_build::configure()
.build_client(true)
.compile_well_known_types(true)
.client_mod_attribute(".", r#"#[cfg(feature = "client")]"#)
.build_server(false)
.server_mod_attribute(".", r#"#[cfg(feature = "server")]"#)
.build_server(true)
.out_dir(out_dir)
.extern_path(".tendermint", "::tendermint_proto")
.type_attribute(".google.protobuf.Any", attrs_serde)
.type_attribute(".google.protobuf.Timestamp", attrs_serde)
.type_attribute(".google.protobuf.Duration", attrs_serde)
.type_attribute(".ibc.core.client.v1", attrs_serde)
.type_attribute(".ibc.core.client.v1.Height", attrs_ord)
.type_attribute(".ibc.core.client.v1.Height", attrs_jsonschema)
.field_attribute(".ibc.core.client.v1.Height", attrs_serde_default)
.type_attribute(".ibc.core.commitment.v1", attrs_serde)
.type_attribute(".ibc.core.commitment.v1.MerkleRoot", attrs_jsonschema)
.field_attribute(
".ibc.core.commitment.v1.MerkleRoot.hash",
attrs_serde_base64,
)
.field_attribute(
".ibc.core.commitment.v1.MerkleRoot.hash",
attrs_jsonschema_str,
)
.type_attribute(".ibc.core.commitment.v1.MerklePrefix", attrs_jsonschema)
.field_attribute(
".ibc.core.commitment.v1.MerklePrefix.key_prefix",
attrs_serde_base64,
)
.field_attribute(
".ibc.core.commitment.v1.MerklePrefix.key_prefix",
attrs_jsonschema_str,
)
.type_attribute(".ibc.core.channel.v1", attrs_serde)
.type_attribute(".ibc.core.channel.v1.Channel", attrs_jsonschema)
.type_attribute(".ibc.core.channel.v1.Counterparty", attrs_jsonschema)
.type_attribute(".ibc.core.connection.v1", attrs_serde)
.type_attribute(".ibc.core.connection.v1.ConnectionEnd", attrs_jsonschema)
.type_attribute(".ibc.core.connection.v1.Counterparty", attrs_jsonschema)
.type_attribute(".ibc.core.connection.v1.Version", attrs_jsonschema)
.type_attribute(".ibc.core.types.v1", attrs_serde)
.type_attribute(".ibc.applications.transfer.v1", attrs_serde)
.type_attribute(".ibc.applications.transfer.v2", attrs_serde)
.type_attribute(
".ibc.applications.interchain_accounts.controller.v1",
attrs_serde,
)
.type_attribute(".ics23", attrs_serde)
.type_attribute(".ics23.LeafOp", attrs_eq)
.type_attribute(".ics23.LeafOp", attrs_jsonschema)
.field_attribute(".ics23.LeafOp.prehash_key", attrs_serde_default)
.field_attribute(".ics23.LeafOp.prefix", attrs_serde_base64)
.field_attribute(".ics23.LeafOp.prefix", attrs_jsonschema_str)
.type_attribute(".ics23.InnerOp", attrs_jsonschema)
.field_attribute(".ics23.InnerOp.prefix", attrs_serde_base64)
.field_attribute(".ics23.InnerOp.prefix", attrs_jsonschema_str)
.field_attribute(".ics23.InnerOp.suffix", attrs_serde_base64)
.field_attribute(".ics23.InnerOp.suffix", attrs_jsonschema_str)
.type_attribute(".ics23.InnerOp", attrs_eq)
.type_attribute(".ics23.ProofSpec", attrs_eq)
.type_attribute(".ics23.ProofSpec", attrs_jsonschema)
.field_attribute(".ics23.ProofSpec.max_depth", attrs_serde_default)
.field_attribute(".ics23.ProofSpec.min_depth", attrs_serde_default)
.type_attribute(".ics23.InnerSpec", attrs_eq)
.type_attribute(".ics23.InnerSpec", attrs_jsonschema)
.field_attribute(".ics23.InnerSpec.empty_child", attrs_serde_default)
.field_attribute(".ics23.InnerSpec.empty_child", attrs_serde_base64)
.field_attribute(".ics23.InnerSpec.empty_child", attrs_jsonschema_str)
.type_attribute(".lcp.service.enclave.v1", attrs_serde)
.type_attribute(".lcp.service.elc.v1", attrs_serde)
.file_descriptor_set_path(descriptor_path)
.compile(&protos, &includes);

match compilation {
Expand All @@ -251,86 +115,4 @@ impl CompileCmd {
}
}
}

fn copy_generated_files(from_dir_lcp: &Path, from_dir_ibc: &Path, to_dir: &Path) {
println!(
"[info ] Copying generated files into '{}'...",
to_dir.display()
);

// Remove old compiled files
remove_dir_all(&to_dir).unwrap_or_default();
create_dir_all(&to_dir).unwrap();

// Copy new compiled files (prost does not use folder structures)
// Copy the SDK files first
let errors_sdk = WalkDir::new(from_dir_lcp)
.into_iter()
.filter_map(|e| e.ok())
.filter(|e| e.file_type().is_file())
.map(|e| {
copy(
e.path(),
format!(
"{}/{}",
to_dir.display(),
&e.file_name().to_os_string().to_str().unwrap()
),
)
})
.filter_map(|e| e.err())
.collect::<Vec<_>>();

if !errors_sdk.is_empty() {
for e in errors_sdk {
println!("[error] Error while copying SDK-compiled file: {}", e);
}

panic!("[error] Aborted.");
}

// Copy the IBC-go files second, double-checking if anything is overwritten
let errors_ibc = WalkDir::new(from_dir_ibc)
.into_iter()
.filter_map(|e| e.ok())
.filter(|e| e.file_type().is_file())
.map(|e| {
let generated_fname = e.file_name().to_owned().into_string().unwrap();
let prefix = &generated_fname[0..6];

let target_fname = format!(
"{}/{}",
to_dir.display(),
generated_fname,
);

// If it's a cosmos-relevant file and it exists, we should not overwrite it.
if Path::new(&target_fname).exists() && prefix.eq("cosmos") {
let original_cosmos_file = std::fs::read(target_fname.clone()).unwrap();
let new_cosmos_file = std::fs::read(e.path()).unwrap();
if original_cosmos_file != new_cosmos_file {
println!(
"[warn ] Cosmos-related file exists already {}! Ignoring the one generated from IBC-go {:?}",
target_fname, e.path()
);
}
Ok(0)
} else {
copy(
e.path(),
target_fname,
)
}
})
.filter_map(|e| e.err())
.collect::<Vec<_>>();

if !errors_ibc.is_empty() {
for e in errors_ibc {
println!("[error] Error while copying IBC-go compiled file: {}", e);
}

panic!("[error] Aborted.");
}
}
}
Binary file modified proto/src/descriptor.bin
View file
Binary file not shown.
3 changes: 0 additions & 3 deletions proto/src/prost/cosmos.upgrade.v1beta1.rs
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
/// Plan specifies information about a planned upgrade and when it should occur.
#[derive(::serde::Serialize, ::serde::Deserialize)]
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct Plan {
Expand Down Expand Up @@ -27,7 +26,6 @@ pub struct Plan {
}
/// SoftwareUpgradeProposal is a gov Content type for initiating a software
/// upgrade.
#[derive(::serde::Serialize, ::serde::Deserialize)]
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct SoftwareUpgradeProposal {
Expand All @@ -40,7 +38,6 @@ pub struct SoftwareUpgradeProposal {
}
/// CancelSoftwareUpgradeProposal is a gov Content type for cancelling a software
/// upgrade.
#[derive(::serde::Serialize, ::serde::Deserialize)]
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct CancelSoftwareUpgradeProposal {
Expand Down
Loading

0 comments on commit 6eeedc7

Please sign in to comment.