Skip to content

Commit 7080077

Browse files
nadineloepfenadineloepfe
committed
switching from bash to python script
Signed-off-by: nadine.loepfe <[email protected]>
1 parent 1e5551e commit 7080077

File tree

4 files changed

+238
-88
lines changed

4 files changed

+238
-88
lines changed

‎.github/workflows/publish.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ jobs:
2828
run: pip install build pdm-backend "grpcio-tools==1.68.1"
2929

3030
- name: Generate Protobuf
31-
run: bash ./generate_proto.sh
31+
run: python generate_proto.py
3232

3333
- name: Build wheel and sdist
3434
run: python -m build

‎.github/workflows/test.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ jobs:
2525
run: uv sync --all-extras --dev
2626

2727
- name: Generate Proto Files
28-
run: bash ./generate_proto.sh
28+
run: python generate_proto.py
2929

3030
- name: Prepare Hiero Solo
3131
id: solo

‎generate_proto.py

Lines changed: 236 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,236 @@
1+
#!/usr/bin/env python3
2+
"""
3+
Generate Hiero protobuf Python files from Hedera's hedera-protobufs repo.
4+
Works on Windows, macOS, and Linux.
5+
6+
Usage (locally or in CI):
7+
python generate_proto.py
8+
"""
9+
10+
import os
11+
import sys
12+
import shutil
13+
import subprocess
14+
import tarfile
15+
import re
16+
import urllib.request
17+
from pathlib import Path
18+
19+
# Configure your directories and versions
20+
HAPI_VERSION = "v0.57.3"
21+
PROTOS_DIR = Path(".protos")
22+
SERVICES_DIR = Path("src/hiero_sdk_python/hapi/services")
23+
MIRROR_DIR = Path("src/hiero_sdk_python/hapi/mirror")
24+
25+
26+
def clean_and_prepare_directories():
27+
"""Remove old directories and create the required folder structure."""
28+
print("Setting up directories...")
29+
30+
# 1) Clear .protos dir
31+
if PROTOS_DIR.is_dir():
32+
shutil.rmtree(PROTOS_DIR)
33+
PROTOS_DIR.mkdir(parents=True, exist_ok=True)
34+
35+
# 2) Clear services and mirror dirs
36+
if SERVICES_DIR.is_dir():
37+
shutil.rmtree(SERVICES_DIR)
38+
if MIRROR_DIR.is_dir():
39+
shutil.rmtree(MIRROR_DIR)
40+
41+
# 3) Recreate needed structure
42+
(SERVICES_DIR / "auxiliary" / "tss").mkdir(parents=True, exist_ok=True)
43+
(SERVICES_DIR / "event").mkdir(parents=True, exist_ok=True)
44+
MIRROR_DIR.mkdir(parents=True, exist_ok=True)
45+
46+
# 4) Create __init__.py placeholders
47+
(SERVICES_DIR / "__init__.py").touch()
48+
(MIRROR_DIR / "__init__.py").touch()
49+
50+
51+
def download_and_extract_protos():
52+
"""Download and extract the protobuf tar for the specified HAPI_VERSION."""
53+
print(f"Downloading Hiero protobufs version {HAPI_VERSION}...")
54+
55+
url = f"https://github.com/hashgraph/hedera-protobufs/archive/refs/tags/{HAPI_VERSION}.tar.gz"
56+
tarball_path = PROTOS_DIR / "hedera-protobufs.tar.gz"
57+
58+
# Download .tar.gz
59+
with urllib.request.urlopen(url) as response, open(tarball_path, "wb") as out_file:
60+
shutil.copyfileobj(response, out_file)
61+
62+
print(f"Extracting tarball to {PROTOS_DIR}...")
63+
with tarfile.open(tarball_path, mode="r:gz") as tar_ref:
64+
for member in tar_ref.getmembers():
65+
path_parts = member.name.split(os.sep)
66+
if len(path_parts) > 1:
67+
new_path = os.sep.join(path_parts[1:])
68+
else:
69+
continue
70+
member.name = new_path
71+
if member.name:
72+
tar_ref.extract(member, path=PROTOS_DIR)
73+
74+
tarball_path.unlink(missing_ok=True)
75+
76+
for child in PROTOS_DIR.iterdir():
77+
if child.name not in ("platform", "services", "mirror"):
78+
if child.is_dir():
79+
shutil.rmtree(child)
80+
else:
81+
child.unlink()
82+
83+
84+
def run_protoc_for_services():
85+
"""
86+
Compile 'services' and 'platform' protobuf files into Python files
87+
inside the services directory.
88+
"""
89+
print("Compiling service and platform protobuf files...")
90+
91+
cmd = [
92+
sys.executable, "-m", "grpc_tools.protoc",
93+
f"--proto_path={PROTOS_DIR / 'platform'}",
94+
f"--proto_path={PROTOS_DIR / 'services'}",
95+
f"--pyi_out={SERVICES_DIR}",
96+
f"--python_out={SERVICES_DIR}",
97+
f"--grpc_python_out={SERVICES_DIR}",
98+
*(str(p) for p in (PROTOS_DIR / "services").glob("*.proto")),
99+
*(str(p) for p in (PROTOS_DIR / "services" / "auxiliary" / "tss").glob("*.proto")),
100+
*(str(p) for p in (PROTOS_DIR / "platform" / "event").glob("*.proto")),
101+
]
102+
103+
completed = subprocess.run(cmd, capture_output=True)
104+
if completed.returncode != 0:
105+
print("Failed to compile service & platform protos:")
106+
print(completed.stderr.decode())
107+
sys.exit(1)
108+
109+
110+
def adjust_imports_services():
111+
"""
112+
Adjust the imports in the compiled Python files for 'services' & 'platform'
113+
to ensure relative paths are correct.
114+
"""
115+
print("Adjusting imports for service and platform protobuf files...")
116+
117+
for py_file in SERVICES_DIR.rglob("*.py"):
118+
with open(py_file, "r", encoding="utf-8") as f:
119+
content = f.read()
120+
121+
# 1) Lines that start with "import something_pb2 as something__pb2" -> prefix 'from . '
122+
content = re.sub(
123+
r'^(import .*_pb2 as .*__pb2)',
124+
r'from . \1',
125+
content,
126+
flags=re.MULTILINE
127+
)
128+
129+
# 2) Lines that start with "from auxiliary.tss" -> "from .auxiliary.tss"
130+
content = re.sub(
131+
r'^from auxiliary\.tss',
132+
'from .auxiliary.tss',
133+
content,
134+
flags=re.MULTILINE
135+
)
136+
137+
# 3) Lines that start with "from event" -> "from .event"
138+
content = re.sub(
139+
r'^from event',
140+
'from .event',
141+
content,
142+
flags=re.MULTILINE
143+
)
144+
145+
with open(py_file, "w", encoding="utf-8") as f:
146+
f.write(content)
147+
148+
149+
def run_protoc_for_mirror():
150+
"""
151+
Compile 'mirror' protobuf files into Python files inside the mirror directory.
152+
"""
153+
print("Compiling mirror protobuf files...")
154+
155+
cmd = [
156+
sys.executable, "-m", "grpc_tools.protoc",
157+
f"--proto_path={PROTOS_DIR / 'mirror'}",
158+
f"--proto_path={PROTOS_DIR / 'services'}",
159+
f"--python_out={MIRROR_DIR}",
160+
f"--grpc_python_out={MIRROR_DIR}",
161+
*(str(p) for p in (PROTOS_DIR / "mirror").glob("*.proto")),
162+
]
163+
164+
completed = subprocess.run(cmd, capture_output=True)
165+
if completed.returncode != 0:
166+
print("Failed to compile mirror protos:")
167+
print(completed.stderr.decode())
168+
sys.exit(1)
169+
170+
171+
def adjust_imports_mirror():
172+
"""
173+
Adjust the imports in the compiled Python files for 'mirror' to ensure paths
174+
reference the correct modules (services vs mirror).
175+
"""
176+
print("Adjusting imports for mirror protobuf files...")
177+
for py_file in MIRROR_DIR.rglob("*.py"):
178+
with open(py_file, "r", encoding="utf-8") as f:
179+
content = f.read()
180+
content = re.sub(
181+
r'^import basic_types_pb2 as',
182+
'import hiero_sdk_python.hapi.services.basic_types_pb2 as',
183+
content,
184+
flags=re.MULTILINE
185+
)
186+
content = re.sub(
187+
r'^import timestamp_pb2 as',
188+
'import hiero_sdk_python.hapi.services.timestamp_pb2 as',
189+
content,
190+
flags=re.MULTILINE
191+
)
192+
content = re.sub(
193+
r'^import consensus_submit_message_pb2 as',
194+
'import hiero_sdk_python.hapi.services.consensus_submit_message_pb2 as',
195+
content,
196+
flags=re.MULTILINE
197+
)
198+
content = re.sub(
199+
r'^import consensus_service_pb2 as',
200+
'import hiero_sdk_python.hapi.mirror.consensus_service_pb2 as',
201+
content,
202+
flags=re.MULTILINE
203+
)
204+
content = re.sub(
205+
r'^import mirror_network_service_pb2 as',
206+
'import hiero_sdk_python.hapi.mirror.mirror_network_service_pb2 as',
207+
content,
208+
flags=re.MULTILINE
209+
)
210+
211+
with open(py_file, "w", encoding="utf-8") as f:
212+
f.write(content)
213+
214+
215+
def verify_generation_success():
216+
"""Ensure protobuf files have been generated for both services and mirror."""
217+
if any(SERVICES_DIR.iterdir()) and any(MIRROR_DIR.iterdir()):
218+
print("All protobuf files have been generated and adjusted successfully!")
219+
else:
220+
print("Error: Protobuf file generation or adjustment failed.")
221+
sys.exit(1)
222+
223+
224+
def main():
225+
"""Main entry point."""
226+
clean_and_prepare_directories()
227+
download_and_extract_protos()
228+
run_protoc_for_services()
229+
adjust_imports_services()
230+
run_protoc_for_mirror()
231+
adjust_imports_mirror()
232+
verify_generation_success()
233+
234+
235+
if __name__ == "__main__":
236+
main()

‎generate_proto.sh

Lines changed: 0 additions & 86 deletions
This file was deleted.

0 commit comments

Comments
 (0)