Merge pull request #81 from iron-fish/feat-example-download-script

feat: add example script for downloading chain data

Author: dguenther

.gitignore

@@ -9,5 +9,6 @@ node_modules/
 *testdb*
 *yarn-error.log
 test/cache/
-blocks*
+*blocks*
+*byteRanges*
 manifest.json

example/.env.example

@@ -1,3 +1,3 @@
 SPENDING_KEY=""
-WALLET_SERVER_HOST="walletserver.ironfish.network:50051"
+WALLET_SERVER_HOST="https://walletserver.ironfish.network"
 BLOCK_PROCESSOR_BATCH_SIZE=99

example/package.json

@@ -5,6 +5,8 @@
   "main": "index.js",
   "scripts": {
     "dev": "ts-node src/index.ts",
+    "download": "ts-node src/downloadChain.ts",
+    "download-api-only": "LATEST_FINALIZED_SEQUENCE=422949 ts-node src/downloadChain.ts",
     "gen-api": "yarn swagger-typescript-api -p ../src/swagger/swagger.json -o src/api",
     "test": "echo \"Error: no test specified\" && exit 1"
   },
@@ -19,5 +21,8 @@
     "swagger-typescript-api": "^13.0.3",
     "ts-node": "^10.9.1",
     "typescript": "^5.1.6"
+  },
+  "dependencies": {
+    "axios": "^1.7.2"
   }
 }

example/src/Client/utils/BlockProcessor.ts

@@ -136,7 +136,7 @@ export class BlockProcessor {
 
     for (const transaction of block.transactions) {
       for (const output of transaction.outputs) {
-        notes.push(Buffer.from(output.note, "hex"));
+        notes.push(output.note);
       }
     }
 

example/src/downloadChain.ts

@@ -0,0 +1,128 @@
+import { config } from "dotenv";
+config();
+
+import axios from "axios";
+import zlib from "zlib";
+import fs from "fs";
+import { promisify } from "util";
+import { LightBlock } from "../../src/models/lightstreamer";
+import { ManifestFile } from "../../src/upload";
+import { Api } from "./api/Api";
+
+// Define the URL for the manifest
+const BLOCKS_URL = "https://testnet.lightblocks.ironfish.network/";
+const MANIFEST_PATH = BLOCKS_URL + "manifest.json";
+
+// Function to fetch JSON data from a URL
+async function fetchJson<T>(url: string): Promise<T> {
+  const response = await axios.get(url);
+  return response.data as T;
+}
+
+// Function to download a file from a URL and save it locally
+async function downloadFile(url: string, dest: string): Promise<void> {
+  const response = await axios.get(url, { responseType: "stream" });
+  const writer = fs.createWriteStream(dest);
+
+  response.data.pipe(writer);
+
+  return new Promise((resolve, reject) => {
+    writer.on("finish", resolve);
+    writer.on("error", reject);
+  });
+}
+
+const gunzip = promisify(zlib.gunzip);
+
+async function readGzippedFile(byteRangesFile: string) {
+  const compressedData = fs.readFileSync(byteRangesFile);
+  const decompressedData = await gunzip(compressedData);
+  const byteRanges = decompressedData
+    .toString("utf-8")
+    .split("\n")
+    .filter((line) => line.trim() !== "");
+  return byteRanges;
+}
+
+// Function to download finalized blocks and convert them to LightBlock
+async function downloadFinalizedBlocks(): Promise<number> {
+  const latestFinalizedBlockSequence = 0;
+  try {
+    // Fetch the manifest file
+    const manifest = await fetchJson<ManifestFile>(MANIFEST_PATH);
+
+    for (const chunk of manifest.chunks) {
+      if (chunk.finalized) {
+        // Download the block binary file
+        const blocksUrl = `${BLOCKS_URL}${chunk.blocks}`;
+        const blocksFile = `blocks_${chunk.range.start}_${chunk.range.end}`;
+        await downloadFile(blocksUrl, blocksFile);
+
+        // Download the byte ranges file
+        const byteRangesUrl = `${BLOCKS_URL}${chunk.byteRangesFile}`;
+        const byteRangesFile = `byteRanges_${chunk.range.start}_${chunk.range.end}.csv.gz`;
+        await downloadFile(byteRangesUrl, byteRangesFile);
+
+        // Read the byte ranges file and convert each block
+        const byteRanges = await readGzippedFile(byteRangesFile);
+        const blockBuffer = fs.readFileSync(blocksFile);
+        for (const range of byteRanges) {
+          const [_, start, end] = range.split(",").map(Number);
+          const blockBufferSub = blockBuffer.subarray(start, end + 1);
+          const lightBlock = LightBlock.decode(blockBufferSub);
+          // Process the LightBlock as needed
+          console.log(
+            `Downloaded LightBlock sequence: ${
+              lightBlock.sequence
+            }, hash: ${lightBlock.hash.toString("hex")}`,
+          );
+        }
+      }
+    }
+  } catch (error) {
+    console.error(`Failed to download finalized blocks: ${error}`);
+  }
+  return latestFinalizedBlockSequence;
+}
+
+async function main() {
+  if (!process.env["WALLET_SERVER_HOST"]) {
+    console.error("WALLET_SERVER_HOST environment variable is required");
+    process.exit(1);
+  }
+
+  // Allows skipping download and only download unfinalized blocks
+  let latestFinalizedSequence;
+  if (process.env["LATEST_FINALIZED_SEQUENCE"]) {
+    latestFinalizedSequence = Number(process.env["LATEST_FINALIZED_SEQUENCE"]);
+  } else {
+    // Download finalized blocks and returns the latest finalized sequence
+    latestFinalizedSequence = await downloadFinalizedBlocks();
+  }
+  const api = new Api({ baseUrl: process.env["WALLET_SERVER_HOST"] });
+  const latestReponse = await api.latestBlock.getLatestBlock();
+  const latestBlockSequence = latestReponse.data.sequence;
+
+  const chunkSize = 100;
+  for (
+    let i = latestFinalizedSequence + 1;
+    i < latestBlockSequence;
+    i += chunkSize
+  ) {
+    const start = i;
+    const end = Math.min(i + chunkSize - 1, latestBlockSequence);
+    console.log(`Downloading blocks from ${start} to ${end}`);
+    const blocksResponse = await api.blockRange.getBlockRange({ start, end });
+    for (const block of blocksResponse.data) {
+      const lightBlock = LightBlock.decode(Buffer.from(block, "hex"));
+      // Process the LightBlock as needed
+      console.log(
+        `API Downloaded LightBlock sequence: ${
+          lightBlock.sequence
+        }, hash: ${lightBlock.hash.toString("hex")}`,
+      );
+    }
+  }
+}
+
+main();