From 57b7e85d7f0dd5a43458e70b849c61e07579f5b7 Mon Sep 17 00:00:00 2001
From: Jack Lavigne <jacklavigne00@gmail.com>
Date: Fri, 4 Apr 2025 21:38:06 +0200
Subject: [PATCH 1/3] fix: download in chunks
---
src/wasm/src/funcs.rs | 63 ++++++++++++++++++++++++++++++++++++-------
1 file changed, 54 insertions(+), 9 deletions(-)
diff --git a/src/wasm/src/funcs.rs b/src/wasm/src/funcs.rs
index 0ed467f..97413e1 100644
--- a/src/wasm/src/funcs.rs
+++ b/src/wasm/src/funcs.rs
@@ -1,6 +1,6 @@
use std::{
fs::{self, OpenOptions},
- io::Cursor,
+ io::{Cursor, Write},
};
use anyhow::{anyhow, Context, Result};
@@ -21,8 +21,12 @@ use crate::{
DownloadProgressEvent, DownloadProgressPhase, InterfaceEvent,
};
+/// The URL to get the latest available cycle number
const LATEST_CYCLE_ENDPOINT: &str = "https://navdata.api.navigraph.com/info";
+/// The max size in bytes of each request during the download function (set to 4MB curently)
+const DOWNLOAD_CHUNK_SIZE_BYTES: usize = 4 * 1024 * 1024;
+
/// The trait definition for a function that can be called through the navigation data interface
trait Function: DeserializeOwned {
type ReturnType: Serialize;
@@ -70,13 +74,54 @@ impl Function for DownloadNavigationData {
unzipped: None,
})?;
- // Download the data
- let data = NetworkRequestBuilder::new(&self.url)
- .context("can't create new NetworkRequestBuilder")?
- .get()
- .context(".get() returned None")?
- .wait_for_data()
- .await?;
+ // We need to download the data in chunks of DOWNLOAD_CHUNK_SIZE_BYTES to avoid a timeout, so we need to keep track of a "working" accumulation of all responses
+ let mut bytes = vec![];
+
+ let mut current_byte_index = 0;
+ loop {
+ // Dispatch the request
+ let range_end = current_byte_index + DOWNLOAD_CHUNK_SIZE_BYTES - 1;
+ let request = NetworkRequestBuilder::new(&self.url)
+ .context("can't create new NetworkRequestBuilder")?
+ .with_header(&format!("Range: bytes={current_byte_index}-{range_end}"))
+ .context(".with_header() returned None")?
+ .get()
+ .context(".get() returned None")?;
+
+ request.wait_for_data().await?;
+
+ // Get the size of actual data. The response will be as long as the requested range is, but content-length contains the amount we actually want to read
+ let content_length = request
+ .header_section("content-length")
+ .context("no content-length header")?
+ .trim()
+ .parse::<usize>()?;
+
+ // Check if we somehow have no more data (file size would be a perfect multiple of DOWNLOAD_CHUNK_SIZE_BYTES)
+ if content_length == 0 {
+ break;
+ }
+
+ let data = request.data().ok_or(anyhow!("no data"))?;
+
+ // Make sure we don't panic if server sent less data than claimed (should never happen, but avoid a panic)
+ if data.len() < content_length {
+ return Err(anyhow!(
+ "Received less data ({}) than content-length ({})",
+ data.len(),
+ content_length
+ ));
+ }
+
+ bytes.write_all(&data[..content_length])?;
+
+ // Check if we have hit the last chunk
+ if content_length < DOWNLOAD_CHUNK_SIZE_BYTES {
+ break;
+ }
+
+ current_byte_index += content_length;
+ }
// Only close connection if DATABASE_STATE has already been initialized - otherwise we end up unnecessarily copying the bundled data and instantly replacing it (due to initialization logic in database state)
if Lazy::get(&DATABASE_STATE).is_some() {
@@ -103,7 +148,7 @@ impl Function for DownloadNavigationData {
})?;
// Load the zip archive
- let mut zip = ZipArchive::new(Cursor::new(data))?;
+ let mut zip = ZipArchive::new(Cursor::new(bytes))?;
// Ensure parent folder exists (ignore the result as it will return an error if it already exists)
let _ = fs::create_dir_all(WORK_NAVIGATION_DATA_FOLDER);
From eac4adc67c2f8b97c0b796ba3130036d422a0443 Mon Sep 17 00:00:00 2001
From: Jack Lavigne <jacklavigne00@gmail.com>
Date: Mon, 7 Apr 2025 15:36:01 +0200
Subject: [PATCH 2/3] feat: report download percent
---
example/gauge/Components/Pages/Auth/Auth.tsx | 18 +------
.../interface/NavigationDataInterfaceTypes.ts | 12 +----
src/wasm/src/funcs.rs | 51 +++++++++++--------
src/wasm/src/lib.rs | 16 ++----
4 files changed, 36 insertions(+), 61 deletions(-)
diff --git a/example/gauge/Components/Pages/Auth/Auth.tsx b/example/gauge/Components/Pages/Auth/Auth.tsx
index 132d830..077763d 100644
--- a/example/gauge/Components/Pages/Auth/Auth.tsx
+++ b/example/gauge/Components/Pages/Auth/Auth.tsx
@@ -1,6 +1,5 @@
import { ComponentProps, DisplayComponent, FSComponent, VNode } from "@microsoft/msfs-sdk";
import {
- DownloadProgressPhase,
NavigationDataStatus,
NavigraphEventType,
NavigraphNavigationDataInterface,
@@ -29,22 +28,7 @@ export class AuthPage extends DisplayComponent<AuthPageProps> {
super(props);
this.props.navigationDataInterface.onEvent(NavigraphEventType.DownloadProgress, data => {
- switch (data.phase) {
- case DownloadProgressPhase.Downloading:
- this.displayMessage("Downloading navigation data...");
- break;
- case DownloadProgressPhase.Cleaning:
- if (!data.deleted) return;
- this.displayMessage(`Cleaning destination directory. ${data.deleted} files deleted so far`);
- break;
- case DownloadProgressPhase.Extracting: {
- // Ensure non-null
- if (!data.unzipped || !data.total_to_unzip) return;
- const percent = Math.round((data.unzipped / data.total_to_unzip) * 100);
- this.displayMessage(`Unzipping files... ${percent}% complete`);
- break;
- }
- }
+ this.displayMessage(`Downloaded ${data.downloaded_bytes}/${data.total_bytes} bytes`);
});
}
diff --git a/src/ts/interface/NavigationDataInterfaceTypes.ts b/src/ts/interface/NavigationDataInterfaceTypes.ts
index 3e20254..180f06d 100644
--- a/src/ts/interface/NavigationDataInterfaceTypes.ts
+++ b/src/ts/interface/NavigationDataInterfaceTypes.ts
@@ -11,17 +11,9 @@ export enum NavigraphEventType {
DownloadProgress = "DownloadProgress",
}
-export enum DownloadProgressPhase {
- Downloading = "Downloading",
- Cleaning = "Cleaning",
- Extracting = "Extracting",
-}
-
export interface DownloadProgressData {
- phase: DownloadProgressPhase;
- deleted: number | null;
- total_to_unzip: number | null;
- unzipped: number | null;
+ total_bytes: number;
+ downloaded_bytes: number;
}
export enum NavigraphFunction {
diff --git a/src/wasm/src/funcs.rs b/src/wasm/src/funcs.rs
index 97413e1..02a6282 100644
--- a/src/wasm/src/funcs.rs
+++ b/src/wasm/src/funcs.rs
@@ -18,7 +18,7 @@ use crate::{
WORK_NAVIGATION_DATA_FOLDER,
},
futures::AsyncNetworkRequest,
- DownloadProgressEvent, DownloadProgressPhase, InterfaceEvent,
+ DownloadProgressEvent, InterfaceEvent,
};
/// The URL to get the latest available cycle number
@@ -66,12 +66,30 @@ impl Function for DownloadNavigationData {
type ReturnType = ();
async fn run(&mut self) -> Result<Self::ReturnType> {
- // Send an initial progress event TODO: remove these in a breaking version, these are only here for backwards compatibility
+ // Figure out total size of download (this request is acting like a HEAD since we don't have those in this environment. Nothing actually gets downloaded since we are constraining the range)
+ let request = NetworkRequestBuilder::new(&self.url)
+ .context("can't create new NetworkRequestBuilder")?
+ .with_header(&format!("Range: bytes=0-0"))
+ .context(".with_header() returned None")?
+ .get()
+ .context(".get() returned None")?;
+
+ request.wait_for_data().await?;
+
+ // Try parsing the content-range header
+ let total_bytes = request
+ .header_section("content-range")
+ .context("no content-range header")?
+ .trim()
+ .split("/")
+ .last()
+ .ok_or(anyhow!("invalid content-range"))?
+ .parse::<usize>()?;
+
+ // Report the amount
InterfaceEvent::send_download_progress_event(DownloadProgressEvent {
- phase: DownloadProgressPhase::Downloading,
- deleted: None,
- total_to_unzip: None,
- unzipped: None,
+ total_bytes,
+ downloaded_bytes: 0,
})?;
// We need to download the data in chunks of DOWNLOAD_CHUNK_SIZE_BYTES to avoid a timeout, so we need to keep track of a "working" accumulation of all responses
@@ -121,6 +139,12 @@ impl Function for DownloadNavigationData {
}
current_byte_index += content_length;
+
+ // Send the current download amount
+ InterfaceEvent::send_download_progress_event(DownloadProgressEvent {
+ total_bytes,
+ downloaded_bytes: current_byte_index,
+ })?;
}
// Only close connection if DATABASE_STATE has already been initialized - otherwise we end up unnecessarily copying the bundled data and instantly replacing it (due to initialization logic in database state)
@@ -132,21 +156,6 @@ impl Function for DownloadNavigationData {
.close_connection()?;
}
- // Send the deleting and extraction events
- InterfaceEvent::send_download_progress_event(DownloadProgressEvent {
- phase: DownloadProgressPhase::Cleaning,
- deleted: Some(2),
- total_to_unzip: None,
- unzipped: None,
- })?;
-
- InterfaceEvent::send_download_progress_event(DownloadProgressEvent {
- phase: DownloadProgressPhase::Extracting,
- deleted: None,
- total_to_unzip: Some(2),
- unzipped: None,
- })?;
-
// Load the zip archive
let mut zip = ZipArchive::new(Cursor::new(bytes))?;
diff --git a/src/wasm/src/lib.rs b/src/wasm/src/lib.rs
index 86c0a6a..916f248 100644
--- a/src/wasm/src/lib.rs
+++ b/src/wasm/src/lib.rs
@@ -21,28 +21,18 @@ mod sentry_gauge;
/// Amount of MS between dispatches of the heartbeat commbus event
const HEARTBEAT_INTERVAL_MS: u128 = 1000;
-/// The current phase of downloading
-#[derive(Serialize)]
-pub enum DownloadProgressPhase {
- Downloading,
- Cleaning,
- Extracting,
-}
-
/// The data associated with the `DownloadProgress` event
#[derive(Serialize)]
pub struct DownloadProgressEvent {
- pub phase: DownloadProgressPhase,
- pub deleted: Option<usize>,
- pub total_to_unzip: Option<usize>,
- pub unzipped: Option<usize>,
+ pub total_bytes: usize,
+ pub downloaded_bytes: usize,
}
/// The types of events that can be emitted from the interface
#[derive(Serialize)]
enum NavigraphEventType {
Heartbeat,
- DownloadProgress, // TODO: remove in a future version. here for backwards compatibility
+ DownloadProgress,
}
/// The structure of an event message
From 0f995e4593f273bba3d79be8bdf703215023560d Mon Sep 17 00:00:00 2001
From: Jack Lavigne <jacklavigne00@gmail.com>
Date: Mon, 7 Apr 2025 16:08:38 +0200
Subject: [PATCH 3/3] feat: include chunk number in download report
---
example/gauge/Components/Pages/Auth/Auth.tsx | 4 +-
.../interface/NavigationDataInterfaceTypes.ts | 2 +
src/wasm/Cargo.toml | 2 +-
src/wasm/src/funcs.rs | 71 ++++++-------------
src/wasm/src/lib.rs | 6 ++
5 files changed, 33 insertions(+), 52 deletions(-)
diff --git a/example/gauge/Components/Pages/Auth/Auth.tsx b/example/gauge/Components/Pages/Auth/Auth.tsx
index 077763d..eabd267 100644
--- a/example/gauge/Components/Pages/Auth/Auth.tsx
+++ b/example/gauge/Components/Pages/Auth/Auth.tsx
@@ -28,7 +28,9 @@ export class AuthPage extends DisplayComponent<AuthPageProps> {
super(props);
this.props.navigationDataInterface.onEvent(NavigraphEventType.DownloadProgress, data => {
- this.displayMessage(`Downloaded ${data.downloaded_bytes}/${data.total_bytes} bytes`);
+ this.displayMessage(
+ `Downloaded ${data.downloaded_bytes}/${data.total_bytes} bytes (chunk ${data.current_chunk}/${data.total_chunks})`,
+ );
});
}
diff --git a/src/ts/interface/NavigationDataInterfaceTypes.ts b/src/ts/interface/NavigationDataInterfaceTypes.ts
index 180f06d..14ea3d4 100644
--- a/src/ts/interface/NavigationDataInterfaceTypes.ts
+++ b/src/ts/interface/NavigationDataInterfaceTypes.ts
@@ -14,6 +14,8 @@ export enum NavigraphEventType {
export interface DownloadProgressData {
total_bytes: number;
downloaded_bytes: number;
+ current_chunk: number;
+ total_chunks: number;
}
export enum NavigraphFunction {
diff --git a/src/wasm/Cargo.toml b/src/wasm/Cargo.toml
index fc3b7ff..3117f75 100644
--- a/src/wasm/Cargo.toml
+++ b/src/wasm/Cargo.toml
@@ -1,6 +1,6 @@
[package]
name = "msfs-navigation-data-interface"
-version = "1.2.0-rc1"
+version = "1.2.0-rc2"
edition = "2021"
[lib]
diff --git a/src/wasm/src/funcs.rs b/src/wasm/src/funcs.rs
index 02a6282..3128cee 100644
--- a/src/wasm/src/funcs.rs
+++ b/src/wasm/src/funcs.rs
@@ -86,65 +86,36 @@ impl Function for DownloadNavigationData {
.ok_or(anyhow!("invalid content-range"))?
.parse::<usize>()?;
- // Report the amount
- InterfaceEvent::send_download_progress_event(DownloadProgressEvent {
- total_bytes,
- downloaded_bytes: 0,
- })?;
+ // Total amount of chunks to download
+ let total_chunks = total_bytes.div_ceil(DOWNLOAD_CHUNK_SIZE_BYTES);
// We need to download the data in chunks of DOWNLOAD_CHUNK_SIZE_BYTES to avoid a timeout, so we need to keep track of a "working" accumulation of all responses
let mut bytes = vec![];
- let mut current_byte_index = 0;
- loop {
+ for i in 0..total_chunks {
+ // Calculate the range for the current chunk
+ let range_start = i * DOWNLOAD_CHUNK_SIZE_BYTES;
+ let range_end = ((i + 1) * DOWNLOAD_CHUNK_SIZE_BYTES - 1).min(total_bytes - 1);
+
+ // Report the current download progress
+ InterfaceEvent::send_download_progress_event(DownloadProgressEvent {
+ total_bytes,
+ downloaded_bytes: range_start,
+ current_chunk: i,
+ total_chunks,
+ })?;
+
// Dispatch the request
- let range_end = current_byte_index + DOWNLOAD_CHUNK_SIZE_BYTES - 1;
- let request = NetworkRequestBuilder::new(&self.url)
+ let data = NetworkRequestBuilder::new(&self.url)
.context("can't create new NetworkRequestBuilder")?
- .with_header(&format!("Range: bytes={current_byte_index}-{range_end}"))
+ .with_header(&format!("Range: bytes={range_start}-{range_end}"))
.context(".with_header() returned None")?
.get()
- .context(".get() returned None")?;
-
- request.wait_for_data().await?;
-
- // Get the size of actual data. The response will be as long as the requested range is, but content-length contains the amount we actually want to read
- let content_length = request
- .header_section("content-length")
- .context("no content-length header")?
- .trim()
- .parse::<usize>()?;
-
- // Check if we somehow have no more data (file size would be a perfect multiple of DOWNLOAD_CHUNK_SIZE_BYTES)
- if content_length == 0 {
- break;
- }
-
- let data = request.data().ok_or(anyhow!("no data"))?;
+ .context(".get() returned None")?
+ .wait_for_data()
+ .await?;
- // Make sure we don't panic if server sent less data than claimed (should never happen, but avoid a panic)
- if data.len() < content_length {
- return Err(anyhow!(
- "Received less data ({}) than content-length ({})",
- data.len(),
- content_length
- ));
- }
-
- bytes.write_all(&data[..content_length])?;
-
- // Check if we have hit the last chunk
- if content_length < DOWNLOAD_CHUNK_SIZE_BYTES {
- break;
- }
-
- current_byte_index += content_length;
-
- // Send the current download amount
- InterfaceEvent::send_download_progress_event(DownloadProgressEvent {
- total_bytes,
- downloaded_bytes: current_byte_index,
- })?;
+ bytes.write_all(&data)?;
}
// Only close connection if DATABASE_STATE has already been initialized - otherwise we end up unnecessarily copying the bundled data and instantly replacing it (due to initialization logic in database state)
diff --git a/src/wasm/src/lib.rs b/src/wasm/src/lib.rs
index 916f248..0994442 100644
--- a/src/wasm/src/lib.rs
+++ b/src/wasm/src/lib.rs
@@ -24,8 +24,14 @@ const HEARTBEAT_INTERVAL_MS: u128 = 1000;
/// The data associated with the `DownloadProgress` event
#[derive(Serialize)]
pub struct DownloadProgressEvent {
+ /// The total amount of bytes to download
pub total_bytes: usize,
+ /// The amount of bytes downloaded
pub downloaded_bytes: usize,
+ /// The chunk number (starting at 0) of the current download
+ pub current_chunk: usize,
+ /// The total number of chunks needed to download
+ pub total_chunks: usize,
}
/// The types of events that can be emitted from the interface