Merge branch 'video-fullscreen'

ads-webasm/Cargo.toml

@@ -6,10 +6,8 @@ edition = "2021"
 rust-version = "1.76"
 
 [dependencies]
-log = "0.4"
 yew = { version = "0.21.0", features = ["csr"] }
 js-sys = "0.3"
-gloo = "0.8.0"
 gloo-file = "0.2"
 wasm-bindgen = "0.2.92"
 serde = "1.0"
@@ -20,10 +18,6 @@ chrono = { version = "0.4.23", default-features = false, features = [
     "std",
     "wasmbind",
 ] }
-uuid = { version = "1.2.2", default-features = false, features = [
-    "js",
-    "v4",
-] } # add feature flag required for uuid crate
 csv = { version = "1.1", optional = true }
 
 yew-tincture = "0.2.2"

ads-webasm/src/components/mod.rs

@@ -1,6 +1,3 @@
-mod video_field;
-pub use self::video_field::VideoField;
-
 mod ranged_value;
 pub use self::ranged_value::RangedValue;
 

ads-webasm/src/lib.rs

@@ -1,3 +1,2 @@
 #![recursion_limit = "1000"]
 pub mod components;
-pub mod video_data;

http-video-streaming/http-video-streaming-types/src/lib.rs

@@ -18,6 +18,7 @@ pub struct Point {
 
 #[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
 pub struct ToClient {
+    pub fno: u64,
     pub firehose_frame_data_url: String,
     pub found_points: Vec<Point>,
     /// Indicates which region of the entire image is "valid".
@@ -28,7 +29,6 @@ pub struct ToClient {
     pub valid_display: Option<Shape>,
     /// Annotations associated with this particular image, e.g. from tracking.
     pub annotations: Vec<DrawableShape>,
-    pub fno: u64,
     pub ts_rfc3339: String, // timestamp in RFC3339 format
     pub ck: ConnectionKey,
 }
@@ -72,14 +72,6 @@ pub enum Shape {
     MultipleCircles(Vec<CircleParams>),
 }
 
-// from client to server
-#[derive(Debug, Serialize, Deserialize, Clone)]
-pub struct FirehoseCallbackInner {
-    pub ck: ConnectionKey,
-    pub fno: usize,
-    pub ts_rfc3339: String,
-}
-
 #[derive(Debug, Serialize, Deserialize, Clone, PartialEq)]
 pub struct RgbaColor {
     r: u8,

http-video-streaming/src/lib.rs

@@ -9,9 +9,7 @@ use basic_frame::DynamicFrame;
 use bui_backend_session_types::ConnectionKey;
 use event_stream_types::{ConnectionEvent, ConnectionEventType, EventChunkSender};
 
-pub use http_video_streaming_types::{
-    CircleParams, DrawableShape, FirehoseCallbackInner, Point, Shape, ToClient,
-};
+pub use http_video_streaming_types::{CircleParams, DrawableShape, Point, Shape, ToClient};
 
 type Result<T> = std::result::Result<T, Error>;
 
@@ -43,12 +41,6 @@ fn _test_annotated_frame_is_send() {
     implements::<AnnotatedFrame>();
 }
 
-#[derive(Debug)]
-pub struct FirehoseCallback {
-    pub arrival_time: chrono::DateTime<chrono::Utc>,
-    pub inner: FirehoseCallbackInner,
-}
-
 struct PerSender {
     out: EventChunkSender,
     frame_lifo: Option<Arc<Mutex<AnnotatedFrame>>>,
@@ -88,17 +80,7 @@ impl PerSender {
         self.fno += 1;
         self.frame_lifo = Some(frame);
     }
-    fn got_callback(&mut self, msg: FirehoseCallback) {
-        match chrono::DateTime::parse_from_rfc3339(&msg.inner.ts_rfc3339) {
-            // match chrono::DateTime<chrono::FixedOffset>::parse_from_rfc3339(&msg.inner.ts_rfc3339) {
-            Ok(sent_time) => {
-                let latency = msg.arrival_time.signed_duration_since(sent_time);
-                tracing::trace!("latency: {:?}", latency);
-            }
-            Err(e) => {
-                tracing::error!("failed to parse timestamp in callback: {:?}", e);
-            }
-        }
+    fn got_callback(&mut self, _msg: ConnectionKey) {
         self.ready_to_send = true;
     }
     async fn service(&mut self) -> Result<()> {
@@ -204,11 +186,11 @@ impl TaskState {
         }
         Ok(())
     }
-    fn handle_callback(&mut self, callback: FirehoseCallback) -> Result<()> {
-        if let Some(ps) = self.per_sender_map.get_mut(&callback.inner.ck) {
-            ps.got_callback(callback)
+    fn handle_callback(&mut self, ck: ConnectionKey) -> Result<()> {
+        if let Some(ps) = self.per_sender_map.get_mut(&ck) {
+            ps.got_callback(ck)
         } else {
-            tracing::warn!(
+            tracing::debug!(
                 "Got firehose_callback for non-existant connection key. \
                             Did connection disconnect?"
             );
@@ -220,7 +202,7 @@ impl TaskState {
 pub async fn firehose_task(
     connection_callback_rx: tokio::sync::mpsc::Receiver<ConnectionEvent>,
     mut firehose_rx: tokio::sync::mpsc::Receiver<AnnotatedFrame>,
-    firehose_callback_rx: tokio::sync::mpsc::Receiver<FirehoseCallback>,
+    firehose_callback_rx: tokio::sync::mpsc::Receiver<ConnectionKey>,
 ) -> Result<()> {
     // Wait for the first frame so we don't need to deal with an Option<>.
     let first_frame = firehose_rx.recv().await.unwrap();
@@ -235,6 +217,7 @@ pub async fn firehose_task(
         tokio_stream::wrappers::ReceiverStream::new(connection_callback_rx);
     let mut firehose_callback_rx =
         tokio_stream::wrappers::ReceiverStream::new(firehose_callback_rx);
+    let mut interval = tokio::time::interval(std::time::Duration::from_millis(100));
     loop {
         tokio::select! {
             opt_new_connection = connection_callback_rx.next() => {
@@ -273,8 +256,10 @@ pub async fn firehose_task(
                     }
                 }
             },
+            _ = interval.tick() => {
+                task_state.service().await?;
+            }
         }
-        task_state.service().await?; // should use a timer for this??
     }
     tracing::debug!("firehose task done.");
     Ok(())

strand-cam-storetype/Cargo.toml

@@ -10,6 +10,7 @@ serde = { version = "1.0", default-features = false, features = ["derive"] }
 serde_json = "1.0"
 serde_yaml = "0.9"
 
+bui-backend-session-types = { path = "../bui-backend-session/types" }
 led-box-comms = { path = "../led-box-comms" }
 ci2-types = { path = "../ci2/ci2-types" }
 ci2-remote-control = { path = "../ci2-remote-control" }

strand-cam-storetype/src/lib.rs

@@ -3,7 +3,6 @@ use std::net::{IpAddr, Ipv4Addr, SocketAddr, SocketAddrV4};
 use rust_cam_bui_types::RecordingPath;
 use serde::{Deserialize, Serialize};
 
-use http_video_streaming_types::FirehoseCallbackInner;
 use http_video_streaming_types::{CircleParams, Shape};
 
 use ci2_remote_control::{BitrateSelection, CodecSelection, RecordingFrameRate, TagFamily};
@@ -29,6 +28,7 @@ pub use led_box_comms::ToDevice as ToLedBoxDevice;
 // `http://strand-cam/strand-cam-events`.
 pub const STRAND_CAM_EVENTS_URL_PATH: &str = "strand-cam-events";
 pub const STRAND_CAM_EVENT_NAME: &str = "strand-cam";
+pub const CONN_KEY_EVENT_NAME: &str = "connection-key";
 
 #[derive(Debug, PartialEq, Clone, Serialize, Deserialize)]
 #[serde(deny_unknown_fields)]
@@ -213,7 +213,7 @@ impl Default for CheckerboardCalState {
 #[serde(deny_unknown_fields)]
 pub enum CallbackType {
     ToCamera(ci2_remote_control::CamArg),
-    FirehoseNotify(FirehoseCallbackInner),
+    FirehoseNotify(bui_backend_session_types::ConnectionKey),
     // used only with image-tracker crate
     TakeCurrentImageAsBackground,
     // used only with image-tracker crate

strand-cam/src/strand-cam.rs

@@ -31,7 +31,7 @@ use fmf::FMFWriter;
 use formats::PixFmt;
 use timestamped_frame::ExtraTimeData;
 
-use video_streaming::{AnnotatedFrame, FirehoseCallback};
+use video_streaming::AnnotatedFrame;
 
 use std::{
     path::{Path, PathBuf},
@@ -422,7 +422,7 @@ impl<T: std::fmt::Debug> IgnoreSendError for StdResult<(), tokio::sync::mpsc::er
 
 #[derive(Clone)]
 struct StrandCamCallbackSenders {
-    firehose_callback_tx: tokio::sync::mpsc::Sender<FirehoseCallback>,
+    firehose_callback_tx: tokio::sync::mpsc::Sender<ConnectionKey>,
     cam_args_tx: tokio::sync::mpsc::Sender<CamArg>,
     led_box_tx_std: tokio::sync::mpsc::Sender<ToLedBoxDevice>,
     #[allow(dead_code)]
@@ -802,6 +802,25 @@ async fn events_handler(
     let key = ConnectionSessionKey::new(session_key.0, addr);
     let (tx, body) = app_state.event_broadcaster.new_connection(key);
 
+    // Send the connection key
+    {
+        let frame_string = format!(
+            "event: {}\ndata: {}\n\n",
+            strand_cam_storetype::CONN_KEY_EVENT_NAME,
+            addr
+        );
+        match tx
+            .send(Ok(http_body::Frame::data(frame_string.into())))
+            .await
+        {
+            Ok(()) => {}
+            Err(tokio::sync::mpsc::error::SendError(_)) => {
+                // The receiver was dropped because the connection closed. Should probably do more here.
+                tracing::debug!("initial send error");
+            }
+        }
+    }
+
     // Send an initial copy of our state.
     let shared_store = app_state.shared_store_arc.read().as_ref().clone();
     let frame_string = to_event_frame(&shared_store);
@@ -865,16 +884,11 @@ async fn callback_handler(
                 .await
                 .ignore_send_error();
         }
-        CallbackType::FirehoseNotify(inner) => {
-            let arrival_time = chrono::Utc::now();
-            let fc = FirehoseCallback {
-                arrival_time,
-                inner,
-            };
+        CallbackType::FirehoseNotify(ck) => {
             app_state
                 .callback_senders
                 .firehose_callback_tx
-                .send(fc)
+                .send(ck)
                 .await
                 .ignore_send_error();
         }

strand-cam/yew_frontend/Cargo.toml

@@ -10,7 +10,6 @@ crate-type = ["cdylib", "rlib"]
 
 [dependencies]
 anyhow = "1.0"
-log = "0.4"
 wasm-logger = "0.2.0"
 wasm-bindgen = "0.2.92"
 wasm-bindgen-futures = "0.4"
@@ -22,9 +21,11 @@ serde = { version = "1.0", default-features = false }
 serde_json = "1.0"
 serde_yaml = "0.9"
 http = "0.2"
-backtrace = "0.3.40"                                  # here only to require this high version number, not used directly
 yew-tincture = "0.2.2"
+gloo-timers = "0.3.0"
+uuid = { version = "1.2.2", default-features = false, features = ["js", "v4"] }
 
+bui-backend-session-types = { path = "../../bui-backend-session/types" }
 strand-cam-storetype = { path = "../../strand-cam-storetype", default-features = false }
 flydra-feature-detector-types = { path = "../../flydra-feature-detector/flydra-feature-detector-types", default-features = false }
 http-video-streaming-types = { path = "../../http-video-streaming/http-video-streaming-types" }

strand-cam/yew_frontend/src/components/mod.rs

@@ -5,3 +5,6 @@ mod led_box_control;
 pub use self::led_box_control::LedBoxControl;
 
 mod led_control;
+
+mod video_field;
+pub(crate) use self::video_field::VideoField;