update: streaming + performance uplift

Author: tadghh

Cargo.lock

@@ -1,17 +1,21 @@
 [package]
 edition="2021"
 name   ="json-gen-actix"
-version="0.1.0"
+version="0.1.1"
 
 [dependencies]
-actix-web  ={ version="4.9.0", default-features=false, features=["macros"] }
-fake       ={ version="3.1.0", default-features=false, features=["derive"] }
-num_cpus   ="1.16.0"
-parking_lot="0.12.3"
-rand       ="0.8.5"
-rand_chacha="0.3.1"
-rayon      ={ version="1.10.0", default-features=false }
-serde      ={ version="1.0.217", features=["derive"] }
+actix-web   ={ version="4.9.0", default-features=false, features=["macros"] }
+dtoa        ="1.0.9"
+fake        ={ version="3.1.0", default-features=false, features=["derive"] }
+itoa        ="1.0.14"
+num_cpus    ="1.16.0"
+parking_lot ="0.12.3"
+rand        ="0.8.5"
+rand_chacha ="0.3.1"
+rayon       ={ version="1.10.0", default-features=false }
+serde       ={ version="1.0.217", features=["derive"] }
+tokio       ="1.43.0"
+tokio-stream="0.1.17"
 
 [profile.release]
 codegen-units=1
@@ -20,3 +24,10 @@ lto          ="thin"
 opt-level    ="z"
 panic        ="abort"
 strip        =true
+
+[dev-dependencies]
+criterion="0.5"
+
+[[bench]]
+harness=false
+name   ="data_generation"

Cargo.toml

@@ -0,0 +1,127 @@
+use actix_web::web;
+use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion};
+use rand::{Rng, SeedableRng};
+use rand_chacha::ChaCha8Rng;
+use std::sync::Arc;
+
+use json_gen_actix::{
+    processing::{
+        write_location_csv_simd, write_location_json_simd, BusinessLocation, DataPools,
+        JsonPatterns, OutputFormat, StreamGenerator,
+    },
+    util::ProgressInfo,
+};
+
+fn bench_formats(c: &mut Criterion) {
+    let mut group = c.benchmark_group("data_generation");
+    let data_pools = web::Data::new(Arc::new(DataPools::new()));
+    let rng = ChaCha8Rng::seed_from_u64(42);
+
+    for size in [64, 256, 1024].iter() {
+        group.bench_with_input(BenchmarkId::new("json_pretty", size), size, |b, &size| {
+            b.iter(|| {
+                let mut generator = StreamGenerator::new(
+                    0,
+                    rng.clone(),
+                    data_pools.clone(),
+                    true,
+                    OutputFormat::JSON,
+                    true,
+                    Arc::new(ProgressInfo::new(1.0)),
+                    size * 1024,
+                );
+                generator.generate_chunk()
+            })
+        });
+
+        group.bench_with_input(BenchmarkId::new("json_compact", size), size, |b, &size| {
+            b.iter(|| {
+                let mut generator = StreamGenerator::new(
+                    0,
+                    rng.clone(),
+                    data_pools.clone(),
+                    false,
+                    OutputFormat::JSON,
+                    true,
+                    Arc::new(ProgressInfo::new(1.0)),
+                    size * 1024,
+                );
+                generator.generate_chunk()
+            })
+        });
+
+        group.bench_with_input(BenchmarkId::new("csv", size), size, |b, &size| {
+            b.iter(|| {
+                let mut generator = StreamGenerator::new(
+                    0,
+                    rng.clone(),
+                    data_pools.clone(),
+                    false,
+                    OutputFormat::CSV,
+                    true,
+                    Arc::new(ProgressInfo::new(1.0)),
+                    size * 1024,
+                );
+                generator.generate_chunk()
+            })
+        });
+    }
+    group.finish();
+}
+
+fn bench_simd_operations(c: &mut Criterion) {
+    let mut group = c.benchmark_group("simd_ops");
+    let location = BusinessLocation {
+        id: 1,
+        name: "Test Company".into(),
+        industry: "Technology".into(),
+        revenue: 1000000.0,
+        employees: 100,
+        city: "Test City".into(),
+        state: "Test State".into(),
+        country: "Test Country".into(),
+    };
+
+    group.bench_function("json_simd_write", |b| {
+        b.iter(|| {
+            let mut output = Vec::with_capacity(1024);
+            write_location_json_simd(&location, &mut output, true, &JsonPatterns::new(), true);
+        })
+    });
+
+    group.bench_function("csv_simd_write", |b| {
+        b.iter(|| {
+            let mut output = Vec::with_capacity(1024);
+            write_location_csv_simd(&location, &mut output);
+        })
+    });
+    group.finish();
+}
+
+fn bench_data_generation(c: &mut Criterion) {
+    let mut group = c.benchmark_group("data_gen");
+    let data_pools = web::Data::new(Arc::new(DataPools::new()));
+    let rng = ChaCha8Rng::seed_from_u64(42);
+
+    group.bench_function("location_gen", |b| {
+        b.iter(|| BusinessLocation {
+            id: 1,
+            name: data_pools.names[0].clone(),
+            industry: data_pools.industries[0].clone(),
+            revenue: rng.clone().gen_range(100000.0..100000000.0),
+            employees: rng.clone().gen_range(10..10000),
+            city: data_pools.cities[0].clone(),
+            state: data_pools.states[0].clone(),
+            country: data_pools.countries[0].clone(),
+        })
+    });
+    group.finish();
+}
+
+criterion_group!(
+    benches,
+    bench_formats,
+    bench_simd_operations,
+    bench_data_generation
+);
+criterion_main!(benches);

benches/data_generation.rs

@@ -18,3 +18,7 @@ curl "http://127.0.0.1:8080/generate?size=100mb&format=csv"
 - The `size` parameter specifies the target size of the generated content.
 - The `format` parameter supports either JSON or CSV.
   - default: `json`
+
+### Bugs
+
+- Progress indicator may end up falling short.

readme.md

@@ -0,0 +1,3 @@
+#![feature(portable_simd)]
+pub mod processing;
+pub mod util;