From 34a98a99bd2d675774aed53b8d11a4d6445e9267 Mon Sep 17 00:00:00 2001 From: Wolfgang Silbermayr Date: Mon, 27 Feb 2023 13:59:09 +0100 Subject: [PATCH 1/2] rustfmt --- main.rs | 62 ++++++++++++++++++++++++++++++++++----------------------- 1 file changed, 37 insertions(+), 25 deletions(-) diff --git a/main.rs b/main.rs index e18a402..ad00fdd 100644 --- a/main.rs +++ b/main.rs @@ -1,15 +1,14 @@ -use lambda_runtime::{handler_fn, Error}; use aws_lambda_events::s3::S3Event; -use serde_json::Value; -use flate2::read::{GzDecoder}; +use flate2::read::GzDecoder; use flate2::write::GzEncoder; -use flate2::GzBuilder; use flate2::Compression; -use std::io::Read; +use flate2::GzBuilder; +use lambda_runtime::{handler_fn, Error}; +use serde_json::Value; +use std::fs::File; use std::io::prelude::*; use std::io::BufReader; -use std::fs::File; - +use std::io::Read; #[tokio::main] async fn main() -> Result<(), lambda_runtime::Error> { @@ -19,21 +18,26 @@ async fn main() -> Result<(), lambda_runtime::Error> { } async fn handler(req: Value, _ctx: lambda_runtime::Context) -> Result<(), Box> { - let events: S3Event = serde_json::from_value(req).unwrap();; + let events: S3Event = serde_json::from_value(req).unwrap(); for e in events.records { let bucket_name: String = e.s3.bucket.name.expect("Unable to get s3 bucket name."); let key: String = e.s3.object.key.expect("unable to get s3 file key"); - + let config: aws_config::SdkConfig = aws_config::load_from_env().await; let s3_client: aws_sdk_s3::Client = aws_sdk_s3::Client::new(&config); let data: aws_lambda_events::bytes::Bytes = s3_client - .get_object() - .bucket(&bucket_name) - .key(&key) - .send() - .await.unwrap().body - .collect().await.unwrap().into_bytes(); - + .get_object() + .bucket(&bucket_name) + .key(&key) + .send() + .await + .unwrap() + .body + .collect() + .await + .unwrap() + .into_bytes(); + let mut d: GzDecoder<&[u8]> = GzDecoder::new(&data[..]); let mut csv_data: String = String::new(); d.read_to_string(&mut csv_data).unwrap(); @@ -48,27 +52,35 @@ async fn handler(req: Value, _ctx: lambda_runtime::Context) -> Result<(), Box = GzBuilder::new() - .filename("tab_converted.txt") - .write(f, Compression::default()); - gz.write_all(tab_converted.as_bytes()).expect("failed to write bytes to file"); + .filename("tab_converted.txt") + .write(f, Compression::default()); + gz.write_all(tab_converted.as_bytes()) + .expect("failed to write bytes to file"); gz.finish().expect("failed to flush bytes to file"); let file: File = File::open("/tmp/file.gz").expect("problem reading file"); let mut reader: BufReader = BufReader::new(file); let mut buffer: Vec = Vec::new(); - + reader.read_to_end(&mut buffer).expect("error"); let remote_uri: &String = &key.replace("fixed_width_raw/", "tab_converted/"); - s3_client.put_object().bucket(&bucket_name).key(remote_uri).body(buffer.into()).send().await.unwrap(); - + s3_client + .put_object() + .bucket(&bucket_name) + .key(remote_uri) + .body(buffer.into()) + .send() + .await + .unwrap(); } Ok(()) } - From c4b24d94254d130f58e53d92f7d393b92cba39e1 Mon Sep 17 00:00:00 2001 From: Wolfgang Silbermayr Date: Mon, 27 Feb 2023 14:11:11 +0100 Subject: [PATCH 2/2] Some optimizations --- main.rs | 45 ++++++++++++++++----------------------------- 1 file changed, 16 insertions(+), 29 deletions(-) diff --git a/main.rs b/main.rs index ad00fdd..3b35678 100644 --- a/main.rs +++ b/main.rs @@ -1,14 +1,12 @@ use aws_lambda_events::s3::S3Event; -use flate2::read::GzDecoder; +use flate2::bufread::GzDecoder; use flate2::write::GzEncoder; use flate2::Compression; use flate2::GzBuilder; use lambda_runtime::{handler_fn, Error}; use serde_json::Value; -use std::fs::File; use std::io::prelude::*; -use std::io::BufReader; -use std::io::Read; +use std::io::{BufRead, BufReader}; #[tokio::main] async fn main() -> Result<(), lambda_runtime::Error> { @@ -38,46 +36,35 @@ async fn handler(req: Value, _ctx: lambda_runtime::Context) -> Result<(), Box = GzDecoder::new(&data[..]); - let mut csv_data: String = String::new(); - d.read_to_string(&mut csv_data).unwrap(); + let decoder: BufReader> = BufReader::new(GzDecoder::new(&data[..])); - let split: std::str::Lines<'_> = csv_data.lines(); - let result_vector: Vec<&str> = split.collect(); + let output = Vec::new(); + let mut encoder: GzEncoder> = GzBuilder::new() + .filename("tab_converted.txt") + .write(output, Compression::default()); - let mut tab_converted: String = String::new(); - for line in result_vector.iter().skip(1) { + for line in decoder.lines().skip(1) { + let line = line.unwrap(); let date: &&str = &line[0..14].trim(); let serial_number: &&str = &line[15..35].trim(); let model: &&str = &line[36..78].trim(); let capacity_bytes: &&str = &line[79..97].trim(); let failure: &&str = &line[98..108].trim(); - let tab_line: String = format!( - "{}\t{}\t{}\t{}\t{}\n", + writeln!( + &mut encoder, + "{}\t{}\t{}\t{}\t{}", date, serial_number, model, capacity_bytes, failure - ); - tab_converted.push_str(&tab_line); + ) + .unwrap(); } - let f: File = File::create("/tmp/file.gz").expect("failed to create file"); - let mut gz: GzEncoder = GzBuilder::new() - .filename("tab_converted.txt") - .write(f, Compression::default()); - gz.write_all(tab_converted.as_bytes()) - .expect("failed to write bytes to file"); - gz.finish().expect("failed to flush bytes to file"); - - let file: File = File::open("/tmp/file.gz").expect("problem reading file"); - let mut reader: BufReader = BufReader::new(file); - let mut buffer: Vec = Vec::new(); - - reader.read_to_end(&mut buffer).expect("error"); + let output = encoder.finish().expect("failed to flush bytes to file"); let remote_uri: &String = &key.replace("fixed_width_raw/", "tab_converted/"); s3_client .put_object() .bucket(&bucket_name) .key(remote_uri) - .body(buffer.into()) + .body(output.into()) .send() .await .unwrap();