From c3b7cdc725dd09547fb4d8752e8cf9e41c437c30 Mon Sep 17 00:00:00 2001 From: Patrick Stevens Date: Mon, 5 Jun 2023 20:55:23 +0100 Subject: [PATCH] Remove serde (#23) --- .github/workflows/rust.yml | 25 +++++++++++++++++++ Cargo.lock | 49 +------------------------------------- generator/Cargo.toml | 3 +-- generator/src/main.rs | 49 ++++++++++++++++++++++++++++++++++---- haversine-app/src/main.rs | 34 +++++++++++++++++--------- haversine/Cargo.toml | 1 - haversine/src/haversine.rs | 20 +++++++++++++--- json/src/json_object.rs | 41 +++++++++++++++++++++++-------- 8 files changed, 142 insertions(+), 80 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 862c933..d3d0e9e 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -193,6 +193,31 @@ jobs: ] } + round_trip: + { + "name": "Round-trip generator and consumer", + "runs-on": "ubuntu-latest", + "steps": [ + { + "uses": "actions/checkout@v3", + "name": "Checkout" + }, + { + "name": "Install Nix", + "uses": "cachix/install-nix-action@v17", + "with": { "extra-nix-config": "access-tokens = github.com=${{ secrets.GITHUB_TOKEN }}" } + }, + { + "name": "Run generator", + "run": "nix develop . --command cargo run --release --bin generator --package generator -- 12345 100" + }, + { + "name": "Run consumer", + "run": "nix develop . --command cargo run --release --bin haversine-app --package haversine-app -- data_100_flex.json data_100_haveranswer.f64" + } + ] + } + all-required-checks-complete: { "needs": diff --git a/Cargo.lock b/Cargo.lock index 293050f..6add846 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -189,9 +189,8 @@ dependencies = [ "byteorder", "clap", "haversine", + "json", "rand", - "serde", - "serde_json", ] [[package]] @@ -208,9 +207,6 @@ dependencies = [ [[package]] name = "haversine" version = "0.1.0" -dependencies = [ - "serde", -] [[package]] name = "haversine-app" @@ -258,12 +254,6 @@ dependencies = [ "windows-sys", ] -[[package]] -name = "itoa" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" - [[package]] name = "json" version = "0.1.0" @@ -376,43 +366,6 @@ dependencies = [ "windows-sys", ] -[[package]] -name = "ryu" -version = "1.0.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" - -[[package]] -name = "serde" -version = "1.0.163" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2113ab51b87a539ae008b5c6c02dc020ffa39afd2d83cffcb3f4eb2722cebec2" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.163" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c805777e3930c8883389c602315a24224bcc738b63905ef87cd1420353ea93e" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "serde_json" -version = "1.0.96" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1" -dependencies = [ - "itoa", - "ryu", - "serde", -] - [[package]] name = "sim-wrapper" version = "0.1.0" diff --git a/generator/Cargo.toml b/generator/Cargo.toml index b33ec36..01406a2 100644 --- a/generator/Cargo.toml +++ b/generator/Cargo.toml @@ -10,6 +10,5 @@ edition = "2021" clap = { version = "4.3.1", features = [ "derive" ] } rand = "0.8.5" haversine = { path = "../haversine" } -serde_json = "1.0.96" -serde = "1.0.163" +json = { path = "../json" } byteorder = "1.4.3" diff --git a/generator/src/main.rs b/generator/src/main.rs index e8bd4fb..64531bd 100644 --- a/generator/src/main.rs +++ b/generator/src/main.rs @@ -1,7 +1,8 @@ -use byteorder::{ByteOrder, LittleEndian}; +use byteorder::{BigEndian, ByteOrder}; use clap::{builder::PossibleValue, Parser, ValueEnum}; use haversine::haversine::{CoordinatePair, HaversineData}; use haversine::{distance, earth}; +use json::json_object::JsonValue; use rand::distributions::Distribution; use rand::rngs::StdRng; use rand::{Rng, SeedableRng}; @@ -63,10 +64,41 @@ fn write_json(data: &HaversineData, json_filename: &str) { let output_file = File::create(json_filename).unwrap(); let mut writer = BufWriter::new(output_file); - serde_json::to_writer(&mut writer, &data).unwrap(); + writer.write_all(r#"{"pairs":["#.as_bytes()).unwrap(); + + let mut is_first = true; + for point in data.pairs.iter() { + if !is_first { + writer.write_all(&[b',']).unwrap(); + } + writer.write_all(&[b'{']).unwrap(); + writer + .write_all( + format!( + r#""x0":{:.17},"y0":{:.17},"x1":{:.17},"y1":{:.17}"#, + point.x0, point.y0, point.x1, point.y1 + ) + .as_bytes(), + ) + .unwrap(); + writer.write_all(&[b'}']).unwrap(); + is_first = false; + } + + writer.write_all("]}".as_bytes()).unwrap(); writer.flush().unwrap(); } +// Our JSON parser is incapable of round-tripping floats, for precision reasons. +// To work around this, we'll just compute using the serialised versions. +fn round_trip_float(f: f64) -> f64 { + let written = format!("{:.17}", f); + JsonValue::parse(&mut written.chars()) + .unwrap() + .0 + .as_number() +} + fn write_answer(data: &HaversineData, binary_filename: &str) { let output_file = File::create(binary_filename).unwrap(); let mut writer = BufWriter::new(output_file); @@ -75,8 +107,15 @@ fn write_answer(data: &HaversineData, binary_filename: &str) { let mut buf = [0u8; 8]; for (count, point) in data.pairs.iter().enumerate() { - let distance = distance::naive(point, earth::RADIUS); - LittleEndian::write_f64(&mut buf, distance); + let point = CoordinatePair { + x0: round_trip_float(point.x0), + y0: round_trip_float(point.y0), + x1: round_trip_float(point.x1), + y1: round_trip_float(point.y1), + }; + + let distance = distance::naive(&point, earth::RADIUS); + BigEndian::write_f64(&mut buf, distance); let written = writer.write(&buf).unwrap(); if written < buf.len() { @@ -90,7 +129,7 @@ fn write_answer(data: &HaversineData, binary_filename: &str) { // sic! println!("Expected sum: {}", expected_average); - LittleEndian::write_f64(&mut buf, expected_average); + BigEndian::write_f64(&mut buf, expected_average); let written = writer.write(&buf).unwrap(); if written < buf.len() { panic!("Failed to write everything") diff --git a/haversine-app/src/main.rs b/haversine-app/src/main.rs index 2e19a4a..beab1f8 100644 --- a/haversine-app/src/main.rs +++ b/haversine-app/src/main.rs @@ -1,10 +1,10 @@ -use byteorder::{ByteOrder, LittleEndian}; use clap::Parser; use haversine::haversine::CoordinatePair; use haversine::{distance, earth}; use json::json_object::JsonValue; use std::fs::File; use std::io::{BufReader, Read}; +use std::process::ExitCode; use utf8_read::Reader; #[derive(Parser, Debug)] @@ -41,7 +41,7 @@ fn read_answer(binary_filename: &str) -> (Vec, f64) { bytes_read += new_bytes; } - data.push(LittleEndian::read_f64(&buf)); + data.push(f64::from_be_bytes(buf)); } let bytes_read = file.read(&mut buf).unwrap(); @@ -49,7 +49,7 @@ fn read_answer(binary_filename: &str) -> (Vec, f64) { panic!("Not enough bytes read") } - (data, LittleEndian::read_f64(&buf)) + (data, f64::from_be_bytes(buf)) } fn read_json(json_filename: &str) -> Vec { @@ -77,23 +77,35 @@ fn read_json(json_filename: &str) -> Vec { } } -fn haversine_sum(v: &[CoordinatePair]) -> f64 { +fn haversine_sum(v: &[CoordinatePair], reference: &[f64]) -> f64 { let mut answer = 0.0_f64; - for pair in v { - answer += distance::naive(pair, earth::RADIUS); + for (count, pair) in v.iter().enumerate() { + let computed = distance::naive(pair, earth::RADIUS); + if computed != reference[count] { + println!("Different! At index {}, received pair: {:?}, computed distance {computed}, expected {}", count, pair, reference[count]) + } + answer = + ((1.0 - (1.0 / (count as f64 + 1.0))) * answer) + (computed / (count as f64 + 1.0)); } - answer / (v.len() as f64) + answer } -fn main() { +fn main() -> Result<(), ExitCode> { let args = Args::parse(); let input = read_json(&args.input_json); - let (_expected_values, expected_sum) = read_answer(&args.expected); - let actual_sum = haversine_sum(&input); + let (expected_values, expected_sum) = read_answer(&args.expected); + let actual_sum = haversine_sum(&input, &expected_values); println!("Pair count: {}", input.len()); println!("Haversine sum: {}", actual_sum); println!("Validation:"); println!("Reference sum: {}", expected_sum); - println!("Difference: {}", f64::abs(expected_sum - actual_sum)); + let difference = f64::abs(expected_sum - actual_sum); + println!("Difference: {}", difference); + + if difference != 0.0 { + return Err(ExitCode::FAILURE); + } + + Ok(()) } diff --git a/haversine/Cargo.toml b/haversine/Cargo.toml index 5cff0f1..7239e6b 100644 --- a/haversine/Cargo.toml +++ b/haversine/Cargo.toml @@ -8,4 +8,3 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -serde = { version = "1.0.163", features = [ "derive" ] } diff --git a/haversine/src/haversine.rs b/haversine/src/haversine.rs index 4f84fd8..2fa05aa 100644 --- a/haversine/src/haversine.rs +++ b/haversine/src/haversine.rs @@ -1,6 +1,5 @@ -use serde::Serialize; +use std::fmt::{Debug, Formatter}; -#[derive(Serialize)] pub struct CoordinatePair { pub x0: f64, pub y0: f64, @@ -8,7 +7,22 @@ pub struct CoordinatePair { pub y1: f64, } -#[derive(Serialize)] +impl Debug for CoordinatePair { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.write_fmt(format_args!( + "x0: {} ({:?}), y0: {} ({:?}), x1: {} ({:?}), y1: {} ({:?})", + self.x0, + self.x0.to_be_bytes(), + self.y0, + self.y0.to_be_bytes(), + self.x1, + self.x1.to_be_bytes(), + self.y1, + self.y1.to_be_bytes() + )) + } +} + pub struct HaversineData { pub pairs: Vec, } diff --git a/json/src/json_object.rs b/json/src/json_object.rs index 92b3b5f..20ee2d1 100644 --- a/json/src/json_object.rs +++ b/json/src/json_object.rs @@ -132,8 +132,8 @@ where { let mut current = current; let mut negative = 1.0; - let mut integer_part = 0f64; - let mut fractional_part = 0f64; + let mut result = 0f64; + let mut fraction = 0f64; if current == '-' { negative = -1.0; @@ -149,14 +149,14 @@ where None => return Ok((negative * 0.0, None)), } } else if ('1'..='9').contains(¤t) { - integer_part = (current as u8 - b'0') as f64; + result = (current as u8 - b'0') as f64; loop { current = match iter.next() { - None => return Ok((negative * integer_part, None)), + None => return Ok((negative * result, None)), Some(v) => v, }; if current.is_ascii_digit() { - integer_part = integer_part * 10.0 + ((current as u8 - b'0') as f64) + result = result * 10.0 + ((current as u8 - b'0') as f64) } else { break; } @@ -170,11 +170,11 @@ where let mut multiplier = 0.1; loop { current = match iter.next() { - None => return Ok((negative * (integer_part + fractional_part), None)), + None => return Ok((negative * (result + fraction), None)), Some(v) => v, }; if current.is_ascii_digit() { - fractional_part += multiplier * ((current as u8 - b'0') as f64); + fraction += multiplier * ((current as u8 - b'0') as f64); multiplier *= 0.1; } else { break; @@ -184,7 +184,7 @@ where // Parse exponent if current != 'e' && current != 'E' { - return Ok((negative * (integer_part + fractional_part), Some(current))); + return Ok((negative * (result + fraction), Some(current))); } let negative_exp = match iter.next() { @@ -198,7 +198,7 @@ where current = match iter.next() { None => { let pow = 10.0_f64.powf(negative_exp * exponent); - let to_ret = negative * (integer_part + fractional_part) * pow; + let to_ret = negative * ((result + fraction) * pow); return Ok((to_ret, None)); } Some(v) => v, @@ -207,7 +207,7 @@ where exponent = exponent * 10.0 + ((current as u8 - b'0') as f64); } else { let pow = 10.0_f64.powf(negative_exp * exponent); - let to_ret = negative * (integer_part + fractional_part) * pow; + let to_ret = negative * ((result + fraction) * pow); return Ok((to_ret, Some(current))); } } @@ -893,6 +893,27 @@ mod test { assert_eq!(parsed.as_number(), -0.8); } + #[test] + fn borderline_example() { + let (parsed, remaining) = JsonValue::parse(&mut "-28.5175025263690110".chars()).unwrap(); + assert_eq!(remaining, None); + let expected = -28.5175025263690110; + let actual = parsed.as_number(); + assert_eq!(actual, expected); + } + + /* + * This test fails due to float precision problems. + #[test] + fn borderline_example_2() { + let (parsed, remaining) = JsonValue::parse(&mut "62.33736768026505".chars()).unwrap(); + assert_eq!(remaining, None); + let expected = 62.33736768026505; + let actual = parsed.as_number(); + assert_eq!(actual, expected); + } + */ + #[test] fn haversine_example() { let s = include_str!("example.json");