fix: test proportion of each scenario of TPCC and add more indicators

KKould · KKould · commit c69ab15816a9 · 2024-11-14T02:26:05.000+08:00
diff --git a/README.md b/README.md
@@ -57,18 +57,6 @@ let tuples = fnck_sql.run("select * from t1")?;
 ### TPCC
 run `cargo run -p tpcc --release` to run tpcc
 
-- CPU: i9-13900HX
-- Memory: 32.0 GB
-- SSD: YMTC PC411-1024GB-B
-```shell
-<90th Percentile RT (MaxRT)>
-   New-Order : 0.882  (0.947)
-     Payment : 0.080  (0.095)
-Order-Status : 0.235  (0.255)
-    Delivery : 5.386  (5.658)
- Stock-Level : 0.001  (0.002)
-```
-
 #### PG Wire Service
 run `cargo run --features="net"` to start server
 ![start](./static/images/start.gif)
diff --git a/tpcc/src/main.rs b/tpcc/src/main.rs
@@ -5,6 +5,7 @@ use crate::order_stat::OrderStatTest;
 use crate::payment::PaymentTest;
 use crate::rt_hist::RtHist;
 use crate::slev::SlevTest;
+use crate::utils::SeqGen;
 use clap::Parser;
 use fnck_sql::db::{DBTransaction, DataBaseBuilder};
 use fnck_sql::errors::DatabaseError;
@@ -20,8 +21,16 @@ mod order_stat;
 mod payment;
 mod rt_hist;
 mod slev;
+mod utils;
 
 pub(crate) const ALLOW_MULTI_WAREHOUSE_TX: bool = true;
+pub(crate) const RT_LIMITS: [Duration; 5] = [
+    Duration::from_millis(500),
+    Duration::from_millis(500),
+    Duration::from_millis(500),
+    Duration::from_secs(8),
+    Duration::from_secs(2),
+];
 
 pub(crate) trait TpccTransaction<S: Storage> {
     type Args;
@@ -54,7 +63,7 @@ struct Args {
     path: String,
     #[clap(long, default_value = "5")]
     max_retry: usize,
-    #[clap(long, default_value = "1080")]
+    #[clap(long, default_value = "720")]
     measure_time: u64,
     #[clap(long, default_value = "1")]
     num_ware: usize,
@@ -72,6 +81,9 @@ fn main() -> Result<(), TpccError> {
     Load::load_ord(&mut rng, &database, args.num_ware)?;
 
     let mut rt_hist = RtHist::new();
+    let mut success = [0usize; 5];
+    let mut late = [0usize; 5];
+    let mut failure = [0usize; 5];
     let tests = vec![
         Box::new(NewOrdTest) as Box<dyn TpccTest<_>>,
         Box::new(PaymentTest),
@@ -81,64 +93,145 @@ fn main() -> Result<(), TpccError> {
     ];
     let tpcc_args = TpccArgs { joins: args.joins };
 
-    let tpcc_start = Instant::now();
     let duration = Duration::new(args.measure_time, 0);
     let mut round_count = 0;
+    let mut seq_gen = SeqGen::new(10, 10, 1, 1, 1);
+    let tpcc_start = Instant::now();
 
     while tpcc_start.elapsed() < duration {
-        for (i, tpcc_test) in tests.iter().enumerate() {
-            let mut is_succeed = false;
-            for j in 0..args.max_retry + 1 {
-                let transaction_start = Instant::now();
-                let mut tx = database.new_transaction()?;
-
-                if let Err(err) =
-                    tpcc_test.do_transaction(&mut rng, &mut tx, args.num_ware, &tpcc_args)
-                {
-                    eprintln!(
-                        "[{}] Error while doing transaction: {}",
-                        tpcc_test.name(),
-                        err
-                    );
+        let i = seq_gen.get();
+        let tpcc_test = &tests[i];
+
+        let mut is_succeed = false;
+        for j in 0..args.max_retry + 1 {
+            let transaction_start = Instant::now();
+            let mut tx = database.new_transaction()?;
+
+            if let Err(err) = tpcc_test.do_transaction(&mut rng, &mut tx, args.num_ware, &tpcc_args)
+            {
+                failure[i] += 1;
+                eprintln!(
+                    "[{}] Error while doing transaction: {}",
+                    tpcc_test.name(),
+                    err
+                );
+            } else {
+                let rt = transaction_start.elapsed();
+                rt_hist.hist_inc(i, rt);
+                is_succeed = true;
+
+                if rt <= RT_LIMITS[i] {
+                    success[i] += 1;
                 } else {
-                    rt_hist.hist_inc(i, transaction_start.elapsed());
-                    is_succeed = true;
-                    break;
-                }
-                if j < args.max_retry {
-                    println!("[{}] Retry for the {}th time", tpcc_test.name(), j + 1);
+                    late[i] += 1;
                 }
+                break;
             }
-            if !is_succeed {
-                return Err(TpccError::MaxRetry);
+            if j < args.max_retry {
+                println!("[{}] Retry for the {}th time", tpcc_test.name(), j + 1);
             }
         }
-        if round_count != 0 && round_count % 4 == 0 {
+        if !is_succeed {
+            return Err(TpccError::MaxRetry);
+        }
+        if round_count != 0 && round_count % 8 == 0 {
             println!(
-                "============ TPCC CheckPoint {} on round {round_count}: ===============",
-                round_count / 4
+                "[TPCC CheckPoint {} on round {round_count}][{}]: 90th Percentile RT: {:.3}",
+                round_count / 4,
+                tpcc_test.name(),
+                rt_hist.hist_ckp(i)
             );
-            for (i, name) in vec![
-                "New-Order",
-                "Payment",
-                "Order-Status",
-                "Delivery",
-                "Stock-Level",
-            ]
-            .into_iter()
-            .enumerate()
-            {
-                println!("{name} 90th Percentile RT: {:.3}", rt_hist.hist_ckp(i));
-            }
-            println!("==========================================================");
         }
         round_count += 1;
     }
+    let actual_tpcc_time = tpcc_start.elapsed();
+    println!("---------------------------------------------------");
+    // Raw Results
+    print_transaction(&success, &late, &failure, |name, success, late, failure| {
+        println!("|{}| sc: {}  lt: {}  fl: {}", name, success, late, failure)
+    });
+    println!("in {} sec.", actual_tpcc_time.as_secs());
+    println!("<Constraint Check> (all must be [OK])");
+    println!("[transaction percentage]");
+
+    let mut j = 0.0;
+    for i in 0..5 {
+        j += (success[i] + late[i]) as f64;
+    }
+    // Payment
+    let f = ((success[1] + late[1]) as f64 / j) * 100.0;
+    print!("    Payment: {:.1}% (>=43.0%)", f);
+    if f >= 43.0 {
+        println!("  [Ok]");
+    } else {
+        println!("  [NG]");
+    }
+    // Order-Status
+    let f = ((success[2] + late[2]) as f64 / j) * 100.0;
+    print!("    Order-Status: {:.1}% (>=4.0%)", f);
+    if f >= 4.0 {
+        println!("  [Ok]");
+    } else {
+        println!("  [NG]");
+    }
+    // Delivery
+    let f = ((success[3] + late[3]) as f64 / j) * 100.0;
+    print!("    Order-Status: {:.1}% (>=4.0%)", f);
+    if f >= 4.0 {
+        println!("  [Ok]");
+    } else {
+        println!("  [NG]");
+    }
+    // Stock-Level
+    let f = ((success[4] + late[4]) as f64 / j) * 100.0;
+    print!("    Order-Status: {:.1}% (>=4.0%)", f);
+    if f >= 4.0 {
+        println!("  [Ok]");
+    } else {
+        println!("  [NG]");
+    }
+    println!("[response time (at least 90%% passed)]");
+    print_transaction(&success, &late, &failure, |name, success, late, _| {
+        let f = (success as f64 / (success + late) as f64) * 100.0;
+        print!("   {}: {:.1}", name, f);
+        if f >= 90.0 {
+            println!("  [OK]");
+        } else {
+            println!("  [NG]");
+        }
+    });
+    print_transaction(&success, &late, &failure, |name, success, late, _| {
+        println!("   {} Total: {}", name, success + late)
+    });
+    println!();
     rt_hist.hist_report();
+    println!("<TpmC>");
+    let tpmc = (success[0] + late[0]) as f64 / (actual_tpcc_time.as_secs_f64() / 60.0);
+    println!("{} Tpmc", tpmc);
 
     Ok(())
 }
 
+fn print_transaction<F: Fn(&str, usize, usize, usize)>(
+    success: &[usize],
+    late: &[usize],
+    failure: &[usize],
+    fn_print: F,
+) {
+    for (i, name) in vec![
+        "New-Order",
+        "Payment",
+        "Order-Status",
+        "Delivery",
+        "Stock-Level",
+    ]
+    .into_iter()
+    .enumerate()
+    {
+        fn_print(name, success[i], late[i], failure[i]);
+    }
+}
+
 fn other_ware(rng: &mut ThreadRng, home_ware: usize, num_ware: usize) -> usize {
     if num_ware == 1 {
         return home_ware;
diff --git a/tpcc/src/utils.rs b/tpcc/src/utils.rs
@@ -0,0 +1,68 @@
+use rand::rngs::ThreadRng;
+use rand::Rng;
+
+pub(crate) struct SeqGen {
+    no: usize,
+    py: usize,
+    os: usize,
+    dl: usize,
+    sl: usize,
+    total: usize,
+    seq: Vec<usize>,
+    next_num: usize,
+    rng: ThreadRng,
+}
+
+impl SeqGen {
+    pub(crate) fn new(n: usize, p: usize, o: usize, d: usize, s: usize) -> Self {
+        let total = n + p + o + d + s;
+
+        Self {
+            no: n,
+            py: p,
+            os: o,
+            dl: d,
+            sl: s,
+            total,
+            seq: vec![0; total],
+            next_num: 0,
+            rng: Default::default(),
+        }
+    }
+
+    pub(crate) fn get(&mut self) -> usize {
+        if self.next_num >= self.total {
+            self.shuffle();
+            self.next_num = 0;
+        }
+        let pos = self.next_num;
+        self.next_num += 1;
+        self.seq[pos]
+    }
+
+    pub(crate) fn shuffle(&mut self) {
+        let mut pos = 0;
+
+        let mut fn_init = |len, tx| {
+            for i in 0..len {
+                self.seq[pos + i] = tx;
+            }
+            pos += len;
+        };
+        fn_init(self.no, 0);
+        fn_init(self.py, 1);
+        fn_init(self.os, 2);
+        fn_init(self.dl, 3);
+        fn_init(self.sl, 4);
+
+        let mut i = 0;
+        for j in (0..self.total).rev() {
+            let rmd = self.rng.gen::<usize>() % (j + 1);
+            let tmp = self.seq[rmd + i];
+            self.seq[rmd + i] = self.seq[i];
+            self.seq[i] = tmp;
+
+            i += 1;
+        }
+    }
+}
