Orml bencher

Cargo.dev.toml

@@ -20,7 +20,7 @@ members = [
 	"xcm-support",
 	"unknown-tokens",
 	"build-script-utils",
-	"weight-meter",
+  "weight-meter",
 ]
 resolver = "2"
 

bencher/Cargo.toml

@@ -0,0 +1,44 @@
+[package]
+name = "orml-bencher"
+description = "Provide macro to benchmark pallets."
+repository = "https://github.com/open-web3-stack/open-runtime-module-library/tree/master/bencher"
+license = "Apache-2.0"
+version = "0.1.0"
+authors = ["Laminar Developers <[email protected]>"]
+edition = "2018"
+
+[[bin]]
+path = "src/bin.rs"
+name = "codegen"
+
+[dependencies]
+linregress = { version = "0.4.0", optional = true }
+handlebars = {version = "3.5.2", optional = true }
+serde = { features = ['derive'], optional = true, version = "1.0.119" }
+serde_json = "1.0"
+codec = { package = "parity-scale-codec", version = "2.0.0", features = ["derive"], default-features = false }
+sp-core = { version = "3.0.0", default-features = false }
+sp-std = { version = "3.0.0", default-features = false }
+sp-io = { version = "3.0.0", default-features = false }
+sp-runtime-interface = { version = "3.0.0", default-features = false }
+sp-state-machine = { version = "0.9.0", default-features = false, optional = true }
+sc-executor = { version = "0.9.0", default-features = false, optional = true }
+sc-client-db = { version = "0.9.0", default-features = false, features = ["with-kvdb-rocksdb"], optional = true }
+frame-benchmarking = { version = "3.1.0", default-features = false }
+
+[features]
+default = ["std"]
+std = [
+    "linregress",
+    "handlebars",
+    "serde/std",
+    "codec/std",
+    "sp-core/std",
+    "sp-std/std",
+    "sp-io/std",
+    "sp-runtime-interface/std",
+    "sp-state-machine/std",
+    "sc-executor/std",
+    "sc-client-db",
+    "frame-benchmarking/std",
+]

bencher/src/bench_runner.rs

@@ -0,0 +1,39 @@
+use frame_benchmarking::{
+	benchmarking,
+	frame_support::sp_runtime::traits::{Block, NumberFor},
+};
+use sc_client_db::BenchmarkingState;
+use sc_executor::{sp_wasm_interface::HostFunctions, WasmExecutionMethod, WasmExecutor};
+use sp_core::traits::{CallInWasm, MissingHostFunctions};
+use sp_io::SubstrateHostFunctions;
+use sp_state_machine::{Ext, OverlayedChanges, StorageTransactionCache};
+
+/// Run benches
+pub fn run<B: Block>(wasm_code: Vec<u8>) -> Vec<u8> {
+	let mut overlay = OverlayedChanges::default();
+	let mut cache = StorageTransactionCache::default();
+	let state = BenchmarkingState::<B>::new(Default::default(), Default::default()).unwrap();
+	let mut ext = Ext::<_, NumberFor<B>, _>::new(&mut overlay, &mut cache, &state, None, None);
+
+	let mut host_functions = benchmarking::HostFunctions::host_functions();
+	host_functions.append(&mut SubstrateHostFunctions::host_functions());
+
+	let executor = WasmExecutor::new(
+		WasmExecutionMethod::Compiled,
+		Default::default(),
+		host_functions,
+		1,
+		None,
+	);
+
+	executor
+		.call_in_wasm(
+			&wasm_code[..],
+			None,
+			"run_benches",
+			&[],
+			&mut ext,
+			MissingHostFunctions::Disallow,
+		)
+		.unwrap()
+}

bencher/src/handler.rs

@@ -0,0 +1,54 @@
+use crate::BenchResult;
+use serde::{Serialize, Deserialize};
+use codec::Decode;
+use linregress::{FormulaRegressionBuilder, RegressionDataBuilder};
+use std::io::Write;
+
+#[derive(Serialize, Deserialize, Default, Debug, Clone)]
+struct BenchData {
+	pub name: String,
+	pub base_weight: u64,
+	pub base_reads: u32,
+	pub base_writes: u32,
+}
+
+/// Handle bench results
+pub fn handle(output: Vec<u8>) {
+	let results = <Vec<BenchResult> as Decode>::decode(&mut &output[..]).unwrap();
+	let data: Vec<BenchData> = results
+		.into_iter()
+		.map(|result| {
+			let name = String::from_utf8_lossy(&result.method).to_string();
+
+			eprintln!("{:#?}", result);
+
+			let y: Vec<f64> = result.elapses.into_iter().map(|x| x as f64).collect();
+			let x: Vec<f64> = (0..y.len()).into_iter().map(|x| x as f64).collect();
+			let data = vec![("Y", y), ("X", x)];
+			let data = RegressionDataBuilder::new().build_from(data).unwrap();
+			let formula = "Y ~ X";
+
+			let model = FormulaRegressionBuilder::new()
+				.data(&data)
+				.formula(formula)
+				.fit()
+				.unwrap();
+
+			BenchData {
+				name,
+				base_weight: model.parameters.intercept_value as u64 * 1_000,
+				base_reads: result.reads,
+				base_writes: result.writes,
+			}
+		})
+		.collect();
+
+	if let Ok(json) = serde_json::to_string(&data) {
+		let stdout = ::std::io::stdout();
+		let mut handle = stdout.lock();
+
+		handle.write_all(&json.as_bytes()).unwrap();
+	} else {
+		eprintln!("Could not write benchdata to JSON");
+	}
+}

bencher/src/lib.rs

@@ -0,0 +1,28 @@
+#![cfg_attr(not(feature = "std"), no_std)]
+
+#[doc(hidden)]
+pub extern crate frame_benchmarking;
+#[doc(hidden)]
+pub extern crate sp_core;
+#[doc(hidden)]
+pub extern crate sp_std;
+
+use codec::{Decode, Encode};
+use sp_std::prelude::Vec;
+
+#[derive(Encode, Decode, Default, Clone, PartialEq, Debug)]
+pub struct BenchResult {
+	pub method: Vec<u8>,
+	pub elapses: Vec<u128>,
+	pub reads: u32,
+	pub repeat_reads: u32,
+	pub writes: u32,
+	pub repeat_writes: u32,
+}
+
+mod macros;
+
+#[cfg(feature = "std")]
+pub mod bench_runner;
+#[cfg(feature = "std")]
+pub mod handler;

bencher/src/macros.rs

@@ -0,0 +1,160 @@
+/// Run benches in WASM environment.
+///
+/// Configure your module to build the mock runtime into wasm code.
+/// Create a `build.rs` like you do with your runtime.
+/// ```.ignore
+/// use substrate_wasm_builder::WasmBuilder;
+/// fn main() {
+///     WasmBuilder::new()
+///         .with_current_project()
+///         .export_heap_base()
+///         .import_memory()
+///         .build()
+/// }
+/// ```
+///
+/// Update mock runtime to be build into wasm code.
+/// ```.ignore
+/// #![cfg_attr(not(feature = "std"), no_std)]
+///
+/// #[cfg(feature = "std")]
+/// include!(concat!(env!("OUT_DIR"), "/wasm_binary.rs"));
+///
+/// #[cfg(feature = "std")]
+/// pub fn wasm_binary_unwrap() -> &'static [u8] { WASM_BINARY.unwrap() }
+/// ..
+/// ```
+///
+/// Create a file `bench_runner.rs` with following code:
+///  ```.ignore
+/// orml_bencher::run_benches!(my_module::benches);
+/// ```
+/// 
+/// Update Cargo.toml by adding:
+/// ```toml
+/// ..
+/// [package]
+/// name = "my-module"
+/// ..
+/// build = 'build.rs'
+///
+/// [build-dependencies]
+/// substrate-wasm-builder = '4.0.0'
+///
+/// [[bench]]
+/// name = 'benches'
+/// harness = false
+/// path = 'bench_runner.rs'
+/// required-features = ['bench']
+///
+/// [features]
+/// bench = []
+/// ..
+/// ```
+/// 
+/// Run bench with features bench: `cargo bench --features=bench`
+#[cfg(feature = "std")]
+#[macro_export]
+macro_rules! run_benches {
+	($benches:path) => {
+		use $benches::{wasm_binary_unwrap, Block};
+		pub fn main() {
+			let output = $crate::bench_runner::run::<Block>(wasm_binary_unwrap().to_vec());
+			$crate::handler::handle(output);
+		}
+	};
+}
+
+/// Define benches
+///
+/// Create a file `src/benches.rs`:
+/// ```.ignore
+/// #![cfg_attr(not(feature = "std"), no_std)]
+/// #![allow(dead_code)]
+///
+/// #[cfg(feature = "std")] // Re-export for bench_runner
+/// pub use crate::mock::{Block, wasm_binary_unwrap};
+///
+/// use crate::mock::YourModule;
+///
+/// fn foo(b: &mut Bencher) {
+///     b.bench("foo", || {
+///         YourModule::foo();
+///     });
+/// }
+///
+/// fn bar(b: &mut Bencher) {
+///     b.bench("bar", || {
+///         YourModule::bar();
+///     });
+/// }
+///
+/// orml_bencher::bench!(foo, bar);
+/// ```
+/// Update `src/lib.rs`:
+/// ```.ignore
+/// #[cfg(any(feature = "bench", test))]
+/// pub mod mock; /* mock runtime needs to be compiled into wasm */
+/// #[cfg(feature = "bench")]
+/// pub mod benches;
+/// ```
+#[macro_export]
+macro_rules! bench {
+    (
+        $($method:path),+
+    ) => {
+        use $crate::BenchResult;
+        use $crate::sp_std::{cmp::max, prelude::Vec};
+        use $crate::frame_benchmarking::{benchmarking, BenchmarkResults};
+
+        #[derive(Default, Clone, PartialEq, Debug)]
+        struct Bencher {
+            pub results: Vec<BenchResult>,
+        }
+
+        impl Bencher {
+            pub fn bench(&mut self, name: &str, block: fn() -> ()) {
+                // Warm up the DB
+                benchmarking::commit_db();
+                benchmarking::wipe_db();
+
+                let mut result = BenchResult {
+                    method: name.as_bytes().to_vec(),
+                    ..Default::default()
+                };
+
+                for _ in 0..50 {
+                    benchmarking::commit_db();
+                    benchmarking::reset_read_write_count();
+
+                    let start_time = benchmarking::current_time();
+                    block();
+                    let end_time = benchmarking::current_time();
+                    let elasped = end_time - start_time;
+                    result.elapses.push(elasped);
+
+                    benchmarking::commit_db();
+                    let (reads, repeat_reads, writes, repeat_writes) = benchmarking::read_write_count();
+
+                    result.reads = max(result.reads, reads);
+                    result.repeat_reads = max(result.repeat_reads, repeat_reads);
+                    result.writes = max(result.writes, writes);
+                    result.repeat_writes = max(result.repeat_writes, repeat_writes);
+
+                    benchmarking::wipe_db();
+                }
+                self.results.push(result);
+            }
+        }
+
+        $crate::sp_core::wasm_export_functions! {
+            fn run_benches() -> Vec<BenchResult> {
+                let mut bencher = Bencher::default();
+                $(
+                    $method(&mut bencher);
+                )+
+                bencher.results
+            }
+        }
+    }
+}

bencher/src/template.hbs

@@ -0,0 +1,23 @@
+{{header}}
+
+#![allow(unused_parens)]
+#![allow(unused_imports)]
+#![allow(dead_code)]
+
+use frame_support::{traits::Get, weights::Weight};
+use sp_std::marker::PhantomData;
+
+pub struct ModuleWeights<T>(PhantomData<T>);
+impl<T: frame_system::Config> ModuleWeights<T> {
+	{{~#each benchmarks as |benchmark|}}
+	pub fn {{benchmark.name~}} () -> Weight {
+		({{underscore benchmark.base_weight}} as Weight)
+			{{~#if (ne benchmark.base_reads "0")}}
+			.saturating_add(T::DbWeight::get().reads({{benchmark.base_reads}} as Weight))
+			{{~/if}}
+			{{~#if (ne benchmark.base_writes "0")}}
+			.saturating_add(T::DbWeight::get().writes({{benchmark.base_writes}} as Weight))
+			{{~/if}}
+	}
+	{{~/each}}
+}

weight-gen/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "weight-gen"
+description = "CLI for generating weight from bencher output"
+license = "Apache-2.0"
+version = "0.1.0"
+authors = ["Laminar Developers <[email protected]>"]
+edition = "2018"
+
+[dependencies]
+serde = { features = ['derive'], optional = true, version = "1.0.119" }
+serde_json = "1.0"
+clap = "3.0.0-beta.2"
+handlebars = {version = "3.5.2", optional = true }
+
+[features]
+default = ["std"]
+std = [
+    "handlebars",
+    "serde/std",
+]