From 9245ede8f8372685b7eab2aa333dc18dc5b21f77 Mon Sep 17 00:00:00 2001 From: brettkolodny Date: Thu, 15 Apr 2021 14:21:52 -0400 Subject: [PATCH] Orml bencher (#452) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Use MultiLocation as xtokens transfer dest type. (#396) * Use MultiLocation as xtokens transfer dest type. * Make clippy happy. * Use xcm-handler to execute XCM locally (#401) * Use cumulus xcm-handler to execute XCM locally. * Add docstring for xtokens config. * Replace XcmError::Undefined usage. (#402) * Replace XcmError::Undefined usage. * make clippy happy * Bump and unify serde versions. (#403) * xtokens and xcm-support documentations (#404) * Add xtokens module docstring. * Add xtokens readme. * Add documentations for xcm-support. * Add xtokens and xcm-support entries in main readme. * Add unit tests for xcm-support. (#405) * Added Minterest to the list of users. (#406) * update step guide link * Handle unknown assets in TransactAsset impl (#409) * Handle unknown assets in TransactAsset impl. * More documentations. * Clean code. * Renaming. * Should try to deposit known asset first. * Return error if no UnknownAsset impl. * Make clippy happy. * Fix description and repo link. (#410) * Unknown tokens pallet (#411) * Impl unknown tokens pallet. * Fix workspace pallet path. * Make clippy happy. * Clippy, be happy. * Unit tests. * Remove nonces from oracle pallet. (#413) * refactor rewards (#412) * Bump rococo v1 dependencies (#418) * Fix mocks. * Replace deprecated. * Update orml-unknown-tokens unit tests. (#421) * add build-script-utils from Substrate (#422) * Update README.md (#420) * Update README.md * Update README.md * Bump impl-trait-for-tuples to 0.2.1 (#424) * update Cargo.toml (#429) * bencher init commit * split into files * clean deps * add docs * use frame_benchmarking apis * add macro bencher_use to export stuff for bench_runner * * generate weights file * refactor * improvements * CLI Options (#446) * Added command line options for output, headers, and templates * Fixed options to conform to standard cli * Added weight-gen cli * fixed dependencies * Replaced unwraps with expects (#449) * Orml bencher (#451) * add Handler (#431) * remove disable-tokens-by-owner (#434) * Cross-chain transfer rework (#432) * Reserve chain trait. * Rework cross-chain transfer. * Remove relay chain balance convert. * Add 'Parse' trait. * Change transfer_multiasset fn signature. * Add transfer dispatchable call. * Update doc. * Use xcm-simulator to mock network. * Send relay chain asset to sibling unit test. * Move location traits into orml-traits. * Add MultiNativeAsset filter for is reserve check. * More unit tests. * Failing edge case unit tests. * Handle zero amount asset case. * Fix mocks. * Renaming. * Update currency adapter to work with new xtokens impl (#436) * Xcm support implementations rework. * Update xtokens mock. * Use CurrencyId convert. (#437) * Use CurrencyId convert. * Apply review suggestions. * Update xtokens docs. (#438) * Update xtokens docs. * Fix typo. * Update imbalances impl. * Don't deposit failure event in orml-unknown-tokens. (#440) * Don't deposit failure event in orml-unknown-tokens. * Patch substrate/polkadot/cumulus. * Fix patch. * Update README.md (#441) Include Zeitgeist into "Projects using ORML" section * Add PoV size in benchmarking. (#442) * Bump cumulus ref in cargo patch. (#443) * fix missing features (#444) * fix missing features * test with benchmarks * update auction weight (#445) * Bump dependencies. (#448) * Replaced unwraps with expects Co-authored-by: Xiliang Chen Co-authored-by: Shaun Wang Co-authored-by: Harald Heckmann Co-authored-by: wangjj9219 <183318287@qq.com> Co-authored-by: Shaun Wang Co-authored-by: dzianis.ramanouski Co-authored-by: Bette <42193328+bette7@users.noreply.github.com> Co-authored-by: wangjj9219 <183318287@qq.com> Co-authored-by: Xiliang Chen Co-authored-by: transxask <68648225+transxask@users.noreply.github.com> Co-authored-by: Aaro Perämaa Co-authored-by: Ermal Kaleci Co-authored-by: Harald Heckmann --- Cargo.dev.toml | 4 +- bencher/Cargo.toml | 40 ++++++++ bencher/src/bench_runner.rs | 39 ++++++++ bencher/src/handler.rs | 54 +++++++++++ bencher/src/lib.rs | 28 ++++++ bencher/src/macros.rs | 160 +++++++++++++++++++++++++++++++ bencher/src/template.hbs | 23 +++++ weight-gen/Cargo.toml | 20 ++++ weight-gen/src/main.rs | 181 ++++++++++++++++++++++++++++++++++++ weight-gen/src/template.hbs | 23 +++++ 10 files changed, 571 insertions(+), 1 deletion(-) create mode 100644 bencher/Cargo.toml create mode 100644 bencher/src/bench_runner.rs create mode 100644 bencher/src/handler.rs create mode 100644 bencher/src/lib.rs create mode 100644 bencher/src/macros.rs create mode 100644 bencher/src/template.hbs create mode 100644 weight-gen/Cargo.toml create mode 100644 weight-gen/src/main.rs create mode 100644 weight-gen/src/template.hbs diff --git a/Cargo.dev.toml b/Cargo.dev.toml index 4b8025012..9ace6b2bd 100644 --- a/Cargo.dev.toml +++ b/Cargo.dev.toml @@ -16,7 +16,9 @@ members = [ "nft", "xtokens", "xcm-support", - "weight-meter", + "unknown-tokens", + "build-script-utils", + "weight-meter", ] resolver = "2" diff --git a/bencher/Cargo.toml b/bencher/Cargo.toml new file mode 100644 index 000000000..0942cc850 --- /dev/null +++ b/bencher/Cargo.toml @@ -0,0 +1,40 @@ +[package] +name = "orml-bencher" +description = "Provide macro to benchmark pallets." +repository = "https://github.com/open-web3-stack/open-runtime-module-library/tree/master/bencher" +license = "Apache-2.0" +version = "0.1.0" +authors = ["Laminar Developers "] +edition = "2018" + +[dependencies] +linregress = { version = "0.4.0", optional = true } +handlebars = {version = "3.5.2", optional = true } +serde = { features = ['derive'], optional = true, version = "1.0.119" } +serde_json = "1.0" +codec = { package = "parity-scale-codec", version = "2.0.0", features = ["derive"], default-features = false } +sp-core = { version = "3.0.0", default-features = false } +sp-std = { version = "3.0.0", default-features = false } +sp-io = { version = "3.0.0", default-features = false } +sp-runtime-interface = { version = "3.0.0", default-features = false } +sp-state-machine = { version = "0.9.0", default-features = false, optional = true } +sc-executor = { version = "0.9.0", default-features = false, optional = true } +sc-client-db = { version = "0.9.0", default-features = false, features = ["with-kvdb-rocksdb"], optional = true } +frame-benchmarking = { version = "3.1.0", default-features = false } + +[features] +default = ["std"] +std = [ + "linregress", + "handlebars", + "serde/std", + "codec/std", + "sp-core/std", + "sp-std/std", + "sp-io/std", + "sp-runtime-interface/std", + "sp-state-machine/std", + "sc-executor/std", + "sc-client-db", + "frame-benchmarking/std", +] diff --git a/bencher/src/bench_runner.rs b/bencher/src/bench_runner.rs new file mode 100644 index 000000000..6f0d49075 --- /dev/null +++ b/bencher/src/bench_runner.rs @@ -0,0 +1,39 @@ +use frame_benchmarking::{ + benchmarking, + frame_support::sp_runtime::traits::{Block, NumberFor}, +}; +use sc_client_db::BenchmarkingState; +use sc_executor::{sp_wasm_interface::HostFunctions, WasmExecutionMethod, WasmExecutor}; +use sp_core::traits::{CallInWasm, MissingHostFunctions}; +use sp_io::SubstrateHostFunctions; +use sp_state_machine::{Ext, OverlayedChanges, StorageTransactionCache}; + +/// Run benches +pub fn run(wasm_code: Vec) -> Vec { + let mut overlay = OverlayedChanges::default(); + let mut cache = StorageTransactionCache::default(); + let state = BenchmarkingState::::new(Default::default(), Default::default()).unwrap(); + let mut ext = Ext::<_, NumberFor, _>::new(&mut overlay, &mut cache, &state, None, None); + + let mut host_functions = benchmarking::HostFunctions::host_functions(); + host_functions.append(&mut SubstrateHostFunctions::host_functions()); + + let executor = WasmExecutor::new( + WasmExecutionMethod::Compiled, + Default::default(), + host_functions, + 1, + None, + ); + + executor + .call_in_wasm( + &wasm_code[..], + None, + "run_benches", + &[], + &mut ext, + MissingHostFunctions::Disallow, + ) + .unwrap() +} diff --git a/bencher/src/handler.rs b/bencher/src/handler.rs new file mode 100644 index 000000000..8533d2db1 --- /dev/null +++ b/bencher/src/handler.rs @@ -0,0 +1,54 @@ +use crate::BenchResult; +use codec::Decode; +use linregress::{FormulaRegressionBuilder, RegressionDataBuilder}; +use serde::{Deserialize, Serialize}; +use std::io::Write; + +#[derive(Serialize, Deserialize, Default, Debug, Clone)] +struct BenchData { + pub name: String, + pub base_weight: u64, + pub base_reads: u32, + pub base_writes: u32, +} + +/// Handle bench results +pub fn handle(output: Vec) { + let results = as Decode>::decode(&mut &output[..]).unwrap(); + let data: Vec = results + .into_iter() + .map(|result| { + let name = String::from_utf8_lossy(&result.method).to_string(); + + eprintln!("{:#?}", result); + + let y: Vec = result.elapses.into_iter().map(|x| x as f64).collect(); + let x: Vec = (0..y.len()).into_iter().map(|x| x as f64).collect(); + let data = vec![("Y", y), ("X", x)]; + let data = RegressionDataBuilder::new().build_from(data).unwrap(); + let formula = "Y ~ X"; + + let model = FormulaRegressionBuilder::new() + .data(&data) + .formula(formula) + .fit() + .unwrap(); + + BenchData { + name, + base_weight: model.parameters.intercept_value as u64 * 1_000, + base_reads: result.reads, + base_writes: result.writes, + } + }) + .collect(); + + if let Ok(json) = serde_json::to_string(&data) { + let stdout = ::std::io::stdout(); + let mut handle = stdout.lock(); + + handle.write_all(&json.as_bytes()).unwrap(); + } else { + eprintln!("Could not write benchdata to JSON"); + } +} diff --git a/bencher/src/lib.rs b/bencher/src/lib.rs new file mode 100644 index 000000000..2ef25f776 --- /dev/null +++ b/bencher/src/lib.rs @@ -0,0 +1,28 @@ +#![cfg_attr(not(feature = "std"), no_std)] + +#[doc(hidden)] +pub extern crate frame_benchmarking; +#[doc(hidden)] +pub extern crate sp_core; +#[doc(hidden)] +pub extern crate sp_std; + +use codec::{Decode, Encode}; +use sp_std::prelude::Vec; + +#[derive(Encode, Decode, Default, Clone, PartialEq, Debug)] +pub struct BenchResult { + pub method: Vec, + pub elapses: Vec, + pub reads: u32, + pub repeat_reads: u32, + pub writes: u32, + pub repeat_writes: u32, +} + +mod macros; + +#[cfg(feature = "std")] +pub mod bench_runner; +#[cfg(feature = "std")] +pub mod handler; diff --git a/bencher/src/macros.rs b/bencher/src/macros.rs new file mode 100644 index 000000000..0175ef40c --- /dev/null +++ b/bencher/src/macros.rs @@ -0,0 +1,160 @@ +/// Run benches in WASM environment. +/// +/// Configure your module to build the mock runtime into wasm code. +/// Create a `build.rs` like you do with your runtime. +/// ```.ignore +/// use substrate_wasm_builder::WasmBuilder; +/// fn main() { +/// WasmBuilder::new() +/// .with_current_project() +/// .export_heap_base() +/// .import_memory() +/// .build() +/// } +/// ``` +/// +/// Update mock runtime to be build into wasm code. +/// ```.ignore +/// #![cfg_attr(not(feature = "std"), no_std)] +/// +/// #[cfg(feature = "std")] +/// include!(concat!(env!("OUT_DIR"), "/wasm_binary.rs")); +/// +/// #[cfg(feature = "std")] +/// pub fn wasm_binary_unwrap() -> &'static [u8] { WASM_BINARY.unwrap() } +/// .. +/// ``` +/// +/// Create a file `bench_runner.rs` with following code: +/// ```.ignore +/// orml_bencher::run_benches!(my_module::benches); +/// ``` +/// +/// Update Cargo.toml by adding: +/// ```toml +/// .. +/// [package] +/// name = "my-module" +/// .. +/// build = 'build.rs' +/// +/// [build-dependencies] +/// substrate-wasm-builder = '4.0.0' +/// +/// [[bench]] +/// name = 'benches' +/// harness = false +/// path = 'bench_runner.rs' +/// required-features = ['bench'] +/// +/// [features] +/// bench = [] +/// .. +/// ``` +/// +/// Run bench with features bench: `cargo bench --features=bench` +#[cfg(feature = "std")] +#[macro_export] +macro_rules! run_benches { + ($benches:path) => { + use $benches::{wasm_binary_unwrap, Block}; + pub fn main() { + let output = $crate::bench_runner::run::(wasm_binary_unwrap().to_vec()); + $crate::handler::handle(output); + } + }; +} + +/// Define benches +/// +/// Create a file `src/benches.rs`: +/// ```.ignore +/// #![cfg_attr(not(feature = "std"), no_std)] +/// #![allow(dead_code)] +/// +/// #[cfg(feature = "std")] // Re-export for bench_runner +/// pub use crate::mock::{Block, wasm_binary_unwrap}; +/// +/// use crate::mock::YourModule; +/// +/// fn foo(b: &mut Bencher) { +/// b.bench("foo", || { +/// YourModule::foo(); +/// }); +/// } +/// +/// fn bar(b: &mut Bencher) { +/// b.bench("bar", || { +/// YourModule::bar(); +/// }); +/// } +/// +/// orml_bencher::bench!(foo, bar); +/// ``` +/// Update `src/lib.rs`: +/// ```.ignore +/// #[cfg(any(feature = "bench", test))] +/// pub mod mock; /* mock runtime needs to be compiled into wasm */ +/// #[cfg(feature = "bench")] +/// pub mod benches; +/// ``` +#[macro_export] +macro_rules! bench { + ( + $($method:path),+ + ) => { + use $crate::BenchResult; + use $crate::sp_std::{cmp::max, prelude::Vec}; + use $crate::frame_benchmarking::{benchmarking, BenchmarkResults}; + + #[derive(Default, Clone, PartialEq, Debug)] + struct Bencher { + pub results: Vec, + } + + impl Bencher { + pub fn bench(&mut self, name: &str, block: fn() -> ()) { + // Warm up the DB + benchmarking::commit_db(); + benchmarking::wipe_db(); + + let mut result = BenchResult { + method: name.as_bytes().to_vec(), + ..Default::default() + }; + + for _ in 0..50 { + benchmarking::commit_db(); + benchmarking::reset_read_write_count(); + + let start_time = benchmarking::current_time(); + block(); + let end_time = benchmarking::current_time(); + let elasped = end_time - start_time; + result.elapses.push(elasped); + + benchmarking::commit_db(); + let (reads, repeat_reads, writes, repeat_writes) = benchmarking::read_write_count(); + + result.reads = max(result.reads, reads); + result.repeat_reads = max(result.repeat_reads, repeat_reads); + result.writes = max(result.writes, writes); + result.repeat_writes = max(result.repeat_writes, repeat_writes); + + benchmarking::wipe_db(); + } + self.results.push(result); + } + } + + $crate::sp_core::wasm_export_functions! { + fn run_benches() -> Vec { + let mut bencher = Bencher::default(); + $( + $method(&mut bencher); + )+ + bencher.results + } + } + } +} diff --git a/bencher/src/template.hbs b/bencher/src/template.hbs new file mode 100644 index 000000000..470b00c25 --- /dev/null +++ b/bencher/src/template.hbs @@ -0,0 +1,23 @@ +{{header}} + +#![allow(unused_parens)] +#![allow(unused_imports)] +#![allow(dead_code)] + +use frame_support::{traits::Get, weights::Weight}; +use sp_std::marker::PhantomData; + +pub struct ModuleWeights(PhantomData); +impl ModuleWeights { + {{~#each benchmarks as |benchmark|}} + pub fn {{benchmark.name~}} () -> Weight { + ({{underscore benchmark.base_weight}} as Weight) + {{~#if (ne benchmark.base_reads "0")}} + .saturating_add(T::DbWeight::get().reads({{benchmark.base_reads}} as Weight)) + {{~/if}} + {{~#if (ne benchmark.base_writes "0")}} + .saturating_add(T::DbWeight::get().writes({{benchmark.base_writes}} as Weight)) + {{~/if}} + } + {{~/each}} +} diff --git a/weight-gen/Cargo.toml b/weight-gen/Cargo.toml new file mode 100644 index 000000000..81f68bd1a --- /dev/null +++ b/weight-gen/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "weight-gen" +description = "CLI for generating weight from bencher output" +license = "Apache-2.0" +version = "0.1.0" +authors = ["Laminar Developers "] +edition = "2018" + +[dependencies] +serde = { features = ['derive'], optional = true, version = "1.0.119" } +serde_json = "1.0" +clap = "3.0.0-beta.2" +handlebars = {version = "3.5.2", optional = true } + +[features] +default = ["std"] +std = [ + "handlebars", + "serde/std", +] \ No newline at end of file diff --git a/weight-gen/src/main.rs b/weight-gen/src/main.rs new file mode 100644 index 000000000..3cdf1052b --- /dev/null +++ b/weight-gen/src/main.rs @@ -0,0 +1,181 @@ +use clap::{AppSettings, Clap}; +use serde::{Deserialize, Serialize}; +use std::io::Read; + +#[derive(Clap)] +#[clap(version = "01.0", author = "Laminar Developers ")] +#[clap(setting = AppSettings::ColoredHelp)] +struct Opts { + input: Option, + #[clap(short, long)] + template: Option, + #[clap(short, long)] + header: Option, + #[clap(short, long)] + out: Option, +} + +#[cfg(feature = "std")] +#[derive(Serialize, Deserialize, Default, Debug, Clone)] +pub struct BenchData { + pub name: String, + pub base_weight: u64, + pub base_reads: u32, + pub base_writes: u32, +} + +#[derive(Serialize, Default, Debug, Clone)] +struct TemplateData { + pub header: String, + pub benchmarks: Vec, +} + +// A Handlebars helper to add an underscore after every 3rd character, +// i.e. a separator for large numbers. +#[derive(Clone, Copy)] +struct UnderscoreHelper; +impl handlebars::HelperDef for UnderscoreHelper { + fn call<'reg: 'rc, 'rc>( + &self, + h: &handlebars::Helper, + _: &handlebars::Handlebars, + _: &handlebars::Context, + _rc: &mut handlebars::RenderContext, + out: &mut dyn handlebars::Output, + ) -> handlebars::HelperResult { + use handlebars::JsonRender; + let param = h.param(0).expect("Unable to retrieve param from handlebars helper"); + let underscore_param = underscore(param.value().render()); + out.write(&underscore_param)?; + Ok(()) + } +} + +// Add an underscore after every 3rd character, i.e. a separator for large +// numbers. +fn underscore(i: Number) -> String +where + Number: std::string::ToString, +{ + let mut s = String::new(); + let i_str = i.to_string(); + let a = i_str.chars().rev().enumerate(); + for (idx, val) in a { + if idx != 0 && idx % 3 == 0 { + s.insert(0, '_'); + } + s.insert(0, val); + } + s +} + +// A helper to join a string of vectors. +#[derive(Clone, Copy)] +struct JoinHelper; +impl handlebars::HelperDef for JoinHelper { + fn call<'reg: 'rc, 'rc>( + &self, + h: &handlebars::Helper, + _: &handlebars::Handlebars, + _: &handlebars::Context, + _rc: &mut handlebars::RenderContext, + out: &mut dyn handlebars::Output, + ) -> handlebars::HelperResult { + use handlebars::JsonRender; + let param = h.param(0).expect("Unable to retrieve param from handlebars helper"); + let value = param.value(); + let joined = if value.is_array() { + value + .as_array() + .unwrap() + .iter() + .map(|v| v.render()) + .collect::>() + .join(" ") + } else { + value.render() + }; + out.write(&joined)?; + Ok(()) + } +} + +fn parse_stdio() -> Option> { + let mut buffer = String::new(); + let stdin = std::io::stdin(); + let mut handle = stdin.lock(); + + handle.read_to_string(&mut buffer).expect("Unable to read from stdin"); + + let lines: Vec<&str> = buffer.split("\n").collect(); + for line in lines { + let json = serde_json::from_str(line); + + if let Ok(data) = json { + return Some(data); + } + } + + None +} + +fn main() { + let opts: Opts = Opts::parse(); + + let benchmarks: Vec = { + if let Some(data) = opts.input { + serde_json::from_str(&data).expect("Could not parse JSON data") + } else { + parse_stdio().expect("Could not parse JSON data") + } + }; + + let mut handlebars = handlebars::Handlebars::new(); + handlebars.register_helper("underscore", Box::new(UnderscoreHelper)); + handlebars.register_helper("join", Box::new(JoinHelper)); + // Don't HTML escape any characters. + handlebars.register_escape_fn(|s| -> String { s.to_string() }); + + // Use empty header if a header path is not given. + let header = { + if let Some(path) = opts.header { + let header_string = ::std::fs::read_to_string(&path).expect(&format!("Header file not found at {}", &path)); + + header_string + } else { + String::from("") + } + }; + + let hbs_data = TemplateData { header, benchmarks }; + + const DEFAULT_TEMPLATE: &str = include_str!("./template.hbs"); + + // Use default template if template path is not given. + let template = { + if let Some(path) = opts.template { + let template_string = + ::std::fs::read_to_string(&path).expect(&format!("Template file not found at {}", &path)); + + template_string + } else { + String::from(DEFAULT_TEMPLATE) + } + }; + + // Write benchmark to file or print to terminal if output path is not given. + if let Some(path) = opts.out { + let mut output_file = + ::std::fs::File::create(&path).expect(&format!("Could not create output file at {}", &path)); + + handlebars + .render_template_to_write(&template, &hbs_data, &mut output_file) + .expect("Unable to render template"); + } else { + let template_string = handlebars + .render_template(&template, &hbs_data) + .expect("Unable to render template"); + + println!("{}", template_string); + } +} diff --git a/weight-gen/src/template.hbs b/weight-gen/src/template.hbs new file mode 100644 index 000000000..470b00c25 --- /dev/null +++ b/weight-gen/src/template.hbs @@ -0,0 +1,23 @@ +{{header}} + +#![allow(unused_parens)] +#![allow(unused_imports)] +#![allow(dead_code)] + +use frame_support::{traits::Get, weights::Weight}; +use sp_std::marker::PhantomData; + +pub struct ModuleWeights(PhantomData); +impl ModuleWeights { + {{~#each benchmarks as |benchmark|}} + pub fn {{benchmark.name~}} () -> Weight { + ({{underscore benchmark.base_weight}} as Weight) + {{~#if (ne benchmark.base_reads "0")}} + .saturating_add(T::DbWeight::get().reads({{benchmark.base_reads}} as Weight)) + {{~/if}} + {{~#if (ne benchmark.base_writes "0")}} + .saturating_add(T::DbWeight::get().writes({{benchmark.base_writes}} as Weight)) + {{~/if}} + } + {{~/each}} +}