diff --git a/Cargo.lock b/Cargo.lock index 0dc8026343a..919fd5a801c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -111,7 +111,7 @@ dependencies = [ [[package]] name = "autocfg" -version = "0.1.2" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -172,7 +172,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "num-bigint 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "num-integer 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)", - "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.98 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -274,7 +274,7 @@ version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "num-integer 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)", - "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.98 (registry+https://github.com/rust-lang/crates.io-index)", "time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -542,7 +542,7 @@ dependencies = [ "diesel_derives 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "num-bigint 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "num-integer 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)", - "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", "pq-sys 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "r2d2 0.8.3 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.40 (registry+https://github.com/rust-lang/crates.io-index)", @@ -891,7 +891,7 @@ dependencies = [ "itertools 0.7.11 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "num-bigint 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", "parity-wasm 0.31.3 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", "reqwest 0.9.18 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1031,7 +1031,7 @@ dependencies = [ "pwasm-utils 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", "semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", "tiny-keccak 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)", - "wasmi 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", + "wasmi 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1740,7 +1740,7 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "num-integer 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)", - "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.98 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1749,13 +1749,27 @@ name = "num-integer" version = "0.1.39" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "num-rational" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "autocfg 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "num-bigint 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "num-integer 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "num-traits" -version = "0.2.6" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "autocfg 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", +] [[package]] name = "num_cpus" @@ -2101,7 +2115,7 @@ name = "rand" version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "autocfg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "autocfg 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.59 (registry+https://github.com/rust-lang/crates.io-index)", "rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "rand_core 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2119,7 +2133,7 @@ name = "rand_chacha" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "autocfg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "autocfg 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2180,7 +2194,7 @@ name = "rand_pcg" version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "autocfg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "autocfg 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", "rand_core 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -3307,10 +3321,13 @@ dependencies = [ [[package]] name = "wasmi" -version = "0.4.5" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ + "libc 0.2.59 (registry+https://github.com/rust-lang/crates.io-index)", "memory_units 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "num-rational 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", "parity-wasm 0.31.3 (registry+https://github.com/rust-lang/crates.io-index)", "wasmi-validation 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -3458,7 +3475,7 @@ dependencies = [ "checksum ascii 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a5fc969a8ce2c9c0c4b0429bb8431544f6658283c8326ba5ff8c762b75369335" "checksum assert_cli 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a29ab7c0ed62970beb0534d637a8688842506d0ff9157de83286dacd065c8149" "checksum atty 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "9a7d5b8723950951411ee34d271d99dddcc2035a16ab25310ea2c8cfd4369652" -"checksum autocfg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a6d640bee2da49f60a4068a7fae53acde8982514ab7bae8b8cea9e88cbcfd799" +"checksum autocfg 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "b671c8fb71b457dd4ae18c4ba1e59aa81793daacc361d82fcd410cef0d491875" "checksum backtrace 0.3.32 (registry+https://github.com/rust-lang/crates.io-index)" = "18b50f5258d1a9ad8396d2d345827875de4261b158124d4c819d9b351454fae5" "checksum backtrace-sys 0.1.28 (registry+https://github.com/rust-lang/crates.io-index)" = "797c830ac25ccc92a7f8a7b9862bde440715531514594a6154e3d4a54dd769b6" "checksum base-x 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d55aa264e822dbafa12db4d54767aff17c6ba55ea2d8559b3e17392c7d000e5d" @@ -3617,7 +3634,8 @@ dependencies = [ "checksum nodrop 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "2f9667ddcc6cc8a43afc9b7917599d7216aa09c463919ea32c59ed6cac8bc945" "checksum num-bigint 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "57450397855d951f1a41305e54851b1a7b8f5d2e349543a02a2effe25459f718" "checksum num-integer 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)" = "e83d528d2677f0518c570baf2b7abdcf0cd2d248860b68507bdcb3e91d4c0cea" -"checksum num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "0b3a5d7cc97d6d30d8b9bc8fa19bf45349ffe46241e8816f50f62f6d6aaabee1" +"checksum num-rational 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f2885278d5fe2adc2f75ced642d52d879bffaceb5a2e0b1d4309ffdfb239b454" +"checksum num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "6ba9a427cfca2be13aa6f6403b0b7e7368fe982bfa16fccc450ce74c46cd9b32" "checksum num_cpus 1.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1a23f0ed30a54abaa0c7e83b1d2d87ada7c3c23078d1d87815af3e3b6385fbba" "checksum opaque-debug 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "93f5bb2e8e8dec81642920ccff6b61f1eb94fa3020c5a325c9851ff604152409" "checksum openssl 0.10.18 (registry+https://github.com/rust-lang/crates.io-index)" = "b90119d71b0a3596588da04bf7c2c42f2978cfa1217a94119d8ec9e963c7729c" @@ -3787,7 +3805,7 @@ dependencies = [ "checksum void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" "checksum walkdir 2.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "c7904a7e2bb3cdf0cf5e783f44204a85a37a93151738fa349f06680f59a98b45" "checksum want 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b6395efa4784b027708f7451087e647ec73cc74f5d9bc2e418404248d679a230" -"checksum wasmi 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)" = "aebbaef470840d157a5c47c8c49f024da7b1b80e90ff729ca982b2b80447e78b" +"checksum wasmi 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "48437c526d40a6a593c50c5367dac825b8d6a04411013e866eca66123fb56faa" "checksum wasmi-validation 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ab380192444b3e8522ae79c0a1976e42a82920916ccdfbce3def89f456ea33f3" "checksum web3 0.8.0 (git+https://github.com/graphprotocol/rust-web3?branch=graph-patches)" = "" "checksum websocket 0.21.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8c9faed2bff8af2ea6b9f8b917d3d00b467583f6781fe3def174a9e33c879703" diff --git a/runtime/wasm/Cargo.toml b/runtime/wasm/Cargo.toml index 02f99b965da..70533af583f 100644 --- a/runtime/wasm/Cargo.toml +++ b/runtime/wasm/Cargo.toml @@ -9,7 +9,7 @@ futures = "0.1.21" hex = "0.3.2" graph = { path = "../../graph" } tiny-keccak = "1.4.2" -wasmi = "0.4" +wasmi = "0.5" pwasm-utils = "0.6.1" bs58 = "0.2.2" graph-runtime-derive = { path = "../derive" } diff --git a/runtime/wasm/src/host.rs b/runtime/wasm/src/host.rs index 9dbb787ac7b..94d5189e0ad 100644 --- a/runtime/wasm/src/host.rs +++ b/runtime/wasm/src/host.rs @@ -4,8 +4,9 @@ use semver::{Version, VersionReq}; use tiny_keccak::keccak256; use std::collections::HashMap; +use std::str::FromStr; use std::thread; -use std::time::Instant; +use std::time::{Duration, Instant}; use super::MappingContext; use crate::module::{ValidModule, WasmiModule, WasmiModuleConfig}; @@ -19,6 +20,8 @@ use graph::prelude::{ use graph::util; use web3::types::{Log, Transaction}; +pub(crate) const TIMEOUT_ENV_VAR: &str = "GRAPH_MAPPING_HANDLER_TIMEOUT"; + pub struct RuntimeHostConfig { subgraph_id: SubgraphDeploymentId, mapping: Mapping, @@ -246,6 +249,10 @@ impl RuntimeHost { ethereum_adapter: ethereum_adapter.clone(), link_resolver: link_resolver.clone(), store: store.clone(), + handler_timeout: std::env::var(TIMEOUT_ENV_VAR) + .ok() + .and_then(|s| u64::from_str(&s).ok()) + .map(Duration::from_secs), }; let valid_module = ValidModule::new(&module_logger, wasmi_config, task_sender) .expect("Failed to validate module"); diff --git a/runtime/wasm/src/host_exports.rs b/runtime/wasm/src/host_exports.rs index fb2546b9e39..444e155d120 100644 --- a/runtime/wasm/src/host_exports.rs +++ b/runtime/wasm/src/host_exports.rs @@ -17,8 +17,6 @@ use web3::types::H160; use crate::module::WasmiModule; -pub(crate) const TIMEOUT_ENV_VAR: &str = "GRAPH_MAPPING_HANDLER_TIMEOUT"; - pub(crate) trait ExportError: fmt::Debug + fmt::Display + Send + Sync + 'static {} impl ExportError for E where E: fmt::Debug + fmt::Display + Send + Sync + 'static {} @@ -49,6 +47,7 @@ pub(crate) struct HostExports { link_resolver: Arc, store: Arc, task_sink: U, + handler_timeout: Option, } impl HostExports @@ -72,6 +71,7 @@ where link_resolver: Arc, store: Arc, task_sink: U, + handler_timeout: Option, ) -> Self { HostExports { subgraph_id, @@ -83,6 +83,7 @@ where link_resolver, store, task_sink, + handler_timeout, } } @@ -532,11 +533,7 @@ where &self, start_time: Instant, ) -> Result<(), HostExportError> { - let mapping_handler_timeout = std::env::var(TIMEOUT_ENV_VAR) - .ok() - .and_then(|s| u64::from_str(&s).ok()) - .map(Duration::from_secs); - if let Some(timeout) = mapping_handler_timeout { + if let Some(timeout) = self.handler_timeout { if start_time.elapsed() > timeout { return Err(HostExportError(format!("Mapping handler timed out"))); } diff --git a/runtime/wasm/src/lib.rs b/runtime/wasm/src/lib.rs index 88111f36985..58c3675e1b2 100644 --- a/runtime/wasm/src/lib.rs +++ b/runtime/wasm/src/lib.rs @@ -1,14 +1,3 @@ -extern crate bs58; -extern crate ethabi; -extern crate futures; -extern crate graph; -extern crate graph_runtime_derive; -extern crate hex; -extern crate pwasm_utils; -extern crate semver; -extern crate tiny_keccak; -extern crate wasmi; - mod asc_abi; mod host; mod module; diff --git a/runtime/wasm/src/module/mod.rs b/runtime/wasm/src/module/mod.rs index c1be8d59936..cb466832179 100644 --- a/runtime/wasm/src/module/mod.rs +++ b/runtime/wasm/src/module/mod.rs @@ -1,6 +1,7 @@ +use std::convert::TryFrom; use std::fmt; use std::ops::Deref; -use std::time::Instant; +use std::time::{Duration, Instant}; use semver::Version; use wasmi::{ @@ -86,6 +87,7 @@ pub struct WasmiModuleConfig { pub ethereum_adapter: Arc, pub link_resolver: Arc, pub store: Arc, + pub handler_timeout: Option, } /// A pre-processed and valid WASM module, ready to be started as a WasmiModule. @@ -161,6 +163,7 @@ where config.link_resolver.clone(), config.store.clone(), task_sink, + config.handler_timeout, ); Ok(ValidModule { @@ -187,6 +190,12 @@ pub(crate) struct WasmiModule { // True if `run_start` has not yet been called on the module. // This is used to prevent mutating store state in start. running_start: bool, + + // First free byte in the heap. + heap_start_ptr: u32, + + // Number of free bytes starting from `heap_start_ptr`. + heap_free_size: u32, } impl WasmiModule @@ -235,6 +244,10 @@ where valid_module: valid_module.clone(), start_time: Instant::now(), running_start: true, + + // `heap_start_ptr` will be set on the first call to `raw_new`. + heap_free_size: 0, + heap_start_ptr: 0, }; this.module = module @@ -407,22 +420,37 @@ where + 'static, { fn raw_new(&mut self, bytes: &[u8]) -> Result { - let address = self - .module - .clone() - .invoke_export( - "memory.allocate", - &[RuntimeValue::I32(bytes.len() as i32)], - self, - ) - .expect("Failed to invoke memory allocation function") - .expect("Function did not return a value") - .try_into::() - .expect("Function did not return u32"); + // We request large chunks from the AssemblyScript allocator and manage them ourselves. + // This assumes the arena allocator is being used in AS. + + static MIN_HEAP_SIZE_INCREMENT: u32 = 10_000; + + let size = u32::try_from(bytes.len()).unwrap(); + if size > self.heap_free_size { + let need = size - self.heap_free_size; + let allocate = need.max(MIN_HEAP_SIZE_INCREMENT); + let allocated_ptr = self + .module + .clone() + .invoke_export("memory.allocate", &[RuntimeValue::from(allocate)], self) + .expect("Failed to invoke memory allocation function") + .expect("Function did not return a value") + .try_into::() + .expect("Function did not return u32"); + self.heap_free_size += allocate; + + // On the first call, initialze `self.heap_start_ptr`. + if self.heap_start_ptr == 0 { + self.heap_start_ptr = allocated_ptr; + } + }; - self.memory.set(address, bytes)?; + let ptr = self.heap_start_ptr; + self.memory.set(ptr, bytes)?; + self.heap_start_ptr += size; + self.heap_free_size -= size; - Ok(address) + Ok(ptr) } fn get(&self, offset: u32, size: u32) -> Result, Error> { diff --git a/runtime/wasm/src/module/test.rs b/runtime/wasm/src/module/test.rs index 92c89abd5ca..1cc5321ca7b 100644 --- a/runtime/wasm/src/module/test.rs +++ b/runtime/wasm/src/module/test.rs @@ -172,6 +172,10 @@ fn test_valid_module( ethereum_adapter: mock_ethereum_adapter, link_resolver: Arc::new(ipfs_api::IpfsClient::default().into()), store: Arc::new(FakeStore), + handler_timeout: std::env::var(crate::host::TIMEOUT_ENV_VAR) + .ok() + .and_then(|s| u64::from_str(&s).ok()) + .map(Duration::from_secs), }, task_sender, ) diff --git a/runtime/wasm/src/module/test/abi.rs b/runtime/wasm/src/module/test/abi.rs index 2daf0d5fe2f..426cfa31080 100644 --- a/runtime/wasm/src/module/test/abi.rs +++ b/runtime/wasm/src/module/test/abi.rs @@ -3,7 +3,7 @@ use super::*; #[test] fn unbounded_loop() { // Set handler timeout to 3 seconds. - env::set_var(host_exports::TIMEOUT_ENV_VAR, "3"); + env::set_var(crate::host::TIMEOUT_ENV_VAR, "3"); let valid_module = test_valid_module(mock_data_source("wasm_test/non_terminating.wasm")); let mut module = WasmiModule::from_valid_module_with_ctx(valid_module, mock_context()).unwrap(); module.start_time = Instant::now();