From d9c541e9deff8e108866bd4263fc1997472c517d Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Mon, 3 Jul 2017 09:35:31 +0200 Subject: [PATCH 01/36] Merge #10704: [tests] nits in dbcrash.py 27c63dc [tests] nits in dbcrash.py (John Newbery) Tree-SHA512: 2a75feeb65e6147e3337200cde982248bea8977a9585d5ee284d62bbc25f6d7c368754da0083aec37338c8f66cf698ee25bbd9e192df14a9fb976b8f75afa986 --- test/functional/test_framework/util.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index f6c326ac8fd7..55b8b481e658 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -601,7 +601,10 @@ def satoshi_round(amount): # Helper to create at least "count" utxos # Pass in a fee that is sufficient for relay and mining new transactions. def create_confirmed_utxos(fee, node, count): - node.generate(int(0.5*count)+101) + to_generate = int(0.5*count)+101 + while to_generate > 0: + node.generate(min(25, to_generate)) + to_generate -= 25 utxos = node.listunspent() iterations = count - len(utxos) addr1 = node.getnewaddress() From fc94fac752e9605c4f96757802d736dcb83dcd45 Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Sun, 16 Jul 2017 23:41:24 +0200 Subject: [PATCH 02/36] =?UTF-8?q?Merge=20#10842:=20Fix=20incorrect=20Doxyg?= =?UTF-8?q?en=20tag=20(@ince=20=E2=86=92=20@since).=20Doxygen=20parameter?= =?UTF-8?q?=20name=20matching.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 2c2e90d Fix incorrect Doxygen tag (@ince → @since). Make Doxygen parameter names match actual parameter names. (practicalswift) Pull request description: Doxygen fixes: * Fix incorrect Doxygen tag (`@ince` → `@since`). * Make Doxygen parameter names match actual parameter names. Tree-SHA512: cb1d37a7d15a90d24affaf5c2bcf462663f0b0c13868bb25401e5d8cb303fcb41e53fdeee8012b2271a509112ef98fc0c070b4194f42cd55361f7e6bc7996342 --- src/policy/fees.cpp | 2 +- src/protocol.h | 2 +- src/util.h | 6 +++--- src/wallet/rpcwallet.h | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/policy/fees.cpp b/src/policy/fees.cpp index a0c37a70d6fa..39efa28f263b 100644 --- a/src/policy/fees.cpp +++ b/src/policy/fees.cpp @@ -64,7 +64,7 @@ class TxConfirmStats * Create new TxConfirmStats. This is called by BlockPolicyEstimator's * constructor with default values. * @param defaultBuckets contains the upper limits for the bucket boundaries - * @param maxConfirms max number of confirms to track + * @param maxPeriods max number of periods to track * @param decay how much to decay the historical moving average per block */ TxConfirmStats(const std::vector& defaultBuckets, unsigned int maxConfirms, double decay); diff --git a/src/protocol.h b/src/protocol.h index 9f363fe058a2..e7ee254f0a18 100644 --- a/src/protocol.h +++ b/src/protocol.h @@ -163,7 +163,7 @@ extern const char *PONG; /** * The notfound message is a reply to a getdata message which requested an * object the receiving node does not have available for relay. - * @ince protocol version 70001. + * @since protocol version 70001. * @see https://bitcoin.org/en/developer-reference#notfound */ extern const char *NOTFOUND; diff --git a/src/util.h b/src/util.h index 0804facb0906..6639300b4534 100644 --- a/src/util.h +++ b/src/util.h @@ -269,7 +269,7 @@ class ArgsManager * Return string argument or default value * * @param strArg Argument to get (e.g. "-foo") - * @param default (e.g. "1") + * @param strDefault (e.g. "1") * @return command-line argument or default value */ std::string GetArg(const std::string& strArg, const std::string& strDefault); @@ -278,7 +278,7 @@ class ArgsManager * Return integer argument or default value * * @param strArg Argument to get (e.g. "-foo") - * @param default (e.g. 1) + * @param nDefault (e.g. 1) * @return command-line argument (0 if invalid number) or default value */ int64_t GetArg(const std::string& strArg, int64_t nDefault); @@ -287,7 +287,7 @@ class ArgsManager * Return boolean argument or default value * * @param strArg Argument to get (e.g. "-foo") - * @param default (true or false) + * @param fDefault (true or false) * @return command-line argument or default value */ bool GetBoolArg(const std::string& strArg, bool fDefault); diff --git a/src/wallet/rpcwallet.h b/src/wallet/rpcwallet.h index bd5dad18ca59..31e2f6a69994 100644 --- a/src/wallet/rpcwallet.h +++ b/src/wallet/rpcwallet.h @@ -16,7 +16,7 @@ void RegisterWalletRPCCommands(CRPCTable &t); * @param[in] request JSONRPCRequest that wishes to access a wallet * @return NULL if no wallet should be used, or a pointer to the CWallet */ -CWallet *GetWalletForJSONRPCRequest(const JSONRPCRequest&); +CWallet *GetWalletForJSONRPCRequest(const JSONRPCRequest& request); std::string HelpRequiringPassphrase(CWallet *); void EnsureWalletIsUnlocked(CWallet *); From ea07a52136acf5154ada7cd98ff3a166c65ce5f9 Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Mon, 17 Jul 2017 04:23:20 +0200 Subject: [PATCH 03/36] Merge #10330: [wallet] fix zapwallettxes interaction with persistent mempool 4c3b538 [logs] fix zapwallettxes startup logs (John Newbery) e7a2181 [wallet] fix zapwallettxes interaction with persistent mempool (John Newbery) ff7365e [tests] fix flake8 warnings in zapwallettxes.py (John Newbery) Pull request description: zapwallettxes previously did not interact well with persistent mempool. zapwallettxes would cause wallet transactions to be zapped, but they would then be reloaded from the mempool on startup. This commit softsets persistmempool to false if zapwallettxes is enabled so transactions are actually zapped. This PR also fixes the zapwallettxes.py functional test, which did not properly test this feature. The test line: ```py assert_raises(JSONRPCException, self.nodes[0].gettransaction, [txid3]) #there must be a expection because the unconfirmed wallettx0 must be gone by now ``` is not actually testing the presence of the transaction since the RPC is being called incorrectly (with an array instead of a string). The `assert_raises()` passes since an assert is raised, but it's not the one the test writer had in mind! Fixes #9710 . Tree-SHA512: e3236efc7a2fd2b3bf1d9e2e8a7726d470c57f5d95cf41b7bde264edc8817bd36a6f3feff52f8de8db0ef64b7247c88b24e7ff7cefaa706cba86fe4e2135a508 --- src/wallet/wallet.cpp | 12 +++- test/functional/zapwallettxes.py | 100 +++++++++++++++---------------- 2 files changed, 57 insertions(+), 55 deletions(-) diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 58b8b69d0e5b..d9ded216ea01 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -5141,13 +5141,19 @@ bool CWallet::ParameterInteraction() } } - // -zapwallettx implies a rescan - if (gArgs.GetBoolArg("-zapwallettxes", false)) { + int zapwallettxes = gArgs.GetArg("-zapwallettxes", 0); + // -zapwallettxes implies dropping the mempool on startup + if (zapwallettxes != 0 && gArgs.SoftSetBoolArg("-persistmempool", false)) { + LogPrintf("%s: parameter interaction: -zapwallettxes=%s -> setting -persistmempool=0\n", __func__, zapwallettxes); + } + + // -zapwallettxes implies a rescan + if (zapwallettxes != 0) { if (is_multiwallet) { return InitError(strprintf("%s is only allowed with a single wallet file", "-zapwallettxes")); } if (gArgs.SoftSetBoolArg("-rescan", true)) { - LogPrintf("%s: parameter interaction: -zapwallettxes= -> setting -rescan=1\n", __func__); + LogPrintf("%s: parameter interaction: -zapwallettxes=%s -> setting -rescan=1\n", __func__, zapwallettxes); } } diff --git a/test/functional/zapwallettxes.py b/test/functional/zapwallettxes.py index 855fdbd0c31b..af867d7a52c7 100755 --- a/test/functional/zapwallettxes.py +++ b/test/functional/zapwallettxes.py @@ -4,77 +4,73 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the zapwallettxes functionality. -- start three bitcoind nodes -- create four transactions on node 0 - two are confirmed and two are - unconfirmed. -- restart node 1 and verify that both the confirmed and the unconfirmed +- start two bitcoind nodes +- create two transactions on node 0 - one is confirmed and one is unconfirmed. +- restart node 0 and verify that both the confirmed and the unconfirmed transactions are still available. -- restart node 0 and verify that the confirmed transactions are still - available, but that the unconfirmed transaction has been zapped. +- restart node 0 with zapwallettxes and persistmempool, and verify that both + the confirmed and the unconfirmed transactions are still available. +- restart node 0 with just zapwallettxed and verify that the confirmed + transactions are still available, but that the unconfirmed transaction has + been zapped. """ from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import * - +from test_framework.util import (assert_equal, + assert_raises_jsonrpc, + ) class ZapWalletTXesTest (BitcoinTestFramework): def __init__(self): super().__init__() self.setup_clean_chain = True - self.num_nodes = 3 - - def setup_network(self): - super().setup_network() - connect_nodes_bi(self.nodes,0,2) + self.num_nodes = 2 - def run_test (self): + def run_test(self): self.log.info("Mining blocks...") self.nodes[0].generate(1) self.sync_all() - self.nodes[1].generate(101) - self.sync_all() - - assert_equal(self.nodes[0].getbalance(), 500) - - txid0 = self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), 11) - txid1 = self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), 10) + self.nodes[1].generate(100) self.sync_all() + + # This transaction will be confirmed + txid1 = self.nodes[0].sendtoaddress(self.nodes[1].getnewaddress(), 10) + self.nodes[0].generate(1) self.sync_all() - - txid2 = self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), 11) - txid3 = self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), 10) - - tx0 = self.nodes[0].gettransaction(txid0) - assert_equal(tx0['txid'], txid0) #tx0 must be available (confirmed) - - tx1 = self.nodes[0].gettransaction(txid1) - assert_equal(tx1['txid'], txid1) #tx1 must be available (confirmed) - - tx2 = self.nodes[0].gettransaction(txid2) - assert_equal(tx2['txid'], txid2) #tx2 must be available (unconfirmed) - - tx3 = self.nodes[0].gettransaction(txid3) - assert_equal(tx3['txid'], txid3) #tx3 must be available (unconfirmed) - - #restart bitcoind + + # This transaction will not be confirmed + txid2 = self.nodes[0].sendtoaddress(self.nodes[1].getnewaddress(), 20) + + # Confirmed and unconfirmed transactions are now in the wallet. + assert_equal(self.nodes[0].gettransaction(txid1)['txid'], txid1) + assert_equal(self.nodes[0].gettransaction(txid2)['txid'], txid2) + + # Stop-start node0. Both confirmed and unconfirmed transactions remain in the wallet. + self.stop_node(0) + self.nodes[0] = self.start_node(0, self.options.tmpdir) + + assert_equal(self.nodes[0].gettransaction(txid1)['txid'], txid1) + assert_equal(self.nodes[0].gettransaction(txid2)['txid'], txid2) + + # Stop node0 and restart with zapwallettxes and persistmempool. The unconfirmed + # transaction is zapped from the wallet, but is re-added when the mempool is reloaded. self.stop_node(0) - self.nodes[0] = self.start_node(0,self.options.tmpdir) - - tx3 = self.nodes[0].gettransaction(txid3) - assert_equal(tx3['txid'], txid3) #tx must be available (unconfirmed) - + self.nodes[0] = self.start_node(0, self.options.tmpdir, ["-persistmempool=1", "-zapwallettxes=2"]) + + assert_equal(self.nodes[0].gettransaction(txid1)['txid'], txid1) + assert_equal(self.nodes[0].gettransaction(txid2)['txid'], txid2) + + # Stop node0 and restart with zapwallettxes, but not persistmempool. + # The unconfirmed transaction is zapped and is no longer in the wallet. self.stop_node(0) - - #restart bitcoind with zapwallettxes - self.nodes[0] = self.start_node(0,self.options.tmpdir, ["-zapwallettxes=1"]) - - assert_raises(JSONRPCException, self.nodes[0].gettransaction, [txid3]) - #there must be an exception because the unconfirmed wallettx0 must be gone by now + self.nodes[0] = self.start_node(0, self.options.tmpdir, ["-zapwallettxes=2"]) - tx0 = self.nodes[0].gettransaction(txid0) - assert_equal(tx0['txid'], txid0) #tx0 (confirmed) must still be available because it was confirmed + # tx1 is still be available because it was confirmed + assert_equal(self.nodes[0].gettransaction(txid1)['txid'], txid1) + # This will raise an exception because the unconfirmed transaction has been zapped + assert_raises_jsonrpc(-5, 'Invalid or non-wallet transaction id', self.nodes[0].gettransaction, txid2) if __name__ == '__main__': - ZapWalletTXesTest ().main () + ZapWalletTXesTest().main() From 9ad533089dc7df9be1bc984afa5c54362304aafb Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Mon, 17 Jul 2017 09:49:23 +0200 Subject: [PATCH 04/36] Merge #10803: Explicitly search for bdb5.3. 1cc251f Explicitly search for bdb5.3. (Patrick Strateman) Pull request description: Some systems do not symlink the major version to the minor version. Tree-SHA512: 09c030f08442cbe54928a6d20bec31aae2662facf60b859ff9febd84f0711f68d7f920b84fb015764585b305d48faf74c5fe9c3c6a713a0809b78ec066187dd9 --- build-aux/m4/bitcoin_find_bdb48.m4 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-aux/m4/bitcoin_find_bdb48.m4 b/build-aux/m4/bitcoin_find_bdb48.m4 index 980f1e8f19cd..b9bf7bf46e54 100644 --- a/build-aux/m4/bitcoin_find_bdb48.m4 +++ b/build-aux/m4/bitcoin_find_bdb48.m4 @@ -12,7 +12,7 @@ AC_DEFUN([BITCOIN_FIND_BDB48],[ bdbpath=X bdb48path=X bdbdirlist= - for _vn in 4.8 48 4 5 ''; do + for _vn in 4.8 48 4 5 5.3 ''; do for _pfx in b lib ''; do bdbdirlist="$bdbdirlist ${_pfx}db${_vn}" done From 083c49e1d27052877e8a0984802b9f3a7d74b143 Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Mon, 17 Jul 2017 13:10:50 +0200 Subject: [PATCH 05/36] Merge #10837: Fix resource leak on error in GetDevURandom a8ae0b2 Fix resource leak (Dag Robole) Pull request description: Fixes a potential file handle leak when size of entropy is invalid Tree-SHA512: 692d24daaf370bba1f842925b037275126f9494f54769650bcf5829c794a0fb8561a86f42347bdf088a484e4f107bce7fa14cd7bdbfb4ecfbeb51968953da3ae --- src/random.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/random.cpp b/src/random.cpp index 41146451c48e..4fbc8a4edce4 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -186,6 +186,7 @@ void GetDevURandom(unsigned char *ent32) do { ssize_t n = read(f, ent32 + have, NUM_OS_RANDOM_BYTES - have); if (n <= 0 || n + have > NUM_OS_RANDOM_BYTES) { + close(f); RandFailure(); } have += n; From 644274fdedcecca43e550588353bd83321eba0a6 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Mon, 17 Jul 2017 11:00:40 -0700 Subject: [PATCH 06/36] Merge #9980: Fix mem access violation merkleblock 8276e70de Adding assert to avoid a memory access violation inside of PartialMerkleTree::CalcHash() (Chris Stewart) Pull request description: Fixing a possible memory access violation in CPartialMerkleTree::CalcHash(). This can happen if we some how a merkle tree with zero txids. I don't think this can happen in practice as we only send merkle block messages on the p2p network as of now -- we cannot receive them. This was found with #8469, specifically using this [generator](https://github.com/Christewart/bitcoin/blob/rapidcheck/src/test/gen/merkleblock_gen.h#L52-L77) which will cause a memory access violation on [this test case](https://github.com/Christewart/bitcoin/blob/rapidcheck/src/test/merkleblock_properties.cpp#L48). Tree-SHA512: b95904ec45ea3f082c7722161d93ee06b24c706fbffa909a6e995ed14788aed2830f91b626da6f0347660c45874a0735dab61c9440b59c949c690af4165c83fb --- src/merkleblock.cpp | 3 +++ src/merkleblock.h | 2 ++ 2 files changed, 5 insertions(+) diff --git a/src/merkleblock.cpp b/src/merkleblock.cpp index b2b003f666ff..fd856fc14811 100644 --- a/src/merkleblock.cpp +++ b/src/merkleblock.cpp @@ -71,6 +71,9 @@ CMerkleBlock::CMerkleBlock(const CBlock& block, const std::set& txids) } uint256 CPartialMerkleTree::CalcHash(int height, unsigned int pos, const std::vector &vTxid) { + //we can never have zero txs in a merkle block, we always need the coinbase tx + //if we do not have this assert, we can hit a memory access violation when indexing into vTxid + assert(vTxid.size() != 0); if (height == 0) { // hash at height 0 is the txids themself return vTxid[pos]; diff --git a/src/merkleblock.h b/src/merkleblock.h index 68fac71a938d..8a974d4aaaea 100644 --- a/src/merkleblock.h +++ b/src/merkleblock.h @@ -121,6 +121,8 @@ class CPartialMerkleTree /** * Used to relay blocks as header + vector * to filtered nodes. + * + * NOTE: The class assumes that the given CBlock has *at least* 1 transaction. If the CBlock has 0 txs, it will hit an assertion. */ class CMerkleBlock { From cea9e1cf01e8c7785144dc88227597b276b8becb Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Tue, 18 Jul 2017 08:24:40 +0200 Subject: [PATCH 07/36] Merge #10855: random: only use getentropy on openbsd 077d01f random: only use getentropy on openbsd (Cory Fields) Pull request description: Follow-up from #10335. I can confirm that this fixes my issue when building against a new glibc + old linux headers for back-compat. Tree-SHA512: a0fcf26995fbd3636f970e729a172c6e1d7c0de371e703f0653cd9776600f438ec43acd2b1eb92f2678a011968da8fbbeef8a54599434851f4c6ffe78291c172 --- src/random.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/random.cpp b/src/random.cpp index 4fbc8a4edce4..c6a9906a1e65 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -227,10 +227,12 @@ void GetOSRand(unsigned char *ent32) RandFailure(); } } -#elif defined(HAVE_GETENTROPY) +#elif defined(HAVE_GETENTROPY) && defined(__OpenBSD__) /* On OpenBSD this can return up to 256 bytes of entropy, will return an * error if more are requested. * The call cannot return less than the requested number of bytes. + getentropy is explicitly limited to openbsd here, as a similar (but not + the same) function may exist on other platforms via glibc. */ if (getentropy(ent32, NUM_OS_RANDOM_BYTES) != 0) { RandFailure(); From dc97d48bbb5f7806a2f3c68d6b729e6b7e44a31f Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Sat, 15 Jul 2017 12:06:13 -0700 Subject: [PATCH 08/36] Merge #10760: Avoid dereference-of-casted-pointer 0aadc11fd Avoid dereference-of-casted-pointer (Pieter Wuille) Pull request description: And prefer a static_cast to the intended reference type. Tree-SHA512: e83b20023a4dca6029b46f7040a8a6fd54e1b42112ec0c87c3c3b567ed641de97a9e2335b57a2efb075491f641e5b977bc226a474276bea0c3c3c71d8d6ac54d --- src/core_memusage.h | 2 +- src/primitives/transaction.h | 4 ++-- src/rpc/blockchain.cpp | 2 +- src/script/interpreter.cpp | 2 +- src/script/script.h | 8 ++++++++ src/test/dbwrapper_tests.cpp | 6 +++--- src/txdb.cpp | 2 +- src/wallet/walletdb.cpp | 16 ++++++++-------- 8 files changed, 25 insertions(+), 17 deletions(-) diff --git a/src/core_memusage.h b/src/core_memusage.h index a6781556843c..9dca695fe4b0 100644 --- a/src/core_memusage.h +++ b/src/core_memusage.h @@ -10,7 +10,7 @@ #include "memusage.h" static inline size_t RecursiveDynamicUsage(const CScript& script) { - return memusage::DynamicUsage(*static_cast(&script)); + return memusage::DynamicUsage(script); } static inline size_t RecursiveDynamicUsage(const COutPoint& out) { diff --git a/src/primitives/transaction.h b/src/primitives/transaction.h index bdcd5d5dc304..e10cbde8a2b1 100644 --- a/src/primitives/transaction.h +++ b/src/primitives/transaction.h @@ -114,7 +114,7 @@ class CTxIn template inline void SerializationOp(Stream& s, Operation ser_action) { READWRITE(prevout); - READWRITE(*(CScriptBase*)(&scriptSig)); + READWRITE(scriptSig); READWRITE(nSequence); } @@ -160,7 +160,7 @@ class CTxOut template inline void SerializationOp(Stream& s, Operation ser_action) { READWRITE(nValue); - READWRITE(*(CScriptBase*)(&scriptPubKey)); + READWRITE(scriptPubKey); } void SetNull() diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 71f5d00af0d2..c58f8e8a1588 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -1036,7 +1036,7 @@ static void ApplyStats(CCoinsStats &stats, CHashWriter& ss, const uint256& hash, stats.nTransactions++; for (const auto output : outputs) { ss << VARINT(output.first + 1); - ss << *(const CScriptBase*)(&output.second.out.scriptPubKey); + ss << output.second.out.scriptPubKey; ss << VARINT(output.second.out.nValue); stats.nTransactionOutputs++; stats.nTotalAmount += output.second.out.nValue; diff --git a/src/script/interpreter.cpp b/src/script/interpreter.cpp index 533915318864..a2d0b3f1947d 100644 --- a/src/script/interpreter.cpp +++ b/src/script/interpreter.cpp @@ -1073,7 +1073,7 @@ class CTransactionSignatureSerializer { // Serialize the script if (nInput != nIn) // Blank out other inputs' signatures - ::Serialize(s, CScriptBase()); + ::Serialize(s, CScript()); else SerializeScriptCode(s); // Serialize the nSequence diff --git a/src/script/script.h b/src/script/script.h index 77eb896503bc..285986083e84 100644 --- a/src/script/script.h +++ b/src/script/script.h @@ -8,6 +8,7 @@ #include "crypto/common.h" #include "prevector.h" +#include "serialize.h" #include #include @@ -401,6 +402,13 @@ class CScript : public CScriptBase CScript(std::vector::const_iterator pbegin, std::vector::const_iterator pend) : CScriptBase(pbegin, pend) { } CScript(const unsigned char* pbegin, const unsigned char* pend) : CScriptBase(pbegin, pend) { } + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action) { + READWRITE(static_cast(*this)); + } + CScript& operator+=(const CScript& b) { insert(end(), b.begin(), b.end()); diff --git a/src/test/dbwrapper_tests.cpp b/src/test/dbwrapper_tests.cpp index 156c19315c65..2141b400432c 100644 --- a/src/test/dbwrapper_tests.cpp +++ b/src/test/dbwrapper_tests.cpp @@ -95,7 +95,7 @@ BOOST_AUTO_TEST_CASE(dbwrapper_iterator) uint256 in2 = InsecureRand256(); BOOST_CHECK(dbw.Write(key2, in2)); - std::unique_ptr it(const_cast(&dbw)->NewIterator()); + std::unique_ptr it(const_cast(dbw).NewIterator()); // Be sure to seek past the obfuscation key (if it exists) it->Seek(key); @@ -210,7 +210,7 @@ BOOST_AUTO_TEST_CASE(iterator_ordering) BOOST_CHECK(dbw.Write(key, value)); } - std::unique_ptr it(const_cast(&dbw)->NewIterator()); + std::unique_ptr it(const_cast(dbw).NewIterator()); for (int c=0; c<2; ++c) { int seek_start; if (c == 0) @@ -286,7 +286,7 @@ BOOST_AUTO_TEST_CASE(iterator_string_ordering) } } - std::unique_ptr it(const_cast(&dbw)->NewIterator()); + std::unique_ptr it(const_cast(dbw).NewIterator()); for (int c=0; c<2; ++c) { int seek_start; if (c == 0) diff --git a/src/txdb.cpp b/src/txdb.cpp index 13fe5b9109d6..fbcf8fc0b6c2 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -129,7 +129,7 @@ bool CBlockTreeDB::ReadLastBlockFile(int &nFile) { CCoinsViewCursor *CCoinsViewDB::Cursor() const { - CCoinsViewDBCursor *i = new CCoinsViewDBCursor(const_cast(&db)->NewIterator(), GetBestBlock()); + CCoinsViewDBCursor *i = new CCoinsViewDBCursor(const_cast(db).NewIterator(), GetBestBlock()); /* It seems that there are no "const iterators" for LevelDB. Since we only need read operations on it, use a const-cast to get around that restriction. */ diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index cf8457c43beb..f9bbbd82b248 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -97,23 +97,23 @@ bool CWalletDB::WriteMasterKey(unsigned int nID, const CMasterKey& kMasterKey) bool CWalletDB::WriteCScript(const uint160& hash, const CScript& redeemScript) { - return WriteIC(std::make_pair(std::string("cscript"), hash), *(const CScriptBase*)(&redeemScript), false); + return WriteIC(std::make_pair(std::string("cscript"), hash), redeemScript, false); } bool CWalletDB::WriteWatchOnly(const CScript &dest, const CKeyMetadata& keyMeta) { - if (!WriteIC(std::make_pair(std::string("watchmeta"), *(const CScriptBase*)(&dest)), keyMeta)) { + if (!WriteIC(std::make_pair(std::string("watchmeta"), dest), keyMeta)) { return false; } - return WriteIC(std::make_pair(std::string("watchs"), *(const CScriptBase*)(&dest)), '1'); + return WriteIC(std::make_pair(std::string("watchs"), dest), '1'); } bool CWalletDB::EraseWatchOnly(const CScript &dest) { - if (!EraseIC(std::make_pair(std::string("watchmeta"), *(const CScriptBase*)(&dest)))) { + if (!EraseIC(std::make_pair(std::string("watchmeta"), dest))) { return false; } - return EraseIC(std::make_pair(std::string("watchs"), *(const CScriptBase*)(&dest))); + return EraseIC(std::make_pair(std::string("watchs"), dest)); } bool CWalletDB::WriteBestBlock(const CBlockLocator& locator) @@ -326,7 +326,7 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, { wss.nWatchKeys++; CScript script; - ssKey >> *(CScriptBase*)(&script); + ssKey >> script; char fYes; ssValue >> fYes; if (fYes == '1') @@ -443,7 +443,7 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, else if (strType == "watchmeta") { CScript script; - ssKey >> *(CScriptBase*)(&script); + ssKey >> script; keyID = CScriptID(script); } @@ -476,7 +476,7 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, uint160 hash; ssKey >> hash; CScript script; - ssValue >> *(CScriptBase*)(&script); + ssValue >> script; if (!pwallet->LoadCScript(script)) { strErr = "Error reading wallet database: LoadCScript failed"; From ab1120956eb1c0afc144c2d598c80eba43cc5566 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Sun, 16 Jul 2017 12:01:31 -0700 Subject: [PATCH 09/36] Merge #10844: Use range based for loop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit d0413c670 Use range based for loop (René Nyffenegger) Pull request description: Instead of iterating over 0 .. 1 and then deciding on an actual desired value, use a range based for loop for the desired value. Tree-SHA512: 0a7a4a80516c9f16cf97fa7d257088b8386360e19b93c4deac3d745b6270ea452c513821686d7d14a159a235763e034f9b14eef222ca15f7eb71c37bd1c2c380 --- src/test/dbwrapper_tests.cpp | 23 +++++------------------ 1 file changed, 5 insertions(+), 18 deletions(-) diff --git a/src/test/dbwrapper_tests.cpp b/src/test/dbwrapper_tests.cpp index 2141b400432c..48c42554303b 100644 --- a/src/test/dbwrapper_tests.cpp +++ b/src/test/dbwrapper_tests.cpp @@ -24,8 +24,7 @@ BOOST_FIXTURE_TEST_SUITE(dbwrapper_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(dbwrapper) { // Perform tests both obfuscated and non-obfuscated. - for (int i = 0; i < 2; i++) { - bool obfuscate = (bool)i; + for (bool obfuscate : {false, true}) { fs::path ph = fs::temp_directory_path() / fs::unique_path(); CDBWrapper dbw(ph, (1 << 20), true, false, obfuscate); char key = 'k'; @@ -45,8 +44,7 @@ BOOST_AUTO_TEST_CASE(dbwrapper) BOOST_AUTO_TEST_CASE(dbwrapper_batch) { // Perform tests both obfuscated and non-obfuscated. - for (int i = 0; i < 2; i++) { - bool obfuscate = (bool)i; + for (bool obfuscate : {false, true}) { fs::path ph = fs::temp_directory_path() / fs::unique_path(); CDBWrapper dbw(ph, (1 << 20), true, false, obfuscate); @@ -82,8 +80,7 @@ BOOST_AUTO_TEST_CASE(dbwrapper_batch) BOOST_AUTO_TEST_CASE(dbwrapper_iterator) { // Perform tests both obfuscated and non-obfuscated. - for (int i = 0; i < 2; i++) { - bool obfuscate = (bool)i; + for (bool obfuscate : {false, true}) { fs::path ph = fs::temp_directory_path() / fs::unique_path(); CDBWrapper dbw(ph, (1 << 20), true, false, obfuscate); @@ -211,12 +208,7 @@ BOOST_AUTO_TEST_CASE(iterator_ordering) } std::unique_ptr it(const_cast(dbw).NewIterator()); - for (int c=0; c<2; ++c) { - int seek_start; - if (c == 0) - seek_start = 0x00; - else - seek_start = 0x80; + for (int seek_start : {0x00, 0x80}) { it->Seek((uint8_t)seek_start); for (int x=seek_start; x<256; ++x) { uint8_t key; @@ -287,12 +279,7 @@ BOOST_AUTO_TEST_CASE(iterator_string_ordering) } std::unique_ptr it(const_cast(dbw).NewIterator()); - for (int c=0; c<2; ++c) { - int seek_start; - if (c == 0) - seek_start = 0; - else - seek_start = 5; + for (int seek_start : {0, 5}) { snprintf(buf, sizeof(buf), "%d", seek_start); StringContentsSerializer seek_key(buf); it->Seek(seek_key); From e4c41e74aa752e52f6a340c976f6a569d704613a Mon Sep 17 00:00:00 2001 From: Pasta Date: Fri, 12 Jul 2019 22:22:12 -0500 Subject: [PATCH 10/36] fix build related to #10760 Signed-off-by: Pasta --- src/script/script.h | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/script/script.h b/src/script/script.h index 285986083e84..ad24cbf036e2 100644 --- a/src/script/script.h +++ b/src/script/script.h @@ -657,15 +657,6 @@ class CScript : public CScriptBase CScriptBase::clear(); shrink_to_fit(); } - - template - void Serialize(Stream& s) const { - s << *(CScriptBase*)this; - } - template - void Unserialize(Stream& s) { - s >> *(CScriptBase*)this; - } }; class CReserveScript From 6320d516b3d888c1aeadbd4e128278fe94b08676 Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Tue, 6 Jun 2017 12:28:45 +0200 Subject: [PATCH 11/36] Merge #10535: [qa] fundrawtx: Fix shutdown race fa62dad [qa] fundrawtx: Fix shutdown race (MarcoFalke) Tree-SHA512: bf1a3dd053a0f950639bb78fc6ef6255cc5e0b3d27efb97d956f4bd3342839c28a91d8d0793ef7473ab0c8beda30034767031060ae1895408513aa1d9f7c9ff4 --- test/functional/fundrawtransaction.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/functional/fundrawtransaction.py b/test/functional/fundrawtransaction.py index b9488f9b431a..8f99eb5ef2eb 100755 --- a/test/functional/fundrawtransaction.py +++ b/test/functional/fundrawtransaction.py @@ -453,7 +453,7 @@ def run_test(self): self.stop_node(2) self.stop_node(3) self.nodes[1].encryptwallet("test") - self.nodes.pop(1) + bitcoind_processes[1].wait(timeout=BITCOIND_PROC_WAIT_TIMEOUT) wait_node(1) self.nodes = self.start_nodes(self.num_nodes, self.options.tmpdir, [['-usehd=0']] * self.num_nodes) From 3dddfc6b482b2a23b6d62f1dc4a1e1b82eb92532 Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Tue, 6 Jun 2017 15:51:13 +0200 Subject: [PATCH 12/36] Merge #9909: tests: Add FindEarliestAtLeast test for edge cases cf66f04 Add FindEarliestAtLeast test for edge cases (Russell Yanofsky) Tree-SHA512: bed2e9b2eefe0014414f1be8f0ba236b1dcc6744a670ffa546100c62817dd06a117cc62cdff80c4988d2b374cae9a57f2bb135fc582ffb51ca2d34ab07bb3fe0 --- src/test/skiplist_tests.cpp | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/src/test/skiplist_tests.cpp b/src/test/skiplist_tests.cpp index 3f457919926b..eea7be4b4db0 100644 --- a/src/test/skiplist_tests.cpp +++ b/src/test/skiplist_tests.cpp @@ -142,4 +142,39 @@ BOOST_AUTO_TEST_CASE(findearliestatleast_test) BOOST_CHECK(vBlocksMain[r].GetAncestor(ret->nHeight) == ret); } } + +BOOST_AUTO_TEST_CASE(findearliestatleast_edge_test) +{ + std::list blocks; + for (unsigned int timeMax : {100, 100, 100, 200, 200, 200, 300, 300, 300}) { + CBlockIndex* prev = blocks.empty() ? nullptr : &blocks.back(); + blocks.emplace_back(); + blocks.back().nHeight = prev ? prev->nHeight + 1 : 0; + blocks.back().pprev = prev; + blocks.back().BuildSkip(); + blocks.back().nTimeMax = timeMax; + } + + CChain chain; + chain.SetTip(&blocks.back()); + + BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(50)->nHeight, 0); + BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(100)->nHeight, 0); + BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(150)->nHeight, 3); + BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(200)->nHeight, 3); + BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(250)->nHeight, 6); + BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(300)->nHeight, 6); + BOOST_CHECK(!chain.FindEarliestAtLeast(350)); + + BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(0)->nHeight, 0); + BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(-1)->nHeight, 0); + + BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(std::numeric_limits::min())->nHeight, 0); + BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(std::numeric_limits::min())->nHeight, 0); + BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(-int64_t(std::numeric_limits::max()) - 1)->nHeight, 0); + BOOST_CHECK(!chain.FindEarliestAtLeast(std::numeric_limits::max())); + BOOST_CHECK(!chain.FindEarliestAtLeast(std::numeric_limits::max())); + BOOST_CHECK(!chain.FindEarliestAtLeast(int64_t(std::numeric_limits::max()) + 1)); +} + BOOST_AUTO_TEST_SUITE_END() From 0fe9f757ecb3aa1cbd47a8c9b902f98adeab7254 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jorge=20Tim=C3=B3n?= Date: Thu, 1 Jun 2017 22:35:42 +0200 Subject: [PATCH 13/36] \#10193 clang-format: Delete ForEachMacros --- src/.clang-format | 1 - 1 file changed, 1 deletion(-) diff --git a/src/.clang-format b/src/.clang-format index f1113e635362..a8f45e4ca790 100644 --- a/src/.clang-format +++ b/src/.clang-format @@ -25,7 +25,6 @@ ContinuationIndentWidth: 4 Cpp11BracedListStyle: true DerivePointerAlignment: false DisableFormat: false -ForEachMacros: [ foreach, Q_FOREACH, BOOST_FOREACH, BOOST_REVERSE_FOREACH ] IndentCaseLabels: false IndentFunctionDeclarationAfterType: false IndentWidth: 4 From c123e10cc80de660252b323492c35e1c86066a93 Mon Sep 17 00:00:00 2001 From: Pasta Date: Fri, 12 Jul 2019 23:56:57 -0500 Subject: [PATCH 14/36] #10193 scripted-diff: Remove #include -BEGIN VERIFY SCRIPT- sed -i ':a;N;$!ba;s/#include \n//' ./src/*.h ./src/*.cpp ./src/*/*.h ./src/*/*.cpp ./src/*/*/*.h ./src/*/*/*.cpp -END VERIFY SCRIPT- Signed-off-by: Pasta --- src/bench/coin_selection.cpp | 1 - src/bloom.cpp | 1 - src/checkpoints.cpp | 1 - src/checkqueue.h | 1 - src/coins.cpp | 1 - src/core_write.cpp | 1 - src/httprpc.cpp | 1 - src/memusage.h | 1 - src/net.h | 1 - src/policy/policy.cpp | 1 - src/qt/addresstablemodel.cpp | 1 - src/qt/recentrequeststablemodel.cpp | 1 - src/qt/transactionrecord.cpp | 1 - src/qt/walletmodel.cpp | 1 - src/rpc/net.cpp | 1 - src/rpc/server.cpp | 1 - src/script/ismine.cpp | 1 - src/script/sign.cpp | 1 - src/script/standard.cpp | 1 - src/sync.cpp | 1 - src/test/getarg_tests.cpp | 1 - src/test/multisig_tests.cpp | 1 - src/test/script_tests.cpp | 1 - src/timedata.cpp | 1 - src/torcontrol.cpp | 1 - src/wallet/crypter.cpp | 1 - src/wallet/db.cpp | 1 - src/wallet/rpcdump.cpp | 1 - src/wallet/test/accounting_tests.cpp | 1 - src/wallet/walletdb.cpp | 1 - 30 files changed, 30 deletions(-) diff --git a/src/bench/coin_selection.cpp b/src/bench/coin_selection.cpp index 0e86568e5bf8..b6881cb0835c 100644 --- a/src/bench/coin_selection.cpp +++ b/src/bench/coin_selection.cpp @@ -5,7 +5,6 @@ #include "bench.h" #include "wallet/wallet.h" -#include #include static void addCoin(const CAmount& nValue, const CWallet& wallet, std::vector& vCoins) diff --git a/src/bloom.cpp b/src/bloom.cpp index 1a58b8714205..7ca1071234bc 100644 --- a/src/bloom.cpp +++ b/src/bloom.cpp @@ -18,7 +18,6 @@ #include #include -#include #define LN2SQUARED 0.4804530139182014246671025263266649717305529515945455 #define LN2 0.6931471805599453094172321214581765680755001343602552 diff --git a/src/checkpoints.cpp b/src/checkpoints.cpp index 6f4f0f2825bc..37490b251502 100644 --- a/src/checkpoints.cpp +++ b/src/checkpoints.cpp @@ -12,7 +12,6 @@ #include -#include namespace Checkpoints { diff --git a/src/checkqueue.h b/src/checkqueue.h index d7b7b836dcf2..408e278d2162 100644 --- a/src/checkqueue.h +++ b/src/checkqueue.h @@ -10,7 +10,6 @@ #include #include -#include #include #include diff --git a/src/coins.cpp b/src/coins.cpp index 801b94d79cd8..a727ae714e69 100644 --- a/src/coins.cpp +++ b/src/coins.cpp @@ -9,7 +9,6 @@ #include "random.h" #include -#include bool CCoinsView::GetCoin(const COutPoint &outpoint, Coin &coin) const { return false; } uint256 CCoinsView::GetBestBlock() const { return uint256(); } diff --git a/src/core_write.cpp b/src/core_write.cpp index edb60b55db55..87664f0ac948 100644 --- a/src/core_write.cpp +++ b/src/core_write.cpp @@ -22,7 +22,6 @@ #include "evo/specialtx.h" #include "llmq/quorums_commitment.h" -#include std::string FormatScript(const CScript& script) { diff --git a/src/httprpc.cpp b/src/httprpc.cpp index 36a1fd651e5d..2104ac75785b 100644 --- a/src/httprpc.cpp +++ b/src/httprpc.cpp @@ -18,7 +18,6 @@ #include #include // boost::trim -#include /** WWW-Authenticate to present with 401 Unauthorized response */ static const char* WWW_AUTH_HEADER_DATA = "Basic realm=\"jsonrpc\""; diff --git a/src/memusage.h b/src/memusage.h index d73b1f937d51..8db0fd606706 100644 --- a/src/memusage.h +++ b/src/memusage.h @@ -15,7 +15,6 @@ #include #include -#include namespace memusage { diff --git a/src/net.h b/src/net.h index 9683184c7544..2f7a545b8084 100644 --- a/src/net.h +++ b/src/net.h @@ -37,7 +37,6 @@ #include #endif -#include #include // "Optimistic send" was introduced in the beginning of the Bitcoin project. I assume this was done because it was diff --git a/src/policy/policy.cpp b/src/policy/policy.cpp index cf07968e15df..864c88445b96 100644 --- a/src/policy/policy.cpp +++ b/src/policy/policy.cpp @@ -13,7 +13,6 @@ #include "util.h" #include "utilstrencodings.h" -#include CAmount GetDustThreshold(const CTxOut& txout, const CFeeRate& dustRelayFeeIn) { diff --git a/src/qt/addresstablemodel.cpp b/src/qt/addresstablemodel.cpp index 9c8b03992a4c..2a6fb40cae8d 100644 --- a/src/qt/addresstablemodel.cpp +++ b/src/qt/addresstablemodel.cpp @@ -11,7 +11,6 @@ #include "base58.h" #include "wallet/wallet.h" -#include #include #include diff --git a/src/qt/recentrequeststablemodel.cpp b/src/qt/recentrequeststablemodel.cpp index e1850983535a..3558861e3554 100644 --- a/src/qt/recentrequeststablemodel.cpp +++ b/src/qt/recentrequeststablemodel.cpp @@ -11,7 +11,6 @@ #include "clientversion.h" #include "streams.h" -#include RecentRequestsTableModel::RecentRequestsTableModel(CWallet *wallet, WalletModel *parent) : QAbstractTableModel(parent), walletModel(parent) diff --git a/src/qt/transactionrecord.cpp b/src/qt/transactionrecord.cpp index 7fde2aa4349e..6e1903dc5161 100644 --- a/src/qt/transactionrecord.cpp +++ b/src/qt/transactionrecord.cpp @@ -15,7 +15,6 @@ #include -#include /* Return positive answer if transaction should be shown in list. */ diff --git a/src/qt/walletmodel.cpp b/src/qt/walletmodel.cpp index b2ff459a3bc3..9ec2dacb4e1d 100644 --- a/src/qt/walletmodel.cpp +++ b/src/qt/walletmodel.cpp @@ -34,7 +34,6 @@ #include #include -#include WalletModel::WalletModel(const PlatformStyle *platformStyle, CWallet *_wallet, OptionsModel *_optionsModel, QObject *parent) : QObject(parent), wallet(_wallet), optionsModel(_optionsModel), addressTableModel(0), diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index efbcea2c6297..ffe5df55427f 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -20,7 +20,6 @@ #include "utilstrencodings.h" #include "version.h" -#include #include diff --git a/src/rpc/server.cpp b/src/rpc/server.cpp index b02c4bacb147..b5261072b6b3 100644 --- a/src/rpc/server.cpp +++ b/src/rpc/server.cpp @@ -18,7 +18,6 @@ #include #include -#include #include #include // for to_upper() #include diff --git a/src/script/ismine.cpp b/src/script/ismine.cpp index fe9215c67da2..ce6033fa70e6 100644 --- a/src/script/ismine.cpp +++ b/src/script/ismine.cpp @@ -11,7 +11,6 @@ #include "script/standard.h" #include "script/sign.h" -#include typedef std::vector valtype; diff --git a/src/script/sign.cpp b/src/script/sign.cpp index 08b88017cdae..a60bd1f0ee1e 100644 --- a/src/script/sign.cpp +++ b/src/script/sign.cpp @@ -12,7 +12,6 @@ #include "script/standard.h" #include "uint256.h" -#include typedef std::vector valtype; diff --git a/src/script/standard.cpp b/src/script/standard.cpp index 4046f12c3ab9..328b3948a8f2 100644 --- a/src/script/standard.cpp +++ b/src/script/standard.cpp @@ -10,7 +10,6 @@ #include "util.h" #include "utilstrencodings.h" -#include typedef std::vector valtype; diff --git a/src/sync.cpp b/src/sync.cpp index 785b4f07cfd4..98b9b63a99c1 100644 --- a/src/sync.cpp +++ b/src/sync.cpp @@ -9,7 +9,6 @@ #include -#include #include #ifdef DEBUG_LOCKCONTENTION diff --git a/src/test/getarg_tests.cpp b/src/test/getarg_tests.cpp index 28bec6374a29..44109f216127 100644 --- a/src/test/getarg_tests.cpp +++ b/src/test/getarg_tests.cpp @@ -9,7 +9,6 @@ #include #include -#include #include BOOST_FIXTURE_TEST_SUITE(getarg_tests, BasicTestingSetup) diff --git a/src/test/multisig_tests.cpp b/src/test/multisig_tests.cpp index d1dc6ed8058b..44013b94e3b8 100644 --- a/src/test/multisig_tests.cpp +++ b/src/test/multisig_tests.cpp @@ -14,7 +14,6 @@ #include "test/test_dash.h" -#include #include typedef std::vector valtype; diff --git a/src/test/script_tests.cpp b/src/test/script_tests.cpp index 18282dd6cc13..0485d715b9a7 100644 --- a/src/test/script_tests.cpp +++ b/src/test/script_tests.cpp @@ -23,7 +23,6 @@ #include #include -#include #include #include diff --git a/src/timedata.cpp b/src/timedata.cpp index 6549ecd53e12..a5c5bd44f3ae 100644 --- a/src/timedata.cpp +++ b/src/timedata.cpp @@ -15,7 +15,6 @@ #include "utilstrencodings.h" #include "warnings.h" -#include static CCriticalSection cs_nTimeOffset; static int64_t nTimeOffset = 0; diff --git a/src/torcontrol.cpp b/src/torcontrol.cpp index b7c07f17f492..50e73d95a8d2 100644 --- a/src/torcontrol.cpp +++ b/src/torcontrol.cpp @@ -17,7 +17,6 @@ #include #include -#include #include #include #include diff --git a/src/wallet/crypter.cpp b/src/wallet/crypter.cpp index 83279e2e2438..c986e5767821 100644 --- a/src/wallet/crypter.cpp +++ b/src/wallet/crypter.cpp @@ -12,7 +12,6 @@ #include #include -#include int CCrypter::BytesToKeySHA512AES(const std::vector& chSalt, const SecureString& strKeyData, int count, unsigned char *key,unsigned char *iv) const { diff --git a/src/wallet/db.cpp b/src/wallet/db.cpp index 5581f77c9eb9..d8da1b943d36 100644 --- a/src/wallet/db.cpp +++ b/src/wallet/db.cpp @@ -18,7 +18,6 @@ #include #endif -#include #include // diff --git a/src/wallet/rpcdump.cpp b/src/wallet/rpcdump.cpp index 6d0362c7ba74..dc6bc0430d57 100644 --- a/src/wallet/rpcdump.cpp +++ b/src/wallet/rpcdump.cpp @@ -27,7 +27,6 @@ #include -#include std::string static EncodeDumpTime(int64_t nTime) { return DateTimeStrFormat("%Y-%m-%dT%H:%M:%SZ", nTime); diff --git a/src/wallet/test/accounting_tests.cpp b/src/wallet/test/accounting_tests.cpp index 927aa980c198..d597bd911dcc 100644 --- a/src/wallet/test/accounting_tests.cpp +++ b/src/wallet/test/accounting_tests.cpp @@ -8,7 +8,6 @@ #include -#include #include extern CWallet* pwalletMain; diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index f9bbbd82b248..4a8375135e20 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -20,7 +20,6 @@ #include -#include #include // From d6d462fd7b75d5dc47ece7d0c1e7bb65b054d30e Mon Sep 17 00:00:00 2001 From: Pasta Date: Fri, 12 Jul 2019 23:58:15 -0500 Subject: [PATCH 15/36] #10193 scripted-diff: Remove BOOST_REVERSE_FOREACH -BEGIN VERIFY SCRIPT- sed -i 's/BOOST_REVERSE_FOREACH(\(.*\), \(.*\))/for (\1 : reverse_iterate(\2))/' ./src/*.h ./src/*.cpp ./src/*/*.h ./src/*/*.cpp ; -END VERIFY SCRIPT- Signed-off-by: Pasta --- src/checkpoints.cpp | 2 +- src/net_processing.cpp | 4 ++-- src/test/prevector_tests.cpp | 4 ++-- src/txmempool.cpp | 2 +- src/validation.cpp | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/checkpoints.cpp b/src/checkpoints.cpp index 37490b251502..fd6666c51751 100644 --- a/src/checkpoints.cpp +++ b/src/checkpoints.cpp @@ -19,7 +19,7 @@ namespace Checkpoints { { const MapCheckpoints& checkpoints = data.mapCheckpoints; - BOOST_REVERSE_FOREACH(const MapCheckpoints::value_type& i, checkpoints) + for (const MapCheckpoints::value_type& i : reverse_iterate(checkpoints)) { const uint256& hash = i.second; BlockMap::const_iterator t = mapBlockIndex.find(hash); diff --git a/src/net_processing.cpp b/src/net_processing.cpp index addf4d2807f9..abe7a0aeea2c 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -900,7 +900,7 @@ void PeerLogicValidation::UpdatedBlockTip(const CBlockIndex *pindexNew, const CB // Relay inventory, but don't relay old inventory during initial block download. connman->ForEachNode([nNewHeight, &vHashes](CNode* pnode) { if (nNewHeight > (pnode->nStartingHeight != -1 ? pnode->nStartingHeight - 2000 : 0)) { - BOOST_REVERSE_FOREACH(const uint256& hash, vHashes) { + for (const uint256& hash : reverse_iterate(vHashes)) { pnode->PushBlockHash(hash); } } @@ -2672,7 +2672,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr } else { std::vector vGetData; // Download as much as possible, from earliest to latest. - BOOST_REVERSE_FOREACH(const CBlockIndex *pindex, vToFetch) { + for (const CBlockIndex *pindex : reverse_iterate(vToFetch)) { if (nodestate->nBlocksInFlight >= MAX_BLOCKS_IN_TRANSIT_PER_PEER) { // Can't download any more from this peer break; diff --git a/src/test/prevector_tests.cpp b/src/test/prevector_tests.cpp index 01308cbbb312..b0e9df8bb676 100644 --- a/src/test/prevector_tests.cpp +++ b/src/test/prevector_tests.cpp @@ -56,13 +56,13 @@ class prevector_tester { for (const T& v : pre_vector) { local_check(v == real_vector[pos++]); } - BOOST_REVERSE_FOREACH(const T& v, pre_vector) { + for (const T& v : reverse_iterate(pre_vector)) { local_check(v == real_vector[--pos]); } for (const T& v : const_pre_vector) { local_check(v == real_vector[pos++]); } - BOOST_REVERSE_FOREACH(const T& v, const_pre_vector) { + for (const T& v : reverse_iterate(const_pre_vector)) { local_check(v == real_vector[--pos]); } CDataStream ss1(SER_DISK, 0); diff --git a/src/txmempool.cpp b/src/txmempool.cpp index 87470ae40c59..d80d36de2196 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -129,7 +129,7 @@ void CTxMemPool::UpdateTransactionsFromBlock(const std::vector &vHashes // This maximizes the benefit of the descendant cache and guarantees that // setMemPoolChildren will be updated, an assumption made in // UpdateForDescendants. - BOOST_REVERSE_FOREACH(const uint256 &hash, vHashesToUpdate) { + for (const uint256 &hash : reverse_iterate(vHashesToUpdate)) { // we cache the in-mempool children to avoid duplicate updates setEntries setChildren; // calculate children from mapNextTx diff --git a/src/validation.cpp b/src/validation.cpp index 0623a5c9e15f..50d677075de8 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -2594,7 +2594,7 @@ static bool ActivateBestChainStep(CValidationState& state, const CChainParams& c nHeight = nTargetHeight; // Connect new blocks. - BOOST_REVERSE_FOREACH(CBlockIndex *pindexConnect, vpindexToConnect) { + for (CBlockIndex *pindexConnect : reverse_iterate(vpindexToConnect)) { if (!ConnectTip(state, chainparams, pindexConnect, pindexConnect == pindexMostWork ? pblock : std::shared_ptr(), connectTrace, disconnectpool)) { if (state.IsInvalid()) { // The block violates a consensus rule. From 96897e51a986d4e8954cde22b528014f9ef0a375 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jorge=20Tim=C3=B3n?= Date: Mon, 19 Jun 2017 01:45:23 +0200 Subject: [PATCH 16/36] \#10193 Fix const_reverse_iterator constructor (pass const ptr) --- src/prevector.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/prevector.h b/src/prevector.h index c5f84f1b7a74..c72546fea429 100644 --- a/src/prevector.h +++ b/src/prevector.h @@ -132,7 +132,7 @@ class prevector { typedef const T* pointer; typedef const T& reference; typedef std::bidirectional_iterator_tag iterator_category; - const_reverse_iterator(T* ptr_) : ptr(ptr_) {} + const_reverse_iterator(const T* ptr_) : ptr(ptr_) {} const_reverse_iterator(reverse_iterator x) : ptr(&(*x)) {} const T& operator*() const { return *ptr; } const T* operator->() const { return ptr; } From fa2cd234b294657d50ece5164d9436f8a7324225 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jorge=20Tim=C3=B3n?= Date: Thu, 13 Apr 2017 00:24:40 +0200 Subject: [PATCH 17/36] \#10193 Introduce src/reverse_iterator.hpp and include it... ...where it will be needed Taken from https://gist.github.com/arvidsson/7231973 with small modifications to fit the bitcoin core project --- src/Makefile.am | 1 + src/checkpoints.cpp | 1 + src/net_processing.cpp | 1 + src/reverse_iterator.h | 39 ++++++++++++++++++++++++++++++++++++ src/test/prevector_tests.cpp | 1 + src/txmempool.cpp | 1 + src/validation.cpp | 1 + 7 files changed, 45 insertions(+) create mode 100644 src/reverse_iterator.h diff --git a/src/Makefile.am b/src/Makefile.am index 4354f2d36f48..e89720f9ba32 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -194,6 +194,7 @@ BITCOIN_CORE_H = \ pow.h \ protocol.h \ random.h \ + reverse_iterator.h \ reverselock.h \ rpc/blockchain.h \ rpc/client.h \ diff --git a/src/checkpoints.cpp b/src/checkpoints.cpp index fd6666c51751..42c55756a073 100644 --- a/src/checkpoints.cpp +++ b/src/checkpoints.cpp @@ -7,6 +7,7 @@ #include "chain.h" #include "chainparams.h" +#include "reverse_iterator.h" #include "validation.h" #include "uint256.h" diff --git a/src/net_processing.cpp b/src/net_processing.cpp index abe7a0aeea2c..0afc0e8e7bca 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -22,6 +22,7 @@ #include "primitives/block.h" #include "primitives/transaction.h" #include "random.h" +#include "reverse_iterator.h" #include "tinyformat.h" #include "txmempool.h" #include "ui_interface.h" diff --git a/src/reverse_iterator.h b/src/reverse_iterator.h new file mode 100644 index 000000000000..409f895ce062 --- /dev/null +++ b/src/reverse_iterator.h @@ -0,0 +1,39 @@ +// Taken from https://gist.github.com/arvidsson/7231973 + +#ifndef BITCOIN_REVERSE_ITERATOR_HPP +#define BITCOIN_REVERSE_ITERATOR_HPP + +/** + * Template used for reverse iteration in C++11 range-based for loops. + * + * std::vector v = {1, 2, 3, 4, 5}; + * for (auto x : reverse_iterate(v)) + * std::cout << x << " "; + */ + +template +class reverse_range +{ + T &x; + +public: + reverse_range(T &x) : x(x) {} + + auto begin() const -> decltype(this->x.rbegin()) + { + return x.rbegin(); + } + + auto end() const -> decltype(this->x.rend()) + { + return x.rend(); + } +}; + +template +reverse_range reverse_iterate(T &x) +{ + return reverse_range(x); +} + +#endif // BITCOIN_REVERSE_ITERATOR_HPP diff --git a/src/test/prevector_tests.cpp b/src/test/prevector_tests.cpp index b0e9df8bb676..24524cf8452d 100644 --- a/src/test/prevector_tests.cpp +++ b/src/test/prevector_tests.cpp @@ -5,6 +5,7 @@ #include #include "prevector.h" +#include "reverse_iterator.h" #include "serialize.h" #include "streams.h" diff --git a/src/txmempool.cpp b/src/txmempool.cpp index d80d36de2196..a0a1393d8637 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -12,6 +12,7 @@ #include "policy/policy.h" #include "policy/fees.h" #include "random.h" +#include "reverse_iterator.h" #include "streams.h" #include "timedata.h" #include "util.h" diff --git a/src/validation.cpp b/src/validation.cpp index 50d677075de8..461616ae25de 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -24,6 +24,7 @@ #include "pow.h" #include "primitives/block.h" #include "primitives/transaction.h" +#include "reverse_iterator.h" #include "script/script.h" #include "script/sigcache.h" #include "script/standard.h" From 2ada45284bc3c6308c51d32a33da64c7ffa83a03 Mon Sep 17 00:00:00 2001 From: Jonas Schnelli Date: Wed, 19 Jul 2017 11:53:20 +0200 Subject: [PATCH 18/36] Merge #10864: Avoid redundant redeclaration of GetWarnings(const string&) e0d459264 Avoid redundant redeclaration of GetWarnings(const string&) (practicalswift) Pull request description: Avoid redundant redeclaration of `GetWarnings(const string&)`. `std::string GetWarnings(const std::string& strFor)` is declared in `warnings.h` and defined in `warnings.cpp`. Tree-SHA512: d1503e00a2073cf080d66eafa303dc9c660a7ac15d4d2abcf2e4aa69cf9622d89a8e3f09324139bb7b8debaa6d1ee4a1c1681d347cebd99b1d3672a4da6d1ace --- src/qt/clientmodel.cpp | 1 + src/rpc/mining.cpp | 1 + src/rpc/misc.cpp | 1 + src/rpc/net.cpp | 2 +- src/validation.h | 8 -------- src/warnings.h | 7 +++++++ 6 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/qt/clientmodel.cpp b/src/qt/clientmodel.cpp index df2a88aaf553..81d9c5f6f6df 100644 --- a/src/qt/clientmodel.cpp +++ b/src/qt/clientmodel.cpp @@ -19,6 +19,7 @@ #include "txmempool.h" #include "ui_interface.h" #include "util.h" +#include "warnings.h" #include "masternode/masternode-sync.h" #include "privatesend/privatesend.h" diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index 7bdec5bb29e4..a823554267d2 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -25,6 +25,7 @@ #include "util.h" #include "utilstrencodings.h" #include "validationinterface.h" +#include "warnings.h" #include "governance/governance-classes.h" #include "masternode/masternode-payments.h" diff --git a/src/rpc/misc.cpp b/src/rpc/misc.cpp index 4e98a61ee197..53e036c76154 100644 --- a/src/rpc/misc.cpp +++ b/src/rpc/misc.cpp @@ -23,6 +23,7 @@ #include "wallet/wallet.h" #include "wallet/walletdb.h" #endif +#include "warnings.h" #include "masternode/masternode-sync.h" #include "spork.h" diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index ffe5df55427f..5c18296e27fb 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -19,7 +19,7 @@ #include "util.h" #include "utilstrencodings.h" #include "version.h" - +#include "warnings.h" #include diff --git a/src/validation.h b/src/validation.h index dceb2a60a31c..daca7342ca1b 100644 --- a/src/validation.h +++ b/src/validation.h @@ -272,14 +272,6 @@ void UnloadBlockIndex(); void ThreadScriptCheck(); /** Check whether we are doing an initial block download (synchronizing from disk or network) */ bool IsInitialBlockDownload(); -/** Format a string that describes several potential problems detected by the core. - * strFor can have three values: - * - "rpc": get critical warnings, which should put the client in safe mode if non-empty - * - "statusbar": get all warnings - * - "gui": get all warnings, translated (where possible) for GUI - * This function only returns the highest priority warning of the set selected by strFor. - */ -std::string GetWarnings(const std::string& strFor); /** Retrieve a transaction (from memory pool, or from disk, if possible) */ bool GetTransaction(const uint256 &hash, CTransactionRef &tx, const Consensus::Params& params, uint256 &hashBlock, bool fAllowSlow = false); /** Find the best known block, and make it the tip of the block chain */ diff --git a/src/warnings.h b/src/warnings.h index a7aa6574264c..fd0ca5394233 100644 --- a/src/warnings.h +++ b/src/warnings.h @@ -14,6 +14,13 @@ void SetfLargeWorkForkFound(bool flag); bool GetfLargeWorkForkFound(); void SetfLargeWorkInvalidChainFound(bool flag); bool GetfLargeWorkInvalidChainFound(); +/** Format a string that describes several potential problems detected by the core. + * strFor can have three values: + * - "rpc": get critical warnings, which should put the client in safe mode if non-empty + * - "statusbar": get all warnings + * - "gui": get all warnings, translated (where possible) for GUI + * This function only returns the highest priority warning of the set selected by strFor. + */ std::string GetWarnings(const std::string& strFor); static const bool DEFAULT_TESTSAFEMODE = false; From b535dfbc4116d69680743678235f8fcc17c692ba Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Wed, 19 Jul 2017 16:56:03 +0200 Subject: [PATCH 19/36] Merge #10857: [RPC] Add a deprecation warning to getinfo's output 1c9b818 getinfo deprecation warning (Andrew Chow) Pull request description: This is an alternative to #10841 This PR implements @gmaxwell's suggestion of a `nag` field for getinfo which warns about the deprecation. Instead of calling it `nag`, I have named it `deprecation-warning`. The output of `getinfo` will look like this: ``` { "version": 149900, "protocolversion": 70015, "walletversion": 139900, "balance": 0.00000000, "blocks": 476281, "timeoffset": 0, "connections": 2, "proxy": "", "difficulty": 804525194568.1318, "testnet": false, "keypoololdest": 1496858803, "keypoolsize": 197, "unlocked_until": 0, "paytxfee": 0.00000000, "relayfee": 0.00001000, "errors": "This is a pre-release test build - use at your own risk - do not use for mining or merchant applications", "deprecation-warning": "WARNING: getinfo is deprecated and will be fully removed in 0.16. Projects should transition to using getblockchaininfo, getnetworkinfo, and getwalletinfo before upgrading to 0.16" } ``` I think this should be tagged for 0.15 Tree-SHA512: ea1bac96a67f797519e8748ddd661cf0a1127cbc38f145b98f10cf9b54dcf0519b353062ce9888e1f51875497299c75ff5147566944451bc3fc117620e773489 --- src/rpc/misc.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/rpc/misc.cpp b/src/rpc/misc.cpp index 53e036c76154..1ee189f426cd 100644 --- a/src/rpc/misc.cpp +++ b/src/rpc/misc.cpp @@ -58,6 +58,7 @@ UniValue getinfo(const JSONRPCRequest& request) "\nDEPRECATED. Returns an object containing various state info.\n" "\nResult:\n" "{\n" + " \"deprecation-warning\": \"...\" (string) warning that the getinfo command is deprecated and will be removed in 0.16\n" " \"version\": xxxxx, (numeric) the server version\n" " \"protocolversion\": xxxxx, (numeric) the protocol version\n" " \"walletversion\": xxxxx, (numeric) the wallet version\n" @@ -66,7 +67,7 @@ UniValue getinfo(const JSONRPCRequest& request) " \"blocks\": xxxxxx, (numeric) the current number of blocks processed in the server\n" " \"timeoffset\": xxxxx, (numeric) the time offset\n" " \"connections\": xxxxx, (numeric) the number of connections\n" - " \"proxy\": \"host:port\", (string, optional) the proxy used by the server\n" + " \"proxy\": \"host:port\", (string, optional) the proxy used by the server\n" " \"difficulty\": xxxxxx, (numeric) the current difficulty\n" " \"testnet\": true|false, (boolean) if the server is using testnet or not\n" " \"keypoololdest\": xxxxxx, (numeric) the timestamp (seconds since Unix epoch) of the oldest pre-generated key in the key pool\n" @@ -74,7 +75,7 @@ UniValue getinfo(const JSONRPCRequest& request) " \"unlocked_until\": ttt, (numeric) the timestamp in seconds since epoch (midnight Jan 1 1970 GMT) that the wallet is unlocked for transfers, or 0 if the wallet is locked\n" " \"paytxfee\": x.xxxx, (numeric) the transaction fee set in " + CURRENCY_UNIT + "/kB\n" " \"relayfee\": x.xxxx, (numeric) minimum relay fee for transactions in " + CURRENCY_UNIT + "/kB\n" - " \"errors\": \"...\" (string) any error messages\n" + " \"errors\": \"...\" (string) any error messages\n" "}\n" "\nExamples:\n" + HelpExampleCli("getinfo", "") @@ -93,6 +94,8 @@ UniValue getinfo(const JSONRPCRequest& request) GetProxy(NET_IPV4, proxy); UniValue obj(UniValue::VOBJ); + obj.push_back(Pair("deprecation-warning", "WARNING: getinfo is deprecated and will be fully removed in 0.16." + " Projects should transition to using getblockchaininfo, getnetworkinfo, and getwalletinfo before upgrading to 0.16")); obj.push_back(Pair("version", CLIENT_VERSION)); obj.push_back(Pair("protocolversion", PROTOCOL_VERSION)); #ifdef ENABLE_WALLET From d587fe3f0eea6812b3310bf5948b7b9cf1da7f6b Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Wed, 19 Jul 2017 22:45:22 +0200 Subject: [PATCH 20/36] Merge #10681: add gdb attach process to test README 7ec3343 add gdb attach process to test README (Gregory Sanders) Pull request description: Saved me many hours. h/t to @jnewbery for the new guide efforts Tree-SHA512: 4d8ba1717eb842201079488f1cfe369d5d5114df5278643a3f996c986a51e3e039ea994a6f2f06bf6607b697388ad6561198da4693e3cb1ca4a8424e6d423d85 --- test/README.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/test/README.md b/test/README.md index 074d98c80f19..bce08f81bffe 100644 --- a/test/README.md +++ b/test/README.md @@ -155,6 +155,26 @@ import pdb; pdb.set_trace() anywhere in the test. You will then be able to inspect variables, as well as call methods that interact with the dashd nodes-under-test. +If further introspection of the bitcoind instances themselves becomes +necessary, this can be accomplished by first setting a pdb breakpoint +at an appropriate location, running the test to that point, then using +`gdb` to attach to the process and debug. + +For instance, to attach to `self.node[1]` during a run: + +```bash +2017-06-27 14:13:56.686000 TestFramework (INFO): Initializing test directory /tmp/user/1000/testo9vsdjo3 +``` + +use the directory path to get the pid from the pid file: + +```bash +cat /tmp/user/1000/testo9vsdjo3/node1/regtest/bitcoind.pid +gdb /home/example/bitcoind +``` + +Note: gdb attach step may require `sudo` + ### Util tests Util tests can be run locally by running `test/util/bitcoin-util-test.py`. From c7dcf79f02beb6d1a64f761ebfb1909f6c60f6d7 Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Thu, 20 Jul 2017 20:16:28 +0200 Subject: [PATCH 21/36] Merge #10821: Add SSE4 optimized SHA256 6b8d872 Protect SSE4 code behind a compile-time flag (Pieter Wuille) fa9be90 Add selftest for SHA256 transform (Pieter Wuille) c1ccb15 Add SSE4 based SHA256 (Pieter Wuille) 2991c91 Add SHA256 dispatcher (Pieter Wuille) 4d50f38 Support multi-block SHA256 transforms (Pieter Wuille) Pull request description: This adds an SSE4 assembly version of the SHA256 transform by Intel, and uses it at run time if SSE4 instructions are available, and use a fallback C++ implementation otherwise. Nearly every x86_64 CPU supports SSE4. The feature is only enabled when compiled with `--enable-experimental-asm`. In order to avoid build dependencies and other complications, the original Intel YASM code was translated to GCC extended asm syntax. This gives around a 50% speedup on the SHA256 benchmark for me. It is based on an earlier patch by @laanwj, though only includes a single assembly version (for now), and removes the YASM dependency. Tree-SHA512: d31c50695ceb45264291537b93c0d7497670be38edf021ca5402eaa7d4e1e0e1ae492326e28d4e93979d066168129e62d1825e0384b1b906d36f85d93dfcb43c --- configure.ac | 11 + src/Makefile.am | 4 + src/crypto/sha256.cpp | 236 +++--- src/crypto/sha256.h | 6 + src/crypto/sha256_sse4.cpp | 1506 ++++++++++++++++++++++++++++++++++++ src/init.cpp | 2 + src/test/test_dash.cpp | 2 + 7 files changed, 1679 insertions(+), 88 deletions(-) create mode 100644 src/crypto/sha256_sse4.cpp diff --git a/configure.ac b/configure.ac index c68d83079a78..a34f37cda753 100644 --- a/configure.ac +++ b/configure.ac @@ -178,6 +178,16 @@ AC_ARG_ENABLE([glibc-back-compat], [use_glibc_compat=$enableval], [use_glibc_compat=no]) +AC_ARG_ENABLE([experimental-asm], + [AS_HELP_STRING([--enable-experimental-asm], + [Enable experimental assembly routines (default is no)])], + [experimental_asm=$enableval], + [experimental_asm=no]) + +if test "x$experimental_asm" = xyes; then + AC_DEFINE(EXPERIMENTAL_ASM, 1, [Define this symbol to build in experimental assembly routines]) +fi + AC_ARG_WITH([system-univalue], [AS_HELP_STRING([--with-system-univalue], [Build with system UniValue (default is no)])], @@ -1218,6 +1228,7 @@ AM_CONDITIONAL([USE_LCOV],[test x$use_lcov = xyes]) AM_CONDITIONAL([GLIBC_BACK_COMPAT],[test x$use_glibc_compat = xyes]) AM_CONDITIONAL([HARDEN],[test x$use_hardening = xyes]) AM_CONDITIONAL([ENABLE_HWCRC32],[test x$enable_hwcrc32 = xyes]) +AM_CONDITIONAL([EXPERIMENTAL_ASM],[test x$experimental_asm = xyes]) AC_DEFINE(CLIENT_VERSION_MAJOR, _CLIENT_VERSION_MAJOR, [Major version]) AC_DEFINE(CLIENT_VERSION_MINOR, _CLIENT_VERSION_MINOR, [Minor version]) diff --git a/src/Makefile.am b/src/Makefile.am index e89720f9ba32..3cb1cba7373d 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -385,6 +385,10 @@ crypto_libdash_crypto_a_SOURCES = \ crypto/sha512.cpp \ crypto/sha512.h +if EXPERIMENTAL_ASM +crypto_libbitcoin_crypto_a_SOURCES += crypto/sha256_sse4.cpp +endif + # x11 crypto_libdash_crypto_a_SOURCES += \ crypto/blake.c \ diff --git a/src/crypto/sha256.cpp b/src/crypto/sha256.cpp index 5b9f00a0a2dc..15d6db90c246 100644 --- a/src/crypto/sha256.cpp +++ b/src/crypto/sha256.cpp @@ -3,10 +3,21 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include "crypto/sha256.h" - #include "crypto/common.h" +#include #include +#include + +#if defined(__x86_64__) || defined(__amd64__) +#if defined(EXPERIMENTAL_ASM) +#include +namespace sha256_sse4 +{ +void Transform(uint32_t* s, const unsigned char* chunk, size_t blocks); +} +#endif +#endif // Internal implementation code. namespace @@ -43,93 +54,142 @@ void inline Initialize(uint32_t* s) s[7] = 0x5be0cd19ul; } -/** Perform one SHA-256 transformation, processing a 64-byte chunk. */ -void Transform(uint32_t* s, const unsigned char* chunk) +/** Perform a number of SHA-256 transformations, processing 64-byte chunks. */ +void Transform(uint32_t* s, const unsigned char* chunk, size_t blocks) { - uint32_t a = s[0], b = s[1], c = s[2], d = s[3], e = s[4], f = s[5], g = s[6], h = s[7]; - uint32_t w0, w1, w2, w3, w4, w5, w6, w7, w8, w9, w10, w11, w12, w13, w14, w15; - - Round(a, b, c, d, e, f, g, h, 0x428a2f98, w0 = ReadBE32(chunk + 0)); - Round(h, a, b, c, d, e, f, g, 0x71374491, w1 = ReadBE32(chunk + 4)); - Round(g, h, a, b, c, d, e, f, 0xb5c0fbcf, w2 = ReadBE32(chunk + 8)); - Round(f, g, h, a, b, c, d, e, 0xe9b5dba5, w3 = ReadBE32(chunk + 12)); - Round(e, f, g, h, a, b, c, d, 0x3956c25b, w4 = ReadBE32(chunk + 16)); - Round(d, e, f, g, h, a, b, c, 0x59f111f1, w5 = ReadBE32(chunk + 20)); - Round(c, d, e, f, g, h, a, b, 0x923f82a4, w6 = ReadBE32(chunk + 24)); - Round(b, c, d, e, f, g, h, a, 0xab1c5ed5, w7 = ReadBE32(chunk + 28)); - Round(a, b, c, d, e, f, g, h, 0xd807aa98, w8 = ReadBE32(chunk + 32)); - Round(h, a, b, c, d, e, f, g, 0x12835b01, w9 = ReadBE32(chunk + 36)); - Round(g, h, a, b, c, d, e, f, 0x243185be, w10 = ReadBE32(chunk + 40)); - Round(f, g, h, a, b, c, d, e, 0x550c7dc3, w11 = ReadBE32(chunk + 44)); - Round(e, f, g, h, a, b, c, d, 0x72be5d74, w12 = ReadBE32(chunk + 48)); - Round(d, e, f, g, h, a, b, c, 0x80deb1fe, w13 = ReadBE32(chunk + 52)); - Round(c, d, e, f, g, h, a, b, 0x9bdc06a7, w14 = ReadBE32(chunk + 56)); - Round(b, c, d, e, f, g, h, a, 0xc19bf174, w15 = ReadBE32(chunk + 60)); - - Round(a, b, c, d, e, f, g, h, 0xe49b69c1, w0 += sigma1(w14) + w9 + sigma0(w1)); - Round(h, a, b, c, d, e, f, g, 0xefbe4786, w1 += sigma1(w15) + w10 + sigma0(w2)); - Round(g, h, a, b, c, d, e, f, 0x0fc19dc6, w2 += sigma1(w0) + w11 + sigma0(w3)); - Round(f, g, h, a, b, c, d, e, 0x240ca1cc, w3 += sigma1(w1) + w12 + sigma0(w4)); - Round(e, f, g, h, a, b, c, d, 0x2de92c6f, w4 += sigma1(w2) + w13 + sigma0(w5)); - Round(d, e, f, g, h, a, b, c, 0x4a7484aa, w5 += sigma1(w3) + w14 + sigma0(w6)); - Round(c, d, e, f, g, h, a, b, 0x5cb0a9dc, w6 += sigma1(w4) + w15 + sigma0(w7)); - Round(b, c, d, e, f, g, h, a, 0x76f988da, w7 += sigma1(w5) + w0 + sigma0(w8)); - Round(a, b, c, d, e, f, g, h, 0x983e5152, w8 += sigma1(w6) + w1 + sigma0(w9)); - Round(h, a, b, c, d, e, f, g, 0xa831c66d, w9 += sigma1(w7) + w2 + sigma0(w10)); - Round(g, h, a, b, c, d, e, f, 0xb00327c8, w10 += sigma1(w8) + w3 + sigma0(w11)); - Round(f, g, h, a, b, c, d, e, 0xbf597fc7, w11 += sigma1(w9) + w4 + sigma0(w12)); - Round(e, f, g, h, a, b, c, d, 0xc6e00bf3, w12 += sigma1(w10) + w5 + sigma0(w13)); - Round(d, e, f, g, h, a, b, c, 0xd5a79147, w13 += sigma1(w11) + w6 + sigma0(w14)); - Round(c, d, e, f, g, h, a, b, 0x06ca6351, w14 += sigma1(w12) + w7 + sigma0(w15)); - Round(b, c, d, e, f, g, h, a, 0x14292967, w15 += sigma1(w13) + w8 + sigma0(w0)); - - Round(a, b, c, d, e, f, g, h, 0x27b70a85, w0 += sigma1(w14) + w9 + sigma0(w1)); - Round(h, a, b, c, d, e, f, g, 0x2e1b2138, w1 += sigma1(w15) + w10 + sigma0(w2)); - Round(g, h, a, b, c, d, e, f, 0x4d2c6dfc, w2 += sigma1(w0) + w11 + sigma0(w3)); - Round(f, g, h, a, b, c, d, e, 0x53380d13, w3 += sigma1(w1) + w12 + sigma0(w4)); - Round(e, f, g, h, a, b, c, d, 0x650a7354, w4 += sigma1(w2) + w13 + sigma0(w5)); - Round(d, e, f, g, h, a, b, c, 0x766a0abb, w5 += sigma1(w3) + w14 + sigma0(w6)); - Round(c, d, e, f, g, h, a, b, 0x81c2c92e, w6 += sigma1(w4) + w15 + sigma0(w7)); - Round(b, c, d, e, f, g, h, a, 0x92722c85, w7 += sigma1(w5) + w0 + sigma0(w8)); - Round(a, b, c, d, e, f, g, h, 0xa2bfe8a1, w8 += sigma1(w6) + w1 + sigma0(w9)); - Round(h, a, b, c, d, e, f, g, 0xa81a664b, w9 += sigma1(w7) + w2 + sigma0(w10)); - Round(g, h, a, b, c, d, e, f, 0xc24b8b70, w10 += sigma1(w8) + w3 + sigma0(w11)); - Round(f, g, h, a, b, c, d, e, 0xc76c51a3, w11 += sigma1(w9) + w4 + sigma0(w12)); - Round(e, f, g, h, a, b, c, d, 0xd192e819, w12 += sigma1(w10) + w5 + sigma0(w13)); - Round(d, e, f, g, h, a, b, c, 0xd6990624, w13 += sigma1(w11) + w6 + sigma0(w14)); - Round(c, d, e, f, g, h, a, b, 0xf40e3585, w14 += sigma1(w12) + w7 + sigma0(w15)); - Round(b, c, d, e, f, g, h, a, 0x106aa070, w15 += sigma1(w13) + w8 + sigma0(w0)); - - Round(a, b, c, d, e, f, g, h, 0x19a4c116, w0 += sigma1(w14) + w9 + sigma0(w1)); - Round(h, a, b, c, d, e, f, g, 0x1e376c08, w1 += sigma1(w15) + w10 + sigma0(w2)); - Round(g, h, a, b, c, d, e, f, 0x2748774c, w2 += sigma1(w0) + w11 + sigma0(w3)); - Round(f, g, h, a, b, c, d, e, 0x34b0bcb5, w3 += sigma1(w1) + w12 + sigma0(w4)); - Round(e, f, g, h, a, b, c, d, 0x391c0cb3, w4 += sigma1(w2) + w13 + sigma0(w5)); - Round(d, e, f, g, h, a, b, c, 0x4ed8aa4a, w5 += sigma1(w3) + w14 + sigma0(w6)); - Round(c, d, e, f, g, h, a, b, 0x5b9cca4f, w6 += sigma1(w4) + w15 + sigma0(w7)); - Round(b, c, d, e, f, g, h, a, 0x682e6ff3, w7 += sigma1(w5) + w0 + sigma0(w8)); - Round(a, b, c, d, e, f, g, h, 0x748f82ee, w8 += sigma1(w6) + w1 + sigma0(w9)); - Round(h, a, b, c, d, e, f, g, 0x78a5636f, w9 += sigma1(w7) + w2 + sigma0(w10)); - Round(g, h, a, b, c, d, e, f, 0x84c87814, w10 += sigma1(w8) + w3 + sigma0(w11)); - Round(f, g, h, a, b, c, d, e, 0x8cc70208, w11 += sigma1(w9) + w4 + sigma0(w12)); - Round(e, f, g, h, a, b, c, d, 0x90befffa, w12 += sigma1(w10) + w5 + sigma0(w13)); - Round(d, e, f, g, h, a, b, c, 0xa4506ceb, w13 += sigma1(w11) + w6 + sigma0(w14)); - Round(c, d, e, f, g, h, a, b, 0xbef9a3f7, w14 + sigma1(w12) + w7 + sigma0(w15)); - Round(b, c, d, e, f, g, h, a, 0xc67178f2, w15 + sigma1(w13) + w8 + sigma0(w0)); - - s[0] += a; - s[1] += b; - s[2] += c; - s[3] += d; - s[4] += e; - s[5] += f; - s[6] += g; - s[7] += h; + while (blocks--) { + uint32_t a = s[0], b = s[1], c = s[2], d = s[3], e = s[4], f = s[5], g = s[6], h = s[7]; + uint32_t w0, w1, w2, w3, w4, w5, w6, w7, w8, w9, w10, w11, w12, w13, w14, w15; + + Round(a, b, c, d, e, f, g, h, 0x428a2f98, w0 = ReadBE32(chunk + 0)); + Round(h, a, b, c, d, e, f, g, 0x71374491, w1 = ReadBE32(chunk + 4)); + Round(g, h, a, b, c, d, e, f, 0xb5c0fbcf, w2 = ReadBE32(chunk + 8)); + Round(f, g, h, a, b, c, d, e, 0xe9b5dba5, w3 = ReadBE32(chunk + 12)); + Round(e, f, g, h, a, b, c, d, 0x3956c25b, w4 = ReadBE32(chunk + 16)); + Round(d, e, f, g, h, a, b, c, 0x59f111f1, w5 = ReadBE32(chunk + 20)); + Round(c, d, e, f, g, h, a, b, 0x923f82a4, w6 = ReadBE32(chunk + 24)); + Round(b, c, d, e, f, g, h, a, 0xab1c5ed5, w7 = ReadBE32(chunk + 28)); + Round(a, b, c, d, e, f, g, h, 0xd807aa98, w8 = ReadBE32(chunk + 32)); + Round(h, a, b, c, d, e, f, g, 0x12835b01, w9 = ReadBE32(chunk + 36)); + Round(g, h, a, b, c, d, e, f, 0x243185be, w10 = ReadBE32(chunk + 40)); + Round(f, g, h, a, b, c, d, e, 0x550c7dc3, w11 = ReadBE32(chunk + 44)); + Round(e, f, g, h, a, b, c, d, 0x72be5d74, w12 = ReadBE32(chunk + 48)); + Round(d, e, f, g, h, a, b, c, 0x80deb1fe, w13 = ReadBE32(chunk + 52)); + Round(c, d, e, f, g, h, a, b, 0x9bdc06a7, w14 = ReadBE32(chunk + 56)); + Round(b, c, d, e, f, g, h, a, 0xc19bf174, w15 = ReadBE32(chunk + 60)); + + Round(a, b, c, d, e, f, g, h, 0xe49b69c1, w0 += sigma1(w14) + w9 + sigma0(w1)); + Round(h, a, b, c, d, e, f, g, 0xefbe4786, w1 += sigma1(w15) + w10 + sigma0(w2)); + Round(g, h, a, b, c, d, e, f, 0x0fc19dc6, w2 += sigma1(w0) + w11 + sigma0(w3)); + Round(f, g, h, a, b, c, d, e, 0x240ca1cc, w3 += sigma1(w1) + w12 + sigma0(w4)); + Round(e, f, g, h, a, b, c, d, 0x2de92c6f, w4 += sigma1(w2) + w13 + sigma0(w5)); + Round(d, e, f, g, h, a, b, c, 0x4a7484aa, w5 += sigma1(w3) + w14 + sigma0(w6)); + Round(c, d, e, f, g, h, a, b, 0x5cb0a9dc, w6 += sigma1(w4) + w15 + sigma0(w7)); + Round(b, c, d, e, f, g, h, a, 0x76f988da, w7 += sigma1(w5) + w0 + sigma0(w8)); + Round(a, b, c, d, e, f, g, h, 0x983e5152, w8 += sigma1(w6) + w1 + sigma0(w9)); + Round(h, a, b, c, d, e, f, g, 0xa831c66d, w9 += sigma1(w7) + w2 + sigma0(w10)); + Round(g, h, a, b, c, d, e, f, 0xb00327c8, w10 += sigma1(w8) + w3 + sigma0(w11)); + Round(f, g, h, a, b, c, d, e, 0xbf597fc7, w11 += sigma1(w9) + w4 + sigma0(w12)); + Round(e, f, g, h, a, b, c, d, 0xc6e00bf3, w12 += sigma1(w10) + w5 + sigma0(w13)); + Round(d, e, f, g, h, a, b, c, 0xd5a79147, w13 += sigma1(w11) + w6 + sigma0(w14)); + Round(c, d, e, f, g, h, a, b, 0x06ca6351, w14 += sigma1(w12) + w7 + sigma0(w15)); + Round(b, c, d, e, f, g, h, a, 0x14292967, w15 += sigma1(w13) + w8 + sigma0(w0)); + + Round(a, b, c, d, e, f, g, h, 0x27b70a85, w0 += sigma1(w14) + w9 + sigma0(w1)); + Round(h, a, b, c, d, e, f, g, 0x2e1b2138, w1 += sigma1(w15) + w10 + sigma0(w2)); + Round(g, h, a, b, c, d, e, f, 0x4d2c6dfc, w2 += sigma1(w0) + w11 + sigma0(w3)); + Round(f, g, h, a, b, c, d, e, 0x53380d13, w3 += sigma1(w1) + w12 + sigma0(w4)); + Round(e, f, g, h, a, b, c, d, 0x650a7354, w4 += sigma1(w2) + w13 + sigma0(w5)); + Round(d, e, f, g, h, a, b, c, 0x766a0abb, w5 += sigma1(w3) + w14 + sigma0(w6)); + Round(c, d, e, f, g, h, a, b, 0x81c2c92e, w6 += sigma1(w4) + w15 + sigma0(w7)); + Round(b, c, d, e, f, g, h, a, 0x92722c85, w7 += sigma1(w5) + w0 + sigma0(w8)); + Round(a, b, c, d, e, f, g, h, 0xa2bfe8a1, w8 += sigma1(w6) + w1 + sigma0(w9)); + Round(h, a, b, c, d, e, f, g, 0xa81a664b, w9 += sigma1(w7) + w2 + sigma0(w10)); + Round(g, h, a, b, c, d, e, f, 0xc24b8b70, w10 += sigma1(w8) + w3 + sigma0(w11)); + Round(f, g, h, a, b, c, d, e, 0xc76c51a3, w11 += sigma1(w9) + w4 + sigma0(w12)); + Round(e, f, g, h, a, b, c, d, 0xd192e819, w12 += sigma1(w10) + w5 + sigma0(w13)); + Round(d, e, f, g, h, a, b, c, 0xd6990624, w13 += sigma1(w11) + w6 + sigma0(w14)); + Round(c, d, e, f, g, h, a, b, 0xf40e3585, w14 += sigma1(w12) + w7 + sigma0(w15)); + Round(b, c, d, e, f, g, h, a, 0x106aa070, w15 += sigma1(w13) + w8 + sigma0(w0)); + + Round(a, b, c, d, e, f, g, h, 0x19a4c116, w0 += sigma1(w14) + w9 + sigma0(w1)); + Round(h, a, b, c, d, e, f, g, 0x1e376c08, w1 += sigma1(w15) + w10 + sigma0(w2)); + Round(g, h, a, b, c, d, e, f, 0x2748774c, w2 += sigma1(w0) + w11 + sigma0(w3)); + Round(f, g, h, a, b, c, d, e, 0x34b0bcb5, w3 += sigma1(w1) + w12 + sigma0(w4)); + Round(e, f, g, h, a, b, c, d, 0x391c0cb3, w4 += sigma1(w2) + w13 + sigma0(w5)); + Round(d, e, f, g, h, a, b, c, 0x4ed8aa4a, w5 += sigma1(w3) + w14 + sigma0(w6)); + Round(c, d, e, f, g, h, a, b, 0x5b9cca4f, w6 += sigma1(w4) + w15 + sigma0(w7)); + Round(b, c, d, e, f, g, h, a, 0x682e6ff3, w7 += sigma1(w5) + w0 + sigma0(w8)); + Round(a, b, c, d, e, f, g, h, 0x748f82ee, w8 += sigma1(w6) + w1 + sigma0(w9)); + Round(h, a, b, c, d, e, f, g, 0x78a5636f, w9 += sigma1(w7) + w2 + sigma0(w10)); + Round(g, h, a, b, c, d, e, f, 0x84c87814, w10 += sigma1(w8) + w3 + sigma0(w11)); + Round(f, g, h, a, b, c, d, e, 0x8cc70208, w11 += sigma1(w9) + w4 + sigma0(w12)); + Round(e, f, g, h, a, b, c, d, 0x90befffa, w12 += sigma1(w10) + w5 + sigma0(w13)); + Round(d, e, f, g, h, a, b, c, 0xa4506ceb, w13 += sigma1(w11) + w6 + sigma0(w14)); + Round(c, d, e, f, g, h, a, b, 0xbef9a3f7, w14 + sigma1(w12) + w7 + sigma0(w15)); + Round(b, c, d, e, f, g, h, a, 0xc67178f2, w15 + sigma1(w13) + w8 + sigma0(w0)); + + s[0] += a; + s[1] += b; + s[2] += c; + s[3] += d; + s[4] += e; + s[5] += f; + s[6] += g; + s[7] += h; + chunk += 64; + } } } // namespace sha256 + +typedef void (*TransformType)(uint32_t*, const unsigned char*, size_t); + +bool SelfTest(TransformType tr) { + static const unsigned char in1[65] = {0, 0x80}; + static const unsigned char in2[129] = { + 0, + 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0 + }; + static const uint32_t init[8] = {0x6a09e667ul, 0xbb67ae85ul, 0x3c6ef372ul, 0xa54ff53aul, 0x510e527ful, 0x9b05688cul, 0x1f83d9abul, 0x5be0cd19ul}; + static const uint32_t out1[8] = {0xe3b0c442ul, 0x98fc1c14ul, 0x9afbf4c8ul, 0x996fb924ul, 0x27ae41e4ul, 0x649b934cul, 0xa495991bul, 0x7852b855ul}; + static const uint32_t out2[8] = {0xce4153b0ul, 0x147c2a86ul, 0x3ed4298eul, 0xe0676bc8ul, 0x79fc77a1ul, 0x2abe1f49ul, 0xb2b055dful, 0x1069523eul}; + uint32_t buf[8]; + memcpy(buf, init, sizeof(buf)); + // Process nothing, and check we remain in the initial state. + tr(buf, nullptr, 0); + if (memcmp(buf, init, sizeof(buf))) return false; + // Process the padded empty string (unaligned) + tr(buf, in1 + 1, 1); + if (memcmp(buf, out1, sizeof(buf))) return false; + // Process 64 spaces (unaligned) + memcpy(buf, init, sizeof(buf)); + tr(buf, in2 + 1, 2); + if (memcmp(buf, out2, sizeof(buf))) return false; + return true; +} + +TransformType Transform = sha256::Transform; + } // namespace +std::string SHA256AutoDetect() +{ +#if defined(EXPERIMENTAL_ASM) && (defined(__x86_64__) || defined(__amd64__)) + uint32_t eax, ebx, ecx, edx; + if (__get_cpuid(1, &eax, &ebx, &ecx, &edx) && (ecx >> 19) & 1) { + Transform = sha256_sse4::Transform; + assert(SelfTest(Transform)); + return "sse4"; + } +#endif + + assert(SelfTest(Transform)); + return "standard"; +} ////// SHA-256 @@ -147,14 +207,14 @@ CSHA256& CSHA256::Write(const unsigned char* data, size_t len) memcpy(buf + bufsize, data, 64 - bufsize); bytes += 64 - bufsize; data += 64 - bufsize; - sha256::Transform(s, buf); + Transform(s, buf, 1); bufsize = 0; } - while (end >= data + 64) { - // Process full chunks directly from the source. - sha256::Transform(s, data); - bytes += 64; - data += 64; + if (end - data >= 64) { + size_t blocks = (end - data) / 64; + Transform(s, data, blocks); + data += 64 * blocks; + bytes += 64 * blocks; } if (end > data) { // Fill the buffer with what remains. diff --git a/src/crypto/sha256.h b/src/crypto/sha256.h index 5b15b6a233b1..51608af9c7d1 100644 --- a/src/crypto/sha256.h +++ b/src/crypto/sha256.h @@ -7,6 +7,7 @@ #include #include +#include /** A hasher class for SHA-256. */ class CSHA256 @@ -25,4 +26,9 @@ class CSHA256 CSHA256& Reset(); }; +/** Autodetect the best available SHA256 implementation. + * Returns the name of the implementation. + */ +std::string SHA256AutoDetect(); + #endif // BITCOIN_CRYPTO_SHA256_H diff --git a/src/crypto/sha256_sse4.cpp b/src/crypto/sha256_sse4.cpp new file mode 100644 index 000000000000..89f529a3abcf --- /dev/null +++ b/src/crypto/sha256_sse4.cpp @@ -0,0 +1,1506 @@ +// Copyright (c) 2017 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// +// This is a translation to GCC extended asm syntax from YASM code by Intel +// (available at the bottom of this file). + +#include +#include + +#if defined(__x86_64__) || defined(__amd64__) + +namespace sha256_sse4 +{ +void Transform(uint32_t* s, const unsigned char* chunk, size_t blocks) +{ + static const uint32_t K256 alignas(16) [] = { + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, + 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, + 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, + 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, + 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, + 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, + 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, + 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, + 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, + 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, + 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, + 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, + 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2, + }; + static const uint32_t FLIP_MASK alignas(16) [] = {0x00010203, 0x04050607, 0x08090a0b, 0x0c0d0e0f}; + static const uint32_t SHUF_00BA alignas(16) [] = {0x03020100, 0x0b0a0908, 0xffffffff, 0xffffffff}; + static const uint32_t SHUF_DC00 alignas(16) [] = {0xffffffff, 0xffffffff, 0x03020100, 0x0b0a0908}; + uint32_t a, b, c, d, f, g, h, y0, y1, y2; + uint64_t tbl; + uint64_t inp_end, inp; + uint32_t xfer alignas(16) [4]; + + __asm__ __volatile__( + "shl $0x6,%2;" + "je Ldone_hash_%=;" + "add %1,%2;" + "mov %2,%14;" + "mov (%0),%3;" + "mov 0x4(%0),%4;" + "mov 0x8(%0),%5;" + "mov 0xc(%0),%6;" + "mov 0x10(%0),%k2;" + "mov 0x14(%0),%7;" + "mov 0x18(%0),%8;" + "mov 0x1c(%0),%9;" + "movdqa %18,%%xmm12;" + "movdqa %19,%%xmm10;" + "movdqa %20,%%xmm11;" + + "Lloop0_%=:" + "lea %17,%13;" + "movdqu (%1),%%xmm4;" + "pshufb %%xmm12,%%xmm4;" + "movdqu 0x10(%1),%%xmm5;" + "pshufb %%xmm12,%%xmm5;" + "movdqu 0x20(%1),%%xmm6;" + "pshufb %%xmm12,%%xmm6;" + "movdqu 0x30(%1),%%xmm7;" + "pshufb %%xmm12,%%xmm7;" + "mov %1,%15;" + "mov $3,%1;" + + "Lloop1_%=:" + "movdqa 0x0(%13),%%xmm9;" + "paddd %%xmm4,%%xmm9;" + "movdqa %%xmm9,%16;" + "movdqa %%xmm7,%%xmm0;" + "mov %k2,%10;" + "ror $0xe,%10;" + "mov %3,%11;" + "palignr $0x4,%%xmm6,%%xmm0;" + "ror $0x9,%11;" + "xor %k2,%10;" + "mov %7,%12;" + "ror $0x5,%10;" + "movdqa %%xmm5,%%xmm1;" + "xor %3,%11;" + "xor %8,%12;" + "paddd %%xmm4,%%xmm0;" + "xor %k2,%10;" + "and %k2,%12;" + "ror $0xb,%11;" + "palignr $0x4,%%xmm4,%%xmm1;" + "xor %3,%11;" + "ror $0x6,%10;" + "xor %8,%12;" + "movdqa %%xmm1,%%xmm2;" + "ror $0x2,%11;" + "add %10,%12;" + "add %16,%12;" + "movdqa %%xmm1,%%xmm3;" + "mov %3,%10;" + "add %12,%9;" + "mov %3,%12;" + "pslld $0x19,%%xmm1;" + "or %5,%10;" + "add %9,%6;" + "and %5,%12;" + "psrld $0x7,%%xmm2;" + "and %4,%10;" + "add %11,%9;" + "por %%xmm2,%%xmm1;" + "or %12,%10;" + "add %10,%9;" + "movdqa %%xmm3,%%xmm2;" + "mov %6,%10;" + "mov %9,%11;" + "movdqa %%xmm3,%%xmm8;" + "ror $0xe,%10;" + "xor %6,%10;" + "mov %k2,%12;" + "ror $0x9,%11;" + "pslld $0xe,%%xmm3;" + "xor %9,%11;" + "ror $0x5,%10;" + "xor %7,%12;" + "psrld $0x12,%%xmm2;" + "ror $0xb,%11;" + "xor %6,%10;" + "and %6,%12;" + "ror $0x6,%10;" + "pxor %%xmm3,%%xmm1;" + "xor %9,%11;" + "xor %7,%12;" + "psrld $0x3,%%xmm8;" + "add %10,%12;" + "add 4+%16,%12;" + "ror $0x2,%11;" + "pxor %%xmm2,%%xmm1;" + "mov %9,%10;" + "add %12,%8;" + "mov %9,%12;" + "pxor %%xmm8,%%xmm1;" + "or %4,%10;" + "add %8,%5;" + "and %4,%12;" + "pshufd $0xfa,%%xmm7,%%xmm2;" + "and %3,%10;" + "add %11,%8;" + "paddd %%xmm1,%%xmm0;" + "or %12,%10;" + "add %10,%8;" + "movdqa %%xmm2,%%xmm3;" + "mov %5,%10;" + "mov %8,%11;" + "ror $0xe,%10;" + "movdqa %%xmm2,%%xmm8;" + "xor %5,%10;" + "ror $0x9,%11;" + "mov %6,%12;" + "xor %8,%11;" + "ror $0x5,%10;" + "psrlq $0x11,%%xmm2;" + "xor %k2,%12;" + "psrlq $0x13,%%xmm3;" + "xor %5,%10;" + "and %5,%12;" + "psrld $0xa,%%xmm8;" + "ror $0xb,%11;" + "xor %8,%11;" + "xor %k2,%12;" + "ror $0x6,%10;" + "pxor %%xmm3,%%xmm2;" + "add %10,%12;" + "ror $0x2,%11;" + "add 8+%16,%12;" + "pxor %%xmm2,%%xmm8;" + "mov %8,%10;" + "add %12,%7;" + "mov %8,%12;" + "pshufb %%xmm10,%%xmm8;" + "or %3,%10;" + "add %7,%4;" + "and %3,%12;" + "paddd %%xmm8,%%xmm0;" + "and %9,%10;" + "add %11,%7;" + "pshufd $0x50,%%xmm0,%%xmm2;" + "or %12,%10;" + "add %10,%7;" + "movdqa %%xmm2,%%xmm3;" + "mov %4,%10;" + "ror $0xe,%10;" + "mov %7,%11;" + "movdqa %%xmm2,%%xmm4;" + "ror $0x9,%11;" + "xor %4,%10;" + "mov %5,%12;" + "ror $0x5,%10;" + "psrlq $0x11,%%xmm2;" + "xor %7,%11;" + "xor %6,%12;" + "psrlq $0x13,%%xmm3;" + "xor %4,%10;" + "and %4,%12;" + "ror $0xb,%11;" + "psrld $0xa,%%xmm4;" + "xor %7,%11;" + "ror $0x6,%10;" + "xor %6,%12;" + "pxor %%xmm3,%%xmm2;" + "ror $0x2,%11;" + "add %10,%12;" + "add 12+%16,%12;" + "pxor %%xmm2,%%xmm4;" + "mov %7,%10;" + "add %12,%k2;" + "mov %7,%12;" + "pshufb %%xmm11,%%xmm4;" + "or %9,%10;" + "add %k2,%3;" + "and %9,%12;" + "paddd %%xmm0,%%xmm4;" + "and %8,%10;" + "add %11,%k2;" + "or %12,%10;" + "add %10,%k2;" + "movdqa 0x10(%13),%%xmm9;" + "paddd %%xmm5,%%xmm9;" + "movdqa %%xmm9,%16;" + "movdqa %%xmm4,%%xmm0;" + "mov %3,%10;" + "ror $0xe,%10;" + "mov %k2,%11;" + "palignr $0x4,%%xmm7,%%xmm0;" + "ror $0x9,%11;" + "xor %3,%10;" + "mov %4,%12;" + "ror $0x5,%10;" + "movdqa %%xmm6,%%xmm1;" + "xor %k2,%11;" + "xor %5,%12;" + "paddd %%xmm5,%%xmm0;" + "xor %3,%10;" + "and %3,%12;" + "ror $0xb,%11;" + "palignr $0x4,%%xmm5,%%xmm1;" + "xor %k2,%11;" + "ror $0x6,%10;" + "xor %5,%12;" + "movdqa %%xmm1,%%xmm2;" + "ror $0x2,%11;" + "add %10,%12;" + "add %16,%12;" + "movdqa %%xmm1,%%xmm3;" + "mov %k2,%10;" + "add %12,%6;" + "mov %k2,%12;" + "pslld $0x19,%%xmm1;" + "or %8,%10;" + "add %6,%9;" + "and %8,%12;" + "psrld $0x7,%%xmm2;" + "and %7,%10;" + "add %11,%6;" + "por %%xmm2,%%xmm1;" + "or %12,%10;" + "add %10,%6;" + "movdqa %%xmm3,%%xmm2;" + "mov %9,%10;" + "mov %6,%11;" + "movdqa %%xmm3,%%xmm8;" + "ror $0xe,%10;" + "xor %9,%10;" + "mov %3,%12;" + "ror $0x9,%11;" + "pslld $0xe,%%xmm3;" + "xor %6,%11;" + "ror $0x5,%10;" + "xor %4,%12;" + "psrld $0x12,%%xmm2;" + "ror $0xb,%11;" + "xor %9,%10;" + "and %9,%12;" + "ror $0x6,%10;" + "pxor %%xmm3,%%xmm1;" + "xor %6,%11;" + "xor %4,%12;" + "psrld $0x3,%%xmm8;" + "add %10,%12;" + "add 4+%16,%12;" + "ror $0x2,%11;" + "pxor %%xmm2,%%xmm1;" + "mov %6,%10;" + "add %12,%5;" + "mov %6,%12;" + "pxor %%xmm8,%%xmm1;" + "or %7,%10;" + "add %5,%8;" + "and %7,%12;" + "pshufd $0xfa,%%xmm4,%%xmm2;" + "and %k2,%10;" + "add %11,%5;" + "paddd %%xmm1,%%xmm0;" + "or %12,%10;" + "add %10,%5;" + "movdqa %%xmm2,%%xmm3;" + "mov %8,%10;" + "mov %5,%11;" + "ror $0xe,%10;" + "movdqa %%xmm2,%%xmm8;" + "xor %8,%10;" + "ror $0x9,%11;" + "mov %9,%12;" + "xor %5,%11;" + "ror $0x5,%10;" + "psrlq $0x11,%%xmm2;" + "xor %3,%12;" + "psrlq $0x13,%%xmm3;" + "xor %8,%10;" + "and %8,%12;" + "psrld $0xa,%%xmm8;" + "ror $0xb,%11;" + "xor %5,%11;" + "xor %3,%12;" + "ror $0x6,%10;" + "pxor %%xmm3,%%xmm2;" + "add %10,%12;" + "ror $0x2,%11;" + "add 8+%16,%12;" + "pxor %%xmm2,%%xmm8;" + "mov %5,%10;" + "add %12,%4;" + "mov %5,%12;" + "pshufb %%xmm10,%%xmm8;" + "or %k2,%10;" + "add %4,%7;" + "and %k2,%12;" + "paddd %%xmm8,%%xmm0;" + "and %6,%10;" + "add %11,%4;" + "pshufd $0x50,%%xmm0,%%xmm2;" + "or %12,%10;" + "add %10,%4;" + "movdqa %%xmm2,%%xmm3;" + "mov %7,%10;" + "ror $0xe,%10;" + "mov %4,%11;" + "movdqa %%xmm2,%%xmm5;" + "ror $0x9,%11;" + "xor %7,%10;" + "mov %8,%12;" + "ror $0x5,%10;" + "psrlq $0x11,%%xmm2;" + "xor %4,%11;" + "xor %9,%12;" + "psrlq $0x13,%%xmm3;" + "xor %7,%10;" + "and %7,%12;" + "ror $0xb,%11;" + "psrld $0xa,%%xmm5;" + "xor %4,%11;" + "ror $0x6,%10;" + "xor %9,%12;" + "pxor %%xmm3,%%xmm2;" + "ror $0x2,%11;" + "add %10,%12;" + "add 12+%16,%12;" + "pxor %%xmm2,%%xmm5;" + "mov %4,%10;" + "add %12,%3;" + "mov %4,%12;" + "pshufb %%xmm11,%%xmm5;" + "or %6,%10;" + "add %3,%k2;" + "and %6,%12;" + "paddd %%xmm0,%%xmm5;" + "and %5,%10;" + "add %11,%3;" + "or %12,%10;" + "add %10,%3;" + "movdqa 0x20(%13),%%xmm9;" + "paddd %%xmm6,%%xmm9;" + "movdqa %%xmm9,%16;" + "movdqa %%xmm5,%%xmm0;" + "mov %k2,%10;" + "ror $0xe,%10;" + "mov %3,%11;" + "palignr $0x4,%%xmm4,%%xmm0;" + "ror $0x9,%11;" + "xor %k2,%10;" + "mov %7,%12;" + "ror $0x5,%10;" + "movdqa %%xmm7,%%xmm1;" + "xor %3,%11;" + "xor %8,%12;" + "paddd %%xmm6,%%xmm0;" + "xor %k2,%10;" + "and %k2,%12;" + "ror $0xb,%11;" + "palignr $0x4,%%xmm6,%%xmm1;" + "xor %3,%11;" + "ror $0x6,%10;" + "xor %8,%12;" + "movdqa %%xmm1,%%xmm2;" + "ror $0x2,%11;" + "add %10,%12;" + "add %16,%12;" + "movdqa %%xmm1,%%xmm3;" + "mov %3,%10;" + "add %12,%9;" + "mov %3,%12;" + "pslld $0x19,%%xmm1;" + "or %5,%10;" + "add %9,%6;" + "and %5,%12;" + "psrld $0x7,%%xmm2;" + "and %4,%10;" + "add %11,%9;" + "por %%xmm2,%%xmm1;" + "or %12,%10;" + "add %10,%9;" + "movdqa %%xmm3,%%xmm2;" + "mov %6,%10;" + "mov %9,%11;" + "movdqa %%xmm3,%%xmm8;" + "ror $0xe,%10;" + "xor %6,%10;" + "mov %k2,%12;" + "ror $0x9,%11;" + "pslld $0xe,%%xmm3;" + "xor %9,%11;" + "ror $0x5,%10;" + "xor %7,%12;" + "psrld $0x12,%%xmm2;" + "ror $0xb,%11;" + "xor %6,%10;" + "and %6,%12;" + "ror $0x6,%10;" + "pxor %%xmm3,%%xmm1;" + "xor %9,%11;" + "xor %7,%12;" + "psrld $0x3,%%xmm8;" + "add %10,%12;" + "add 4+%16,%12;" + "ror $0x2,%11;" + "pxor %%xmm2,%%xmm1;" + "mov %9,%10;" + "add %12,%8;" + "mov %9,%12;" + "pxor %%xmm8,%%xmm1;" + "or %4,%10;" + "add %8,%5;" + "and %4,%12;" + "pshufd $0xfa,%%xmm5,%%xmm2;" + "and %3,%10;" + "add %11,%8;" + "paddd %%xmm1,%%xmm0;" + "or %12,%10;" + "add %10,%8;" + "movdqa %%xmm2,%%xmm3;" + "mov %5,%10;" + "mov %8,%11;" + "ror $0xe,%10;" + "movdqa %%xmm2,%%xmm8;" + "xor %5,%10;" + "ror $0x9,%11;" + "mov %6,%12;" + "xor %8,%11;" + "ror $0x5,%10;" + "psrlq $0x11,%%xmm2;" + "xor %k2,%12;" + "psrlq $0x13,%%xmm3;" + "xor %5,%10;" + "and %5,%12;" + "psrld $0xa,%%xmm8;" + "ror $0xb,%11;" + "xor %8,%11;" + "xor %k2,%12;" + "ror $0x6,%10;" + "pxor %%xmm3,%%xmm2;" + "add %10,%12;" + "ror $0x2,%11;" + "add 8+%16,%12;" + "pxor %%xmm2,%%xmm8;" + "mov %8,%10;" + "add %12,%7;" + "mov %8,%12;" + "pshufb %%xmm10,%%xmm8;" + "or %3,%10;" + "add %7,%4;" + "and %3,%12;" + "paddd %%xmm8,%%xmm0;" + "and %9,%10;" + "add %11,%7;" + "pshufd $0x50,%%xmm0,%%xmm2;" + "or %12,%10;" + "add %10,%7;" + "movdqa %%xmm2,%%xmm3;" + "mov %4,%10;" + "ror $0xe,%10;" + "mov %7,%11;" + "movdqa %%xmm2,%%xmm6;" + "ror $0x9,%11;" + "xor %4,%10;" + "mov %5,%12;" + "ror $0x5,%10;" + "psrlq $0x11,%%xmm2;" + "xor %7,%11;" + "xor %6,%12;" + "psrlq $0x13,%%xmm3;" + "xor %4,%10;" + "and %4,%12;" + "ror $0xb,%11;" + "psrld $0xa,%%xmm6;" + "xor %7,%11;" + "ror $0x6,%10;" + "xor %6,%12;" + "pxor %%xmm3,%%xmm2;" + "ror $0x2,%11;" + "add %10,%12;" + "add 12+%16,%12;" + "pxor %%xmm2,%%xmm6;" + "mov %7,%10;" + "add %12,%k2;" + "mov %7,%12;" + "pshufb %%xmm11,%%xmm6;" + "or %9,%10;" + "add %k2,%3;" + "and %9,%12;" + "paddd %%xmm0,%%xmm6;" + "and %8,%10;" + "add %11,%k2;" + "or %12,%10;" + "add %10,%k2;" + "movdqa 0x30(%13),%%xmm9;" + "paddd %%xmm7,%%xmm9;" + "movdqa %%xmm9,%16;" + "add $0x40,%13;" + "movdqa %%xmm6,%%xmm0;" + "mov %3,%10;" + "ror $0xe,%10;" + "mov %k2,%11;" + "palignr $0x4,%%xmm5,%%xmm0;" + "ror $0x9,%11;" + "xor %3,%10;" + "mov %4,%12;" + "ror $0x5,%10;" + "movdqa %%xmm4,%%xmm1;" + "xor %k2,%11;" + "xor %5,%12;" + "paddd %%xmm7,%%xmm0;" + "xor %3,%10;" + "and %3,%12;" + "ror $0xb,%11;" + "palignr $0x4,%%xmm7,%%xmm1;" + "xor %k2,%11;" + "ror $0x6,%10;" + "xor %5,%12;" + "movdqa %%xmm1,%%xmm2;" + "ror $0x2,%11;" + "add %10,%12;" + "add %16,%12;" + "movdqa %%xmm1,%%xmm3;" + "mov %k2,%10;" + "add %12,%6;" + "mov %k2,%12;" + "pslld $0x19,%%xmm1;" + "or %8,%10;" + "add %6,%9;" + "and %8,%12;" + "psrld $0x7,%%xmm2;" + "and %7,%10;" + "add %11,%6;" + "por %%xmm2,%%xmm1;" + "or %12,%10;" + "add %10,%6;" + "movdqa %%xmm3,%%xmm2;" + "mov %9,%10;" + "mov %6,%11;" + "movdqa %%xmm3,%%xmm8;" + "ror $0xe,%10;" + "xor %9,%10;" + "mov %3,%12;" + "ror $0x9,%11;" + "pslld $0xe,%%xmm3;" + "xor %6,%11;" + "ror $0x5,%10;" + "xor %4,%12;" + "psrld $0x12,%%xmm2;" + "ror $0xb,%11;" + "xor %9,%10;" + "and %9,%12;" + "ror $0x6,%10;" + "pxor %%xmm3,%%xmm1;" + "xor %6,%11;" + "xor %4,%12;" + "psrld $0x3,%%xmm8;" + "add %10,%12;" + "add 4+%16,%12;" + "ror $0x2,%11;" + "pxor %%xmm2,%%xmm1;" + "mov %6,%10;" + "add %12,%5;" + "mov %6,%12;" + "pxor %%xmm8,%%xmm1;" + "or %7,%10;" + "add %5,%8;" + "and %7,%12;" + "pshufd $0xfa,%%xmm6,%%xmm2;" + "and %k2,%10;" + "add %11,%5;" + "paddd %%xmm1,%%xmm0;" + "or %12,%10;" + "add %10,%5;" + "movdqa %%xmm2,%%xmm3;" + "mov %8,%10;" + "mov %5,%11;" + "ror $0xe,%10;" + "movdqa %%xmm2,%%xmm8;" + "xor %8,%10;" + "ror $0x9,%11;" + "mov %9,%12;" + "xor %5,%11;" + "ror $0x5,%10;" + "psrlq $0x11,%%xmm2;" + "xor %3,%12;" + "psrlq $0x13,%%xmm3;" + "xor %8,%10;" + "and %8,%12;" + "psrld $0xa,%%xmm8;" + "ror $0xb,%11;" + "xor %5,%11;" + "xor %3,%12;" + "ror $0x6,%10;" + "pxor %%xmm3,%%xmm2;" + "add %10,%12;" + "ror $0x2,%11;" + "add 8+%16,%12;" + "pxor %%xmm2,%%xmm8;" + "mov %5,%10;" + "add %12,%4;" + "mov %5,%12;" + "pshufb %%xmm10,%%xmm8;" + "or %k2,%10;" + "add %4,%7;" + "and %k2,%12;" + "paddd %%xmm8,%%xmm0;" + "and %6,%10;" + "add %11,%4;" + "pshufd $0x50,%%xmm0,%%xmm2;" + "or %12,%10;" + "add %10,%4;" + "movdqa %%xmm2,%%xmm3;" + "mov %7,%10;" + "ror $0xe,%10;" + "mov %4,%11;" + "movdqa %%xmm2,%%xmm7;" + "ror $0x9,%11;" + "xor %7,%10;" + "mov %8,%12;" + "ror $0x5,%10;" + "psrlq $0x11,%%xmm2;" + "xor %4,%11;" + "xor %9,%12;" + "psrlq $0x13,%%xmm3;" + "xor %7,%10;" + "and %7,%12;" + "ror $0xb,%11;" + "psrld $0xa,%%xmm7;" + "xor %4,%11;" + "ror $0x6,%10;" + "xor %9,%12;" + "pxor %%xmm3,%%xmm2;" + "ror $0x2,%11;" + "add %10,%12;" + "add 12+%16,%12;" + "pxor %%xmm2,%%xmm7;" + "mov %4,%10;" + "add %12,%3;" + "mov %4,%12;" + "pshufb %%xmm11,%%xmm7;" + "or %6,%10;" + "add %3,%k2;" + "and %6,%12;" + "paddd %%xmm0,%%xmm7;" + "and %5,%10;" + "add %11,%3;" + "or %12,%10;" + "add %10,%3;" + "sub $0x1,%1;" + "jne Lloop1_%=;" + "mov $0x2,%1;" + + "Lloop2_%=:" + "paddd 0x0(%13),%%xmm4;" + "movdqa %%xmm4,%16;" + "mov %k2,%10;" + "ror $0xe,%10;" + "mov %3,%11;" + "xor %k2,%10;" + "ror $0x9,%11;" + "mov %7,%12;" + "xor %3,%11;" + "ror $0x5,%10;" + "xor %8,%12;" + "xor %k2,%10;" + "ror $0xb,%11;" + "and %k2,%12;" + "xor %3,%11;" + "ror $0x6,%10;" + "xor %8,%12;" + "add %10,%12;" + "ror $0x2,%11;" + "add %16,%12;" + "mov %3,%10;" + "add %12,%9;" + "mov %3,%12;" + "or %5,%10;" + "add %9,%6;" + "and %5,%12;" + "and %4,%10;" + "add %11,%9;" + "or %12,%10;" + "add %10,%9;" + "mov %6,%10;" + "ror $0xe,%10;" + "mov %9,%11;" + "xor %6,%10;" + "ror $0x9,%11;" + "mov %k2,%12;" + "xor %9,%11;" + "ror $0x5,%10;" + "xor %7,%12;" + "xor %6,%10;" + "ror $0xb,%11;" + "and %6,%12;" + "xor %9,%11;" + "ror $0x6,%10;" + "xor %7,%12;" + "add %10,%12;" + "ror $0x2,%11;" + "add 4+%16,%12;" + "mov %9,%10;" + "add %12,%8;" + "mov %9,%12;" + "or %4,%10;" + "add %8,%5;" + "and %4,%12;" + "and %3,%10;" + "add %11,%8;" + "or %12,%10;" + "add %10,%8;" + "mov %5,%10;" + "ror $0xe,%10;" + "mov %8,%11;" + "xor %5,%10;" + "ror $0x9,%11;" + "mov %6,%12;" + "xor %8,%11;" + "ror $0x5,%10;" + "xor %k2,%12;" + "xor %5,%10;" + "ror $0xb,%11;" + "and %5,%12;" + "xor %8,%11;" + "ror $0x6,%10;" + "xor %k2,%12;" + "add %10,%12;" + "ror $0x2,%11;" + "add 8+%16,%12;" + "mov %8,%10;" + "add %12,%7;" + "mov %8,%12;" + "or %3,%10;" + "add %7,%4;" + "and %3,%12;" + "and %9,%10;" + "add %11,%7;" + "or %12,%10;" + "add %10,%7;" + "mov %4,%10;" + "ror $0xe,%10;" + "mov %7,%11;" + "xor %4,%10;" + "ror $0x9,%11;" + "mov %5,%12;" + "xor %7,%11;" + "ror $0x5,%10;" + "xor %6,%12;" + "xor %4,%10;" + "ror $0xb,%11;" + "and %4,%12;" + "xor %7,%11;" + "ror $0x6,%10;" + "xor %6,%12;" + "add %10,%12;" + "ror $0x2,%11;" + "add 12+%16,%12;" + "mov %7,%10;" + "add %12,%k2;" + "mov %7,%12;" + "or %9,%10;" + "add %k2,%3;" + "and %9,%12;" + "and %8,%10;" + "add %11,%k2;" + "or %12,%10;" + "add %10,%k2;" + "paddd 0x10(%13),%%xmm5;" + "movdqa %%xmm5,%16;" + "add $0x20,%13;" + "mov %3,%10;" + "ror $0xe,%10;" + "mov %k2,%11;" + "xor %3,%10;" + "ror $0x9,%11;" + "mov %4,%12;" + "xor %k2,%11;" + "ror $0x5,%10;" + "xor %5,%12;" + "xor %3,%10;" + "ror $0xb,%11;" + "and %3,%12;" + "xor %k2,%11;" + "ror $0x6,%10;" + "xor %5,%12;" + "add %10,%12;" + "ror $0x2,%11;" + "add %16,%12;" + "mov %k2,%10;" + "add %12,%6;" + "mov %k2,%12;" + "or %8,%10;" + "add %6,%9;" + "and %8,%12;" + "and %7,%10;" + "add %11,%6;" + "or %12,%10;" + "add %10,%6;" + "mov %9,%10;" + "ror $0xe,%10;" + "mov %6,%11;" + "xor %9,%10;" + "ror $0x9,%11;" + "mov %3,%12;" + "xor %6,%11;" + "ror $0x5,%10;" + "xor %4,%12;" + "xor %9,%10;" + "ror $0xb,%11;" + "and %9,%12;" + "xor %6,%11;" + "ror $0x6,%10;" + "xor %4,%12;" + "add %10,%12;" + "ror $0x2,%11;" + "add 4+%16,%12;" + "mov %6,%10;" + "add %12,%5;" + "mov %6,%12;" + "or %7,%10;" + "add %5,%8;" + "and %7,%12;" + "and %k2,%10;" + "add %11,%5;" + "or %12,%10;" + "add %10,%5;" + "mov %8,%10;" + "ror $0xe,%10;" + "mov %5,%11;" + "xor %8,%10;" + "ror $0x9,%11;" + "mov %9,%12;" + "xor %5,%11;" + "ror $0x5,%10;" + "xor %3,%12;" + "xor %8,%10;" + "ror $0xb,%11;" + "and %8,%12;" + "xor %5,%11;" + "ror $0x6,%10;" + "xor %3,%12;" + "add %10,%12;" + "ror $0x2,%11;" + "add 8+%16,%12;" + "mov %5,%10;" + "add %12,%4;" + "mov %5,%12;" + "or %k2,%10;" + "add %4,%7;" + "and %k2,%12;" + "and %6,%10;" + "add %11,%4;" + "or %12,%10;" + "add %10,%4;" + "mov %7,%10;" + "ror $0xe,%10;" + "mov %4,%11;" + "xor %7,%10;" + "ror $0x9,%11;" + "mov %8,%12;" + "xor %4,%11;" + "ror $0x5,%10;" + "xor %9,%12;" + "xor %7,%10;" + "ror $0xb,%11;" + "and %7,%12;" + "xor %4,%11;" + "ror $0x6,%10;" + "xor %9,%12;" + "add %10,%12;" + "ror $0x2,%11;" + "add 12+%16,%12;" + "mov %4,%10;" + "add %12,%3;" + "mov %4,%12;" + "or %6,%10;" + "add %3,%k2;" + "and %6,%12;" + "and %5,%10;" + "add %11,%3;" + "or %12,%10;" + "add %10,%3;" + "movdqa %%xmm6,%%xmm4;" + "movdqa %%xmm7,%%xmm5;" + "sub $0x1,%1;" + "jne Lloop2_%=;" + "add (%0),%3;" + "mov %3,(%0);" + "add 0x4(%0),%4;" + "mov %4,0x4(%0);" + "add 0x8(%0),%5;" + "mov %5,0x8(%0);" + "add 0xc(%0),%6;" + "mov %6,0xc(%0);" + "add 0x10(%0),%k2;" + "mov %k2,0x10(%0);" + "add 0x14(%0),%7;" + "mov %7,0x14(%0);" + "add 0x18(%0),%8;" + "mov %8,0x18(%0);" + "add 0x1c(%0),%9;" + "mov %9,0x1c(%0);" + "mov %15,%1;" + "add $0x40,%1;" + "cmp %14,%1;" + "jne Lloop0_%=;" + + "Ldone_hash_%=:" + + : "+r"(s), "+r"(chunk), "+r"(blocks), "=r"(a), "=r"(b), "=r"(c), "=r"(d), /* e = chunk */ "=r"(f), "=r"(g), "=r"(h), "=r"(y0), "=r"(y1), "=r"(y2), "=r"(tbl), "+m"(inp_end), "+m"(inp), "+m"(xfer) + : "m"(K256), "m"(FLIP_MASK), "m"(SHUF_00BA), "m"(SHUF_DC00) + : "cc", "memory", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7", "xmm8", "xmm9", "xmm10", "xmm11", "xmm12" + ); +} +} + +/* +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; Copyright (c) 2012, Intel Corporation +; +; All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions are +; met: +; +; * Redistributions of source code must retain the above copyright +; notice, this list of conditions and the following disclaimer. +; +; * Redistributions in binary form must reproduce the above copyright +; notice, this list of conditions and the following disclaimer in the +; documentation and/or other materials provided with the +; distribution. +; +; * Neither the name of the Intel Corporation nor the names of its +; contributors may be used to endorse or promote products derived from +; this software without specific prior written permission. +; +; +; THIS SOFTWARE IS PROVIDED BY INTEL CORPORATION "AS IS" AND ANY +; EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +; PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL INTEL CORPORATION OR +; CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +; EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +; PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +; PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +; LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +; NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; Example YASM command lines: +; Windows: yasm -Xvc -f x64 -rnasm -pnasm -o sha256_sse4.obj -g cv8 sha256_sse4.asm +; Linux: yasm -f x64 -f elf64 -X gnu -g dwarf2 -D LINUX -o sha256_sse4.o sha256_sse4.asm +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; This code is described in an Intel White-Paper: +; "Fast SHA-256 Implementations on Intel Architecture Processors" +; +; To find it, surf to http://www.intel.com/p/en_US/embedded +; and search for that title. +; The paper is expected to be released roughly at the end of April, 2012 +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; This code schedules 1 blocks at a time, with 4 lanes per block +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +%define MOVDQ movdqu ;; assume buffers not aligned + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Define Macros + +; addm [mem], reg +; Add reg to mem using reg-mem add and store +%macro addm 2 + add %2, %1 + mov %1, %2 +%endm + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +; COPY_XMM_AND_BSWAP xmm, [mem], byte_flip_mask +; Load xmm with mem and byte swap each dword +%macro COPY_XMM_AND_BSWAP 3 + MOVDQ %1, %2 + pshufb %1, %3 +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +%define X0 xmm4 +%define X1 xmm5 +%define X2 xmm6 +%define X3 xmm7 + +%define XTMP0 xmm0 +%define XTMP1 xmm1 +%define XTMP2 xmm2 +%define XTMP3 xmm3 +%define XTMP4 xmm8 +%define XFER xmm9 + +%define SHUF_00BA xmm10 ; shuffle xBxA -> 00BA +%define SHUF_DC00 xmm11 ; shuffle xDxC -> DC00 +%define BYTE_FLIP_MASK xmm12 + +%ifdef LINUX +%define NUM_BLKS rdx ; 3rd arg +%define CTX rsi ; 2nd arg +%define INP rdi ; 1st arg + +%define SRND rdi ; clobbers INP +%define c ecx +%define d r8d +%define e edx +%else +%define NUM_BLKS r8 ; 3rd arg +%define CTX rdx ; 2nd arg +%define INP rcx ; 1st arg + +%define SRND rcx ; clobbers INP +%define c edi +%define d esi +%define e r8d + +%endif +%define TBL rbp +%define a eax +%define b ebx + +%define f r9d +%define g r10d +%define h r11d + +%define y0 r13d +%define y1 r14d +%define y2 r15d + + + +_INP_END_SIZE equ 8 +_INP_SIZE equ 8 +_XFER_SIZE equ 8 +%ifdef LINUX +_XMM_SAVE_SIZE equ 0 +%else +_XMM_SAVE_SIZE equ 7*16 +%endif +; STACK_SIZE plus pushes must be an odd multiple of 8 +_ALIGN_SIZE equ 8 + +_INP_END equ 0 +_INP equ _INP_END + _INP_END_SIZE +_XFER equ _INP + _INP_SIZE +_XMM_SAVE equ _XFER + _XFER_SIZE + _ALIGN_SIZE +STACK_SIZE equ _XMM_SAVE + _XMM_SAVE_SIZE + +; rotate_Xs +; Rotate values of symbols X0...X3 +%macro rotate_Xs 0 +%xdefine X_ X0 +%xdefine X0 X1 +%xdefine X1 X2 +%xdefine X2 X3 +%xdefine X3 X_ +%endm + +; ROTATE_ARGS +; Rotate values of symbols a...h +%macro ROTATE_ARGS 0 +%xdefine TMP_ h +%xdefine h g +%xdefine g f +%xdefine f e +%xdefine e d +%xdefine d c +%xdefine c b +%xdefine b a +%xdefine a TMP_ +%endm + +%macro FOUR_ROUNDS_AND_SCHED 0 + ;; compute s0 four at a time and s1 two at a time + ;; compute W[-16] + W[-7] 4 at a time + movdqa XTMP0, X3 + mov y0, e ; y0 = e + ror y0, (25-11) ; y0 = e >> (25-11) + mov y1, a ; y1 = a + palignr XTMP0, X2, 4 ; XTMP0 = W[-7] + ror y1, (22-13) ; y1 = a >> (22-13) + xor y0, e ; y0 = e ^ (e >> (25-11)) + mov y2, f ; y2 = f + ror y0, (11-6) ; y0 = (e >> (11-6)) ^ (e >> (25-6)) + movdqa XTMP1, X1 + xor y1, a ; y1 = a ^ (a >> (22-13) + xor y2, g ; y2 = f^g + paddd XTMP0, X0 ; XTMP0 = W[-7] + W[-16] + xor y0, e ; y0 = e ^ (e >> (11-6)) ^ (e >> (25-6)) + and y2, e ; y2 = (f^g)&e + ror y1, (13-2) ; y1 = (a >> (13-2)) ^ (a >> (22-2)) + ;; compute s0 + palignr XTMP1, X0, 4 ; XTMP1 = W[-15] + xor y1, a ; y1 = a ^ (a >> (13-2)) ^ (a >> (22-2)) + ror y0, 6 ; y0 = S1 = (e>>6) & (e>>11) ^ (e>>25) + xor y2, g ; y2 = CH = ((f^g)&e)^g + movdqa XTMP2, XTMP1 ; XTMP2 = W[-15] + ror y1, 2 ; y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22) + add y2, y0 ; y2 = S1 + CH + add y2, [rsp + _XFER + 0*4] ; y2 = k + w + S1 + CH + movdqa XTMP3, XTMP1 ; XTMP3 = W[-15] + mov y0, a ; y0 = a + add h, y2 ; h = h + S1 + CH + k + w + mov y2, a ; y2 = a + pslld XTMP1, (32-7) + or y0, c ; y0 = a|c + add d, h ; d = d + h + S1 + CH + k + w + and y2, c ; y2 = a&c + psrld XTMP2, 7 + and y0, b ; y0 = (a|c)&b + add h, y1 ; h = h + S1 + CH + k + w + S0 + por XTMP1, XTMP2 ; XTMP1 = W[-15] ror 7 + or y0, y2 ; y0 = MAJ = (a|c)&b)|(a&c) + add h, y0 ; h = h + S1 + CH + k + w + S0 + MAJ + +ROTATE_ARGS + movdqa XTMP2, XTMP3 ; XTMP2 = W[-15] + mov y0, e ; y0 = e + mov y1, a ; y1 = a + movdqa XTMP4, XTMP3 ; XTMP4 = W[-15] + ror y0, (25-11) ; y0 = e >> (25-11) + xor y0, e ; y0 = e ^ (e >> (25-11)) + mov y2, f ; y2 = f + ror y1, (22-13) ; y1 = a >> (22-13) + pslld XTMP3, (32-18) + xor y1, a ; y1 = a ^ (a >> (22-13) + ror y0, (11-6) ; y0 = (e >> (11-6)) ^ (e >> (25-6)) + xor y2, g ; y2 = f^g + psrld XTMP2, 18 + ror y1, (13-2) ; y1 = (a >> (13-2)) ^ (a >> (22-2)) + xor y0, e ; y0 = e ^ (e >> (11-6)) ^ (e >> (25-6)) + and y2, e ; y2 = (f^g)&e + ror y0, 6 ; y0 = S1 = (e>>6) & (e>>11) ^ (e>>25) + pxor XTMP1, XTMP3 + xor y1, a ; y1 = a ^ (a >> (13-2)) ^ (a >> (22-2)) + xor y2, g ; y2 = CH = ((f^g)&e)^g + psrld XTMP4, 3 ; XTMP4 = W[-15] >> 3 + add y2, y0 ; y2 = S1 + CH + add y2, [rsp + _XFER + 1*4] ; y2 = k + w + S1 + CH + ror y1, 2 ; y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22) + pxor XTMP1, XTMP2 ; XTMP1 = W[-15] ror 7 ^ W[-15] ror 18 + mov y0, a ; y0 = a + add h, y2 ; h = h + S1 + CH + k + w + mov y2, a ; y2 = a + pxor XTMP1, XTMP4 ; XTMP1 = s0 + or y0, c ; y0 = a|c + add d, h ; d = d + h + S1 + CH + k + w + and y2, c ; y2 = a&c + ;; compute low s1 + pshufd XTMP2, X3, 11111010b ; XTMP2 = W[-2] {BBAA} + and y0, b ; y0 = (a|c)&b + add h, y1 ; h = h + S1 + CH + k + w + S0 + paddd XTMP0, XTMP1 ; XTMP0 = W[-16] + W[-7] + s0 + or y0, y2 ; y0 = MAJ = (a|c)&b)|(a&c) + add h, y0 ; h = h + S1 + CH + k + w + S0 + MAJ + +ROTATE_ARGS + movdqa XTMP3, XTMP2 ; XTMP3 = W[-2] {BBAA} + mov y0, e ; y0 = e + mov y1, a ; y1 = a + ror y0, (25-11) ; y0 = e >> (25-11) + movdqa XTMP4, XTMP2 ; XTMP4 = W[-2] {BBAA} + xor y0, e ; y0 = e ^ (e >> (25-11)) + ror y1, (22-13) ; y1 = a >> (22-13) + mov y2, f ; y2 = f + xor y1, a ; y1 = a ^ (a >> (22-13) + ror y0, (11-6) ; y0 = (e >> (11-6)) ^ (e >> (25-6)) + psrlq XTMP2, 17 ; XTMP2 = W[-2] ror 17 {xBxA} + xor y2, g ; y2 = f^g + psrlq XTMP3, 19 ; XTMP3 = W[-2] ror 19 {xBxA} + xor y0, e ; y0 = e ^ (e >> (11-6)) ^ (e >> (25-6)) + and y2, e ; y2 = (f^g)&e + psrld XTMP4, 10 ; XTMP4 = W[-2] >> 10 {BBAA} + ror y1, (13-2) ; y1 = (a >> (13-2)) ^ (a >> (22-2)) + xor y1, a ; y1 = a ^ (a >> (13-2)) ^ (a >> (22-2)) + xor y2, g ; y2 = CH = ((f^g)&e)^g + ror y0, 6 ; y0 = S1 = (e>>6) & (e>>11) ^ (e>>25) + pxor XTMP2, XTMP3 + add y2, y0 ; y2 = S1 + CH + ror y1, 2 ; y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22) + add y2, [rsp + _XFER + 2*4] ; y2 = k + w + S1 + CH + pxor XTMP4, XTMP2 ; XTMP4 = s1 {xBxA} + mov y0, a ; y0 = a + add h, y2 ; h = h + S1 + CH + k + w + mov y2, a ; y2 = a + pshufb XTMP4, SHUF_00BA ; XTMP4 = s1 {00BA} + or y0, c ; y0 = a|c + add d, h ; d = d + h + S1 + CH + k + w + and y2, c ; y2 = a&c + paddd XTMP0, XTMP4 ; XTMP0 = {..., ..., W[1], W[0]} + and y0, b ; y0 = (a|c)&b + add h, y1 ; h = h + S1 + CH + k + w + S0 + ;; compute high s1 + pshufd XTMP2, XTMP0, 01010000b ; XTMP2 = W[-2] {DDCC} + or y0, y2 ; y0 = MAJ = (a|c)&b)|(a&c) + add h, y0 ; h = h + S1 + CH + k + w + S0 + MAJ + +ROTATE_ARGS + movdqa XTMP3, XTMP2 ; XTMP3 = W[-2] {DDCC} + mov y0, e ; y0 = e + ror y0, (25-11) ; y0 = e >> (25-11) + mov y1, a ; y1 = a + movdqa X0, XTMP2 ; X0 = W[-2] {DDCC} + ror y1, (22-13) ; y1 = a >> (22-13) + xor y0, e ; y0 = e ^ (e >> (25-11)) + mov y2, f ; y2 = f + ror y0, (11-6) ; y0 = (e >> (11-6)) ^ (e >> (25-6)) + psrlq XTMP2, 17 ; XTMP2 = W[-2] ror 17 {xDxC} + xor y1, a ; y1 = a ^ (a >> (22-13) + xor y2, g ; y2 = f^g + psrlq XTMP3, 19 ; XTMP3 = W[-2] ror 19 {xDxC} + xor y0, e ; y0 = e ^ (e >> (11-6)) ^ (e >> (25-6)) + and y2, e ; y2 = (f^g)&e + ror y1, (13-2) ; y1 = (a >> (13-2)) ^ (a >> (22-2)) + psrld X0, 10 ; X0 = W[-2] >> 10 {DDCC} + xor y1, a ; y1 = a ^ (a >> (13-2)) ^ (a >> (22-2)) + ror y0, 6 ; y0 = S1 = (e>>6) & (e>>11) ^ (e>>25) + xor y2, g ; y2 = CH = ((f^g)&e)^g + pxor XTMP2, XTMP3 + ror y1, 2 ; y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22) + add y2, y0 ; y2 = S1 + CH + add y2, [rsp + _XFER + 3*4] ; y2 = k + w + S1 + CH + pxor X0, XTMP2 ; X0 = s1 {xDxC} + mov y0, a ; y0 = a + add h, y2 ; h = h + S1 + CH + k + w + mov y2, a ; y2 = a + pshufb X0, SHUF_DC00 ; X0 = s1 {DC00} + or y0, c ; y0 = a|c + add d, h ; d = d + h + S1 + CH + k + w + and y2, c ; y2 = a&c + paddd X0, XTMP0 ; X0 = {W[3], W[2], W[1], W[0]} + and y0, b ; y0 = (a|c)&b + add h, y1 ; h = h + S1 + CH + k + w + S0 + or y0, y2 ; y0 = MAJ = (a|c)&b)|(a&c) + add h, y0 ; h = h + S1 + CH + k + w + S0 + MAJ + +ROTATE_ARGS +rotate_Xs +%endm + +;; input is [rsp + _XFER + %1 * 4] +%macro DO_ROUND 1 + mov y0, e ; y0 = e + ror y0, (25-11) ; y0 = e >> (25-11) + mov y1, a ; y1 = a + xor y0, e ; y0 = e ^ (e >> (25-11)) + ror y1, (22-13) ; y1 = a >> (22-13) + mov y2, f ; y2 = f + xor y1, a ; y1 = a ^ (a >> (22-13) + ror y0, (11-6) ; y0 = (e >> (11-6)) ^ (e >> (25-6)) + xor y2, g ; y2 = f^g + xor y0, e ; y0 = e ^ (e >> (11-6)) ^ (e >> (25-6)) + ror y1, (13-2) ; y1 = (a >> (13-2)) ^ (a >> (22-2)) + and y2, e ; y2 = (f^g)&e + xor y1, a ; y1 = a ^ (a >> (13-2)) ^ (a >> (22-2)) + ror y0, 6 ; y0 = S1 = (e>>6) & (e>>11) ^ (e>>25) + xor y2, g ; y2 = CH = ((f^g)&e)^g + add y2, y0 ; y2 = S1 + CH + ror y1, 2 ; y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22) + add y2, [rsp + _XFER + %1 * 4] ; y2 = k + w + S1 + CH + mov y0, a ; y0 = a + add h, y2 ; h = h + S1 + CH + k + w + mov y2, a ; y2 = a + or y0, c ; y0 = a|c + add d, h ; d = d + h + S1 + CH + k + w + and y2, c ; y2 = a&c + and y0, b ; y0 = (a|c)&b + add h, y1 ; h = h + S1 + CH + k + w + S0 + or y0, y2 ; y0 = MAJ = (a|c)&b)|(a&c) + add h, y0 ; h = h + S1 + CH + k + w + S0 + MAJ + ROTATE_ARGS +%endm + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; void sha256_sse4(void *input_data, UINT32 digest[8], UINT64 num_blks) +;; arg 1 : pointer to input data +;; arg 2 : pointer to digest +;; arg 3 : Num blocks +section .text +global sha256_sse4 +align 32 +sha256_sse4: + push rbx +%ifndef LINUX + push rsi + push rdi +%endif + push rbp + push r13 + push r14 + push r15 + + sub rsp,STACK_SIZE +%ifndef LINUX + movdqa [rsp + _XMM_SAVE + 0*16],xmm6 + movdqa [rsp + _XMM_SAVE + 1*16],xmm7 + movdqa [rsp + _XMM_SAVE + 2*16],xmm8 + movdqa [rsp + _XMM_SAVE + 3*16],xmm9 + movdqa [rsp + _XMM_SAVE + 4*16],xmm10 + movdqa [rsp + _XMM_SAVE + 5*16],xmm11 + movdqa [rsp + _XMM_SAVE + 6*16],xmm12 +%endif + + shl NUM_BLKS, 6 ; convert to bytes + jz done_hash + add NUM_BLKS, INP ; pointer to end of data + mov [rsp + _INP_END], NUM_BLKS + + ;; load initial digest + mov a,[4*0 + CTX] + mov b,[4*1 + CTX] + mov c,[4*2 + CTX] + mov d,[4*3 + CTX] + mov e,[4*4 + CTX] + mov f,[4*5 + CTX] + mov g,[4*6 + CTX] + mov h,[4*7 + CTX] + + movdqa BYTE_FLIP_MASK, [PSHUFFLE_BYTE_FLIP_MASK wrt rip] + movdqa SHUF_00BA, [_SHUF_00BA wrt rip] + movdqa SHUF_DC00, [_SHUF_DC00 wrt rip] + +loop0: + lea TBL,[K256 wrt rip] + + ;; byte swap first 16 dwords + COPY_XMM_AND_BSWAP X0, [INP + 0*16], BYTE_FLIP_MASK + COPY_XMM_AND_BSWAP X1, [INP + 1*16], BYTE_FLIP_MASK + COPY_XMM_AND_BSWAP X2, [INP + 2*16], BYTE_FLIP_MASK + COPY_XMM_AND_BSWAP X3, [INP + 3*16], BYTE_FLIP_MASK + + mov [rsp + _INP], INP + + ;; schedule 48 input dwords, by doing 3 rounds of 16 each + mov SRND, 3 +align 16 +loop1: + movdqa XFER, [TBL + 0*16] + paddd XFER, X0 + movdqa [rsp + _XFER], XFER + FOUR_ROUNDS_AND_SCHED + + movdqa XFER, [TBL + 1*16] + paddd XFER, X0 + movdqa [rsp + _XFER], XFER + FOUR_ROUNDS_AND_SCHED + + movdqa XFER, [TBL + 2*16] + paddd XFER, X0 + movdqa [rsp + _XFER], XFER + FOUR_ROUNDS_AND_SCHED + + movdqa XFER, [TBL + 3*16] + paddd XFER, X0 + movdqa [rsp + _XFER], XFER + add TBL, 4*16 + FOUR_ROUNDS_AND_SCHED + + sub SRND, 1 + jne loop1 + + mov SRND, 2 +loop2: + paddd X0, [TBL + 0*16] + movdqa [rsp + _XFER], X0 + DO_ROUND 0 + DO_ROUND 1 + DO_ROUND 2 + DO_ROUND 3 + paddd X1, [TBL + 1*16] + movdqa [rsp + _XFER], X1 + add TBL, 2*16 + DO_ROUND 0 + DO_ROUND 1 + DO_ROUND 2 + DO_ROUND 3 + + movdqa X0, X2 + movdqa X1, X3 + + sub SRND, 1 + jne loop2 + + addm [4*0 + CTX],a + addm [4*1 + CTX],b + addm [4*2 + CTX],c + addm [4*3 + CTX],d + addm [4*4 + CTX],e + addm [4*5 + CTX],f + addm [4*6 + CTX],g + addm [4*7 + CTX],h + + mov INP, [rsp + _INP] + add INP, 64 + cmp INP, [rsp + _INP_END] + jne loop0 + +done_hash: +%ifndef LINUX + movdqa xmm6,[rsp + _XMM_SAVE + 0*16] + movdqa xmm7,[rsp + _XMM_SAVE + 1*16] + movdqa xmm8,[rsp + _XMM_SAVE + 2*16] + movdqa xmm9,[rsp + _XMM_SAVE + 3*16] + movdqa xmm10,[rsp + _XMM_SAVE + 4*16] + movdqa xmm11,[rsp + _XMM_SAVE + 5*16] + movdqa xmm12,[rsp + _XMM_SAVE + 6*16] +%endif + + add rsp, STACK_SIZE + + pop r15 + pop r14 + pop r13 + pop rbp +%ifndef LINUX + pop rdi + pop rsi +%endif + pop rbx + + ret + + +section .data +align 64 +K256: + dd 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5 + dd 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5 + dd 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3 + dd 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174 + dd 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc + dd 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da + dd 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7 + dd 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967 + dd 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13 + dd 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85 + dd 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3 + dd 0xd192e819,0xd6990624,0xf40e3585,0x106aa070 + dd 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5 + dd 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3 + dd 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208 + dd 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 + +PSHUFFLE_BYTE_FLIP_MASK: ddq 0x0c0d0e0f08090a0b0405060700010203 + +; shuffle xBxA -> 00BA +_SHUF_00BA: ddq 0xFFFFFFFFFFFFFFFF0b0a090803020100 + +; shuffle xDxC -> DC00 +_SHUF_DC00: ddq 0x0b0a090803020100FFFFFFFFFFFFFFFF +*/ + +#endif diff --git a/src/init.cpp b/src/init.cpp index 35dd2cc58fbf..88baf57a94da 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -1441,6 +1441,8 @@ bool AppInitSanityChecks() // ********************************************************* Step 4: sanity checks // Initialize elliptic curve code + std::string sha256_algo = SHA256AutoDetect(); + LogPrintf("Using the '%s' SHA256 implementation\n", sha256_algo); RandomInit(); ECC_Start(); globalVerifyHandle.reset(new ECCVerifyHandle()); diff --git a/src/test/test_dash.cpp b/src/test/test_dash.cpp index a440feee6cef..ecc6941d4d34 100644 --- a/src/test/test_dash.cpp +++ b/src/test/test_dash.cpp @@ -7,6 +7,7 @@ #include "chainparams.h" #include "consensus/consensus.h" #include "consensus/validation.h" +#include "crypto/sha256.h" #include "fs.h" #include "key.h" #include "validation.h" @@ -38,6 +39,7 @@ extern void noui_connect(); BasicTestingSetup::BasicTestingSetup(const std::string& chainName) { + SHA256AutoDetect(); RandomInit(); ECC_Start(); BLSInit(); From b430366dd97e1511a33c788a89d3003557773aef Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Wed, 17 May 2017 13:02:50 -0700 Subject: [PATCH 22/36] Merge #10199: Better fee estimates 38bc1ec Make more json-like output from estimaterawfee (Alex Morcos) 2d2e170 Comments and improved documentation (Alex Morcos) ef589f8 minor cleanup: remove unnecessary variable (Alex Morcos) 3ee76d6 Introduce a scale factor (Alex Morcos) 5f1f0c6 Historical block span (Alex Morcos) aa19b8e Clean up fee estimate debug printing (Alex Morcos) 10f7cbd Track first recorded height (Alex Morcos) 3810e97 Rewrite estimateSmartFee (Alex Morcos) c7447ec Track failures in fee estimation. (Alex Morcos) 4186d3f Expose estimaterawfee (Alex Morcos) 2681153 minor refactor: explicitly track start of new bucket range and don't update curNearBucket on final loop. (Alex Morcos) 1ba43cc Make EstimateMedianVal smarter about small failures. (Alex Morcos) d3e30bc Refactor to update moving average on fly (Alex Morcos) e5007ba Change parameters for fee estimation and estimates on all 3 time horizons. (Alex Morcos) c0a273f Change file format for fee estimates. (Alex Morcos) Tree-SHA512: 186e7508d86a1f351bb656edcd84ee9091f5f2706331eda9ee29da9c8eb5bf67b8c1f2abf6662835560e7f613b1377099054f20767f41ddcdbc89c4f9e78946d --- src/init.cpp | 1 + src/policy/fees.cpp | 614 ++++++++++++++++++++++------- src/policy/fees.h | 182 ++++++--- src/rpc/client.cpp | 4 + src/rpc/mining.cpp | 105 ++++- src/test/policyestimator_tests.cpp | 47 +-- src/txmempool.cpp | 2 +- test/functional/smartfees.py | 4 +- 8 files changed, 733 insertions(+), 226 deletions(-) diff --git a/src/init.cpp b/src/init.cpp index 88baf57a94da..fa8aebda5906 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -270,6 +270,7 @@ void PrepareShutdown() if (fFeeEstimatesInitialized) { + ::feeEstimator.FlushUnconfirmed(::mempool); fs::path est_path = GetDataDir() / FEE_ESTIMATES_FILENAME; CAutoFile est_fileout(fsbridge::fopen(est_path, "wb"), SER_DISK, CLIENT_VERSION); if (!est_fileout.IsNull()) diff --git a/src/policy/fees.cpp b/src/policy/fees.cpp index 39efa28f263b..58c17792dceb 100644 --- a/src/policy/fees.cpp +++ b/src/policy/fees.cpp @@ -13,6 +13,8 @@ #include "txmempool.h" #include "util.h" +static constexpr double INF_FEERATE = 1e99; + /** * We will instantiate an instance of this class to track transactions that were * included in a block. We will lump transactions into a bucket according to their @@ -25,40 +27,43 @@ class TxConfirmStats { private: //Define the buckets we will group transactions into - std::vector buckets; // The upper-bound of the range for the bucket (inclusive) - std::map bucketMap; // Map of bucket upper-bound to index into all vectors by bucket + const std::vector& buckets; // The upper-bound of the range for the bucket (inclusive) + const std::map& bucketMap; // Map of bucket upper-bound to index into all vectors by bucket // For each bucket X: // Count the total # of txs in each bucket // Track the historical moving average of this total over blocks std::vector txCtAvg; - // and calculate the total for the current block to update the moving average - std::vector curBlockTxCt; // Count the total # of txs confirmed within Y blocks in each bucket // Track the historical moving average of theses totals over blocks - std::vector > confAvg; // confAvg[Y][X] - // and calculate the totals for the current block to update the moving averages - std::vector > curBlockConf; // curBlockConf[Y][X] + std::vector> confAvg; // confAvg[Y][X] + + // Track moving avg of txs which have been evicted from the mempool + // after failing to be confirmed within Y blocks + std::vector> failAvg; // failAvg[Y][X] // Sum the total feerate of all tx's in each bucket // Track the historical moving average of this total over blocks std::vector avg; - // and calculate the total for the current block to update the moving average - std::vector curBlockVal; // Combine the conf counts with tx counts to calculate the confirmation % for each Y,X // Combine the total value with the tx counts to calculate the avg feerate per bucket double decay; + // Resolution (# of blocks) with which confirmations are tracked + unsigned int scale; + // Mempool counts of outstanding transactions // For each bucket X, track the number of transactions in the mempool // that are unconfirmed for each possible confirmation value Y std::vector > unconfTxs; //unconfTxs[Y][X] - // transactions still unconfirmed after MAX_CONFIRMS for each bucket + // transactions still unconfirmed after GetMaxConfirms for each bucket std::vector oldUnconfTxs; + void resizeInMemoryCounters(size_t newbuckets); + public: /** * Create new TxConfirmStats. This is called by BlockPolicyEstimator's @@ -67,9 +72,10 @@ class TxConfirmStats * @param maxPeriods max number of periods to track * @param decay how much to decay the historical moving average per block */ - TxConfirmStats(const std::vector& defaultBuckets, unsigned int maxConfirms, double decay); + TxConfirmStats(const std::vector& defaultBuckets, const std::map& defaultBucketMap, + unsigned int maxPeriods, double decay, unsigned int scale); - /** Clear the state of the curBlock variables to start counting for the new block */ + /** Roll the circular buffer for unconfirmed txs*/ void ClearCurrent(unsigned int nBlockHeight); /** @@ -85,7 +91,7 @@ class TxConfirmStats /** Remove a transaction from mempool tracking stats*/ void removeTx(unsigned int entryHeight, unsigned int nBestSeenHeight, - unsigned int bucketIndex); + unsigned int bucketIndex, bool inBlock); /** Update our estimates by decaying our historical moving average and updating with the data gathered from the current block */ @@ -103,10 +109,11 @@ class TxConfirmStats * @param nBlockHeight the current block height */ double EstimateMedianVal(int confTarget, double sufficientTxVal, - double minSuccess, bool requireGreater, unsigned int nBlockHeight) const; + double minSuccess, bool requireGreater, unsigned int nBlockHeight, + EstimationResult *result = nullptr) const; /** Return the max number of confirms we're tracking */ - unsigned int GetMaxConfirms() const { return confAvg.size(); } + unsigned int GetMaxConfirms() const { return scale * confAvg.size(); } /** Write state of estimation data to a file*/ void Write(CAutoFile& fileout) const; @@ -115,44 +122,47 @@ class TxConfirmStats * Read saved state of estimation data from a file and replace all internal data structures and * variables with this state. */ - void Read(CAutoFile& filein); + void Read(CAutoFile& filein, int nFileVersion, size_t numBuckets); }; TxConfirmStats::TxConfirmStats(const std::vector& defaultBuckets, - unsigned int maxConfirms, double _decay) + const std::map& defaultBucketMap, + unsigned int maxPeriods, double _decay, unsigned int _scale) + : buckets(defaultBuckets), bucketMap(defaultBucketMap) { decay = _decay; - for (unsigned int i = 0; i < defaultBuckets.size(); i++) { - buckets.push_back(defaultBuckets[i]); - bucketMap[defaultBuckets[i]] = i; - } - confAvg.resize(maxConfirms); - curBlockConf.resize(maxConfirms); - unconfTxs.resize(maxConfirms); - for (unsigned int i = 0; i < maxConfirms; i++) { + scale = _scale; + confAvg.resize(maxPeriods); + for (unsigned int i = 0; i < maxPeriods; i++) { confAvg[i].resize(buckets.size()); - curBlockConf[i].resize(buckets.size()); - unconfTxs[i].resize(buckets.size()); + } + failAvg.resize(maxPeriods); + for (unsigned int i = 0; i < maxPeriods; i++) { + failAvg[i].resize(buckets.size()); } - oldUnconfTxs.resize(buckets.size()); - curBlockTxCt.resize(buckets.size()); txCtAvg.resize(buckets.size()); - curBlockVal.resize(buckets.size()); avg.resize(buckets.size()); + + resizeInMemoryCounters(buckets.size()); } -// Zero out the data for the current block +void TxConfirmStats::resizeInMemoryCounters(size_t newbuckets) { + // newbuckets must be passed in because the buckets referred to during Read have not been updated yet. + unconfTxs.resize(GetMaxConfirms()); + for (unsigned int i = 0; i < unconfTxs.size(); i++) { + unconfTxs[i].resize(newbuckets); + } + oldUnconfTxs.resize(newbuckets); +} + +// Roll the unconfirmed txs circular buffer void TxConfirmStats::ClearCurrent(unsigned int nBlockHeight) { for (unsigned int j = 0; j < buckets.size(); j++) { oldUnconfTxs[j] += unconfTxs[nBlockHeight%unconfTxs.size()][j]; unconfTxs[nBlockHeight%unconfTxs.size()][j] = 0; - for (unsigned int i = 0; i < curBlockConf.size(); i++) - curBlockConf[i][j] = 0; - curBlockTxCt[j] = 0; - curBlockVal[j] = 0; } } @@ -162,33 +172,38 @@ void TxConfirmStats::Record(int blocksToConfirm, double val) // blocksToConfirm is 1-based if (blocksToConfirm < 1) return; + int periodsToConfirm = (blocksToConfirm + scale - 1)/scale; unsigned int bucketindex = bucketMap.lower_bound(val)->second; - for (size_t i = blocksToConfirm; i <= curBlockConf.size(); i++) { - curBlockConf[i - 1][bucketindex]++; + for (size_t i = periodsToConfirm; i <= confAvg.size(); i++) { + confAvg[i - 1][bucketindex]++; } - curBlockTxCt[bucketindex]++; - curBlockVal[bucketindex] += val; + txCtAvg[bucketindex]++; + avg[bucketindex] += val; } void TxConfirmStats::UpdateMovingAverages() { for (unsigned int j = 0; j < buckets.size(); j++) { for (unsigned int i = 0; i < confAvg.size(); i++) - confAvg[i][j] = confAvg[i][j] * decay + curBlockConf[i][j]; - avg[j] = avg[j] * decay + curBlockVal[j]; - txCtAvg[j] = txCtAvg[j] * decay + curBlockTxCt[j]; + confAvg[i][j] = confAvg[i][j] * decay; + for (unsigned int i = 0; i < failAvg.size(); i++) + failAvg[i][j] = failAvg[i][j] * decay; + avg[j] = avg[j] * decay; + txCtAvg[j] = txCtAvg[j] * decay; } } // returns -1 on error conditions double TxConfirmStats::EstimateMedianVal(int confTarget, double sufficientTxVal, double successBreakPoint, bool requireGreater, - unsigned int nBlockHeight) const + unsigned int nBlockHeight, EstimationResult *result) const { // Counters for a bucket (or range of buckets) double nConf = 0; // Number of tx's confirmed within the confTarget double totalNum = 0; // Total number of tx's that were ever confirmed int extraNum = 0; // Number of tx's still in mempool for confTarget or longer + double failNum = 0; // Number of tx's that were never confirmed but removed from the mempool after confTarget + int periodTarget = (confTarget + scale - 1)/scale; int maxbucketindex = buckets.size() - 1; @@ -211,12 +226,21 @@ double TxConfirmStats::EstimateMedianVal(int confTarget, double sufficientTxVal, bool foundAnswer = false; unsigned int bins = unconfTxs.size(); + bool newBucketRange = true; + bool passing = true; + EstimatorBucket passBucket; + EstimatorBucket failBucket; // Start counting from highest(default) or lowest feerate transactions for (int bucket = startbucket; bucket >= 0 && bucket <= maxbucketindex; bucket += step) { + if (newBucketRange) { + curNearBucket = bucket; + newBucketRange = false; + } curFarBucket = bucket; - nConf += confAvg[confTarget - 1][bucket]; + nConf += confAvg[periodTarget - 1][bucket]; totalNum += txCtAvg[bucket]; + failNum += failAvg[periodTarget - 1][bucket]; for (unsigned int confct = confTarget; confct < GetMaxConfirms(); confct++) extraNum += unconfTxs[(nBlockHeight - confct)%bins][bucket]; extraNum += oldUnconfTxs[bucket]; @@ -225,24 +249,41 @@ double TxConfirmStats::EstimateMedianVal(int confTarget, double sufficientTxVal, // (Only count the confirmed data points, so that each confirmation count // will be looking at the same amount of data and same bucket breaks) if (totalNum >= sufficientTxVal / (1 - decay)) { - double curPct = nConf / (totalNum + extraNum); + double curPct = nConf / (totalNum + failNum + extraNum); // Check to see if we are no longer getting confirmed at the success rate - if (requireGreater && curPct < successBreakPoint) - break; - if (!requireGreater && curPct > successBreakPoint) - break; - + if ((requireGreater && curPct < successBreakPoint) || (!requireGreater && curPct > successBreakPoint)) { + if (passing == true) { + // First time we hit a failure record the failed bucket + unsigned int failMinBucket = std::min(curNearBucket, curFarBucket); + unsigned int failMaxBucket = std::max(curNearBucket, curFarBucket); + failBucket.start = failMinBucket ? buckets[failMinBucket - 1] : 0; + failBucket.end = buckets[failMaxBucket]; + failBucket.withinTarget = nConf; + failBucket.totalConfirmed = totalNum; + failBucket.inMempool = extraNum; + failBucket.leftMempool = failNum; + passing = false; + } + continue; + } // Otherwise update the cumulative stats, and the bucket variables // and reset the counters else { + failBucket = EstimatorBucket(); // Reset any failed bucket, currently passing foundAnswer = true; + passing = true; + passBucket.withinTarget = nConf; nConf = 0; + passBucket.totalConfirmed = totalNum; totalNum = 0; + passBucket.inMempool = extraNum; + passBucket.leftMempool = failNum; + failNum = 0; extraNum = 0; bestNearBucket = curNearBucket; bestFarBucket = curFarBucket; - curNearBucket = bucket + step; + newBucketRange = true; } } } @@ -254,8 +295,8 @@ double TxConfirmStats::EstimateMedianVal(int confTarget, double sufficientTxVal, // Find the bucket with the median transaction and then report the average feerate from that bucket // This is a compromise between finding the median which we can't since we don't save all tx's // and reporting the average which is less accurate - unsigned int minBucket = bestNearBucket < bestFarBucket ? bestNearBucket : bestFarBucket; - unsigned int maxBucket = bestNearBucket > bestFarBucket ? bestNearBucket : bestFarBucket; + unsigned int minBucket = std::min(bestNearBucket, bestFarBucket); + unsigned int maxBucket = std::max(bestNearBucket, bestFarBucket); for (unsigned int j = minBucket; j <= maxBucket; j++) { txSum += txCtAvg[j]; } @@ -269,83 +310,109 @@ double TxConfirmStats::EstimateMedianVal(int confTarget, double sufficientTxVal, break; } } + + passBucket.start = minBucket ? buckets[minBucket-1] : 0; + passBucket.end = buckets[maxBucket]; } - LogPrint(BCLog::ESTIMATEFEE, "%3d: For conf success %s %4.2f need feerate %s: %12.5g from buckets %8g - %8g Cur Bucket stats %6.2f%% %8.1f/(%.1f+%d mempool)\n", - confTarget, requireGreater ? ">" : "<", successBreakPoint, - requireGreater ? ">" : "<", median, buckets[minBucket], buckets[maxBucket], - 100 * nConf / (totalNum + extraNum), nConf, totalNum, extraNum); + // If we were passing until we reached last few buckets with insufficient data, then report those as failed + if (passing && !newBucketRange) { + unsigned int failMinBucket = std::min(curNearBucket, curFarBucket); + unsigned int failMaxBucket = std::max(curNearBucket, curFarBucket); + failBucket.start = failMinBucket ? buckets[failMinBucket - 1] : 0; + failBucket.end = buckets[failMaxBucket]; + failBucket.withinTarget = nConf; + failBucket.totalConfirmed = totalNum; + failBucket.inMempool = extraNum; + failBucket.leftMempool = failNum; + } + LogPrint(BCLog::ESTIMATEFEE, "FeeEst: %d %s%.0f%% decay %.5f: feerate: %g from (%g - %g) %.2f%% %.1f/(%.1f %d mem %.1f out) Fail: (%g - %g) %.2f%% %.1f/(%.1f %d mem %.1f out)\n", + confTarget, requireGreater ? ">" : "<", 100.0 * successBreakPoint, decay, + median, passBucket.start, passBucket.end, + 100 * passBucket.withinTarget / (passBucket.totalConfirmed + passBucket.inMempool + passBucket.leftMempool), + passBucket.withinTarget, passBucket.totalConfirmed, passBucket.inMempool, passBucket.leftMempool, + failBucket.start, failBucket.end, + 100 * failBucket.withinTarget / (failBucket.totalConfirmed + failBucket.inMempool + failBucket.leftMempool), + failBucket.withinTarget, failBucket.totalConfirmed, failBucket.inMempool, failBucket.leftMempool); + + + if (result) { + result->pass = passBucket; + result->fail = failBucket; + result->decay = decay; + result->scale = scale; + } return median; } void TxConfirmStats::Write(CAutoFile& fileout) const { fileout << decay; - fileout << buckets; + fileout << scale; fileout << avg; fileout << txCtAvg; fileout << confAvg; + fileout << failAvg; } -void TxConfirmStats::Read(CAutoFile& filein) +void TxConfirmStats::Read(CAutoFile& filein, int nFileVersion, size_t numBuckets) { - // Read data file into temporary variables and do some very basic sanity checking - std::vector fileBuckets; - std::vector fileAvg; - std::vector > fileConfAvg; - std::vector fileTxCtAvg; - double fileDecay; - size_t maxConfirms; - size_t numBuckets; - - filein >> fileDecay; - if (fileDecay <= 0 || fileDecay >= 1) - throw std::runtime_error("Corrupt estimates file. Decay must be between 0 and 1 (non-inclusive)"); - filein >> fileBuckets; - numBuckets = fileBuckets.size(); - if (numBuckets <= 1 || numBuckets > 1000) - throw std::runtime_error("Corrupt estimates file. Must have between 2 and 1000 feerate buckets"); - filein >> fileAvg; - if (fileAvg.size() != numBuckets) + // Read data file and do some very basic sanity checking + // buckets and bucketMap are not updated yet, so don't access them + // If there is a read failure, we'll just discard this entire object anyway + size_t maxConfirms, maxPeriods; + + // The current version will store the decay with each individual TxConfirmStats and also keep a scale factor + if (nFileVersion >= 149900) { + filein >> decay; + if (decay <= 0 || decay >= 1) { + throw std::runtime_error("Corrupt estimates file. Decay must be between 0 and 1 (non-inclusive)"); + } + filein >> scale; + } + + filein >> avg; + if (avg.size() != numBuckets) { throw std::runtime_error("Corrupt estimates file. Mismatch in feerate average bucket count"); - filein >> fileTxCtAvg; - if (fileTxCtAvg.size() != numBuckets) + } + filein >> txCtAvg; + if (txCtAvg.size() != numBuckets) { throw std::runtime_error("Corrupt estimates file. Mismatch in tx count bucket count"); - filein >> fileConfAvg; - maxConfirms = fileConfAvg.size(); - if (maxConfirms <= 0 || maxConfirms > 6 * 24 * 7) // one week - throw std::runtime_error("Corrupt estimates file. Must maintain estimates for between 1 and 1008 (one week) confirms"); - for (unsigned int i = 0; i < maxConfirms; i++) { - if (fileConfAvg[i].size() != numBuckets) - throw std::runtime_error("Corrupt estimates file. Mismatch in feerate conf average bucket count"); } - // Now that we've processed the entire feerate estimate data file and not - // thrown any errors, we can copy it to our data structures - decay = fileDecay; - buckets = fileBuckets; - avg = fileAvg; - confAvg = fileConfAvg; - txCtAvg = fileTxCtAvg; - bucketMap.clear(); + filein >> confAvg; + maxPeriods = confAvg.size(); + maxConfirms = scale * maxPeriods; - // Resize the current block variables which aren't stored in the data file - // to match the number of confirms and buckets - curBlockConf.resize(maxConfirms); - for (unsigned int i = 0; i < maxConfirms; i++) { - curBlockConf[i].resize(buckets.size()); + if (maxConfirms <= 0 || maxConfirms > 6 * 24 * 7) { // one week + throw std::runtime_error("Corrupt estimates file. Must maintain estimates for between 1 and 1008 (one week) confirms"); + } + for (unsigned int i = 0; i < maxPeriods; i++) { + if (confAvg[i].size() != numBuckets) { + throw std::runtime_error("Corrupt estimates file. Mismatch in feerate conf average bucket count"); + } } - curBlockTxCt.resize(buckets.size()); - curBlockVal.resize(buckets.size()); - unconfTxs.resize(maxConfirms); - for (unsigned int i = 0; i < maxConfirms; i++) { - unconfTxs[i].resize(buckets.size()); + if (nFileVersion >= 149900) { + filein >> failAvg; + if (maxPeriods != failAvg.size()) { + throw std::runtime_error("Corrupt estimates file. Mismatch in confirms tracked for failures"); + } + for (unsigned int i = 0; i < maxPeriods; i++) { + if (failAvg[i].size() != numBuckets) { + throw std::runtime_error("Corrupt estimates file. Mismatch in one of failure average bucket counts"); + } + } + } else { + failAvg.resize(confAvg.size()); + for (unsigned int i = 0; i < failAvg.size(); i++) { + failAvg[i].resize(numBuckets); + } } - oldUnconfTxs.resize(buckets.size()); - for (unsigned int i = 0; i < buckets.size(); i++) - bucketMap[buckets[i]] = i; + // Resize the current block variables which aren't stored in the data file + // to match the number of confirms and buckets + resizeInMemoryCounters(numBuckets); LogPrint(BCLog::ESTIMATEFEE, "Reading estimates: %u buckets counting confirms up to %u blocks\n", numBuckets, maxConfirms); @@ -359,7 +426,7 @@ unsigned int TxConfirmStats::NewTx(unsigned int nBlockHeight, double val) return bucketindex; } -void TxConfirmStats::removeTx(unsigned int entryHeight, unsigned int nBestSeenHeight, unsigned int bucketindex) +void TxConfirmStats::removeTx(unsigned int entryHeight, unsigned int nBestSeenHeight, unsigned int bucketindex, bool inBlock) { //nBestSeenHeight is not updated yet for the new block int blocksAgo = nBestSeenHeight - entryHeight; @@ -387,6 +454,12 @@ void TxConfirmStats::removeTx(unsigned int entryHeight, unsigned int nBestSeenHe blockIndex, bucketindex); } } + if (!inBlock && (unsigned int)blocksAgo >= scale) { // Only counts as a failure if not confirmed for entire period + unsigned int periodsAgo = blocksAgo / scale; + for (size_t i = 0; i < periodsAgo && i < failAvg.size(); i++) { + failAvg[i][bucketindex]++; + } + } } // This function is called from CTxMemPool::removeUnchecked to ensure @@ -394,12 +467,14 @@ void TxConfirmStats::removeTx(unsigned int entryHeight, unsigned int nBestSeenHe // tracked. Txs that were part of a block have already been removed in // processBlockTx to ensure they are never double tracked, but it is // of no harm to try to remove them again. -bool CBlockPolicyEstimator::removeTx(uint256 hash) +bool CBlockPolicyEstimator::removeTx(uint256 hash, bool inBlock) { LOCK(cs_feeEstimator); std::map::iterator pos = mapMemPoolTxs.find(hash); if (pos != mapMemPoolTxs.end()) { - feeStats->removeTx(pos->second.blockHeight, nBestSeenHeight, pos->second.bucketIndex); + feeStats->removeTx(pos->second.blockHeight, nBestSeenHeight, pos->second.bucketIndex, inBlock); + shortStats->removeTx(pos->second.blockHeight, nBestSeenHeight, pos->second.bucketIndex, inBlock); + longStats->removeTx(pos->second.blockHeight, nBestSeenHeight, pos->second.bucketIndex, inBlock); mapMemPoolTxs.erase(hash); return true; } else { @@ -408,21 +483,28 @@ bool CBlockPolicyEstimator::removeTx(uint256 hash) } CBlockPolicyEstimator::CBlockPolicyEstimator() - : nBestSeenHeight(0), trackedTxs(0), untrackedTxs(0) + : nBestSeenHeight(0), firstRecordedHeight(0), historicalFirst(0), historicalBest(0), trackedTxs(0), untrackedTxs(0) { static_assert(MIN_BUCKET_FEERATE > 0, "Min feerate must be nonzero"); - minTrackedFee = CFeeRate(MIN_BUCKET_FEERATE); - std::vector vfeelist; - for (double bucketBoundary = minTrackedFee.GetFeePerK(); bucketBoundary <= MAX_BUCKET_FEERATE; bucketBoundary *= FEE_SPACING) { - vfeelist.push_back(bucketBoundary); + size_t bucketIndex = 0; + for (double bucketBoundary = MIN_BUCKET_FEERATE; bucketBoundary <= MAX_BUCKET_FEERATE; bucketBoundary *= FEE_SPACING, bucketIndex++) { + buckets.push_back(bucketBoundary); + bucketMap[bucketBoundary] = bucketIndex; } - vfeelist.push_back(INF_FEERATE); - feeStats = new TxConfirmStats(vfeelist, MAX_BLOCK_CONFIRMS, DEFAULT_DECAY); + buckets.push_back(INF_FEERATE); + bucketMap[INF_FEERATE] = bucketIndex; + assert(bucketMap.size() == buckets.size()); + + feeStats = new TxConfirmStats(buckets, bucketMap, MED_BLOCK_PERIODS, MED_DECAY, MED_SCALE); + shortStats = new TxConfirmStats(buckets, bucketMap, SHORT_BLOCK_PERIODS, SHORT_DECAY, SHORT_SCALE); + longStats = new TxConfirmStats(buckets, bucketMap, LONG_BLOCK_PERIODS, LONG_DECAY, LONG_SCALE); } CBlockPolicyEstimator::~CBlockPolicyEstimator() { delete feeStats; + delete shortStats; + delete longStats; } void CBlockPolicyEstimator::processTransaction(const CTxMemPoolEntry& entry, bool validFeeEstimate) @@ -455,12 +537,17 @@ void CBlockPolicyEstimator::processTransaction(const CTxMemPoolEntry& entry, boo CFeeRate feeRate(entry.GetFee(), entry.GetTxSize()); mapMemPoolTxs[hash].blockHeight = txHeight; - mapMemPoolTxs[hash].bucketIndex = feeStats->NewTx(txHeight, (double)feeRate.GetFeePerK()); + unsigned int bucketIndex = feeStats->NewTx(txHeight, (double)feeRate.GetFeePerK()); + mapMemPoolTxs[hash].bucketIndex = bucketIndex; + unsigned int bucketIndex2 = shortStats->NewTx(txHeight, (double)feeRate.GetFeePerK()); + assert(bucketIndex == bucketIndex2); + unsigned int bucketIndex3 = longStats->NewTx(txHeight, (double)feeRate.GetFeePerK()); + assert(bucketIndex == bucketIndex3); } bool CBlockPolicyEstimator::processBlockTx(unsigned int nBlockHeight, const CTxMemPoolEntry* entry) { - if (!removeTx(entry->GetTx().GetHash())) { + if (!removeTx(entry->GetTx().GetHash(), true)) { // This transaction wasn't being tracked for fee estimation return false; } @@ -480,6 +567,8 @@ bool CBlockPolicyEstimator::processBlockTx(unsigned int nBlockHeight, const CTxM CFeeRate feeRate(entry->GetFee(), entry->GetTxSize()); feeStats->Record(blocksToConfirm, (double)feeRate.GetFeePerK()); + shortStats->Record(blocksToConfirm, (double)feeRate.GetFeePerK()); + longStats->Record(blocksToConfirm, (double)feeRate.GetFeePerK()); return true; } @@ -501,21 +590,32 @@ void CBlockPolicyEstimator::processBlock(unsigned int nBlockHeight, // of unconfirmed txs to remove from tracking. nBestSeenHeight = nBlockHeight; - // Clear the current block state and update unconfirmed circular buffer + // Update unconfirmed circular buffer feeStats->ClearCurrent(nBlockHeight); + shortStats->ClearCurrent(nBlockHeight); + longStats->ClearCurrent(nBlockHeight); + + // Decay all exponential averages + feeStats->UpdateMovingAverages(); + shortStats->UpdateMovingAverages(); + longStats->UpdateMovingAverages(); unsigned int countedTxs = 0; - // Repopulate the current block states + // Update averages with data points from current block for (const auto& entry : entries) { if (processBlockTx(nBlockHeight, entry)) countedTxs++; } - // Update all exponential averages with the current block state - feeStats->UpdateMovingAverages(); + if (firstRecordedHeight == 0 && countedTxs > 0) { + firstRecordedHeight = nBestSeenHeight; + LogPrint(BCLog::ESTIMATEFEE, "Blockpolicy first recorded height %u\n", firstRecordedHeight); + } - LogPrint(BCLog::ESTIMATEFEE, "Blockpolicy after updating estimates for %u of %u txs in block, since last block %u of %u tracked, new mempool map size %u\n", - countedTxs, entries.size(), trackedTxs, trackedTxs + untrackedTxs, mapMemPoolTxs.size()); + + LogPrint(BCLog::ESTIMATEFEE, "Blockpolicy estimates updated by %u of %u block txs, since last block %u of %u tracked, mempool map size %u, max target %u from %s\n", + countedTxs, entries.size(), trackedTxs, trackedTxs + untrackedTxs, mapMemPoolTxs.size(), + MaxUsableEstimate(), HistoricalBlockSpan() > BlockSpan() ? "historical" : "current"); trackedTxs = 0; untrackedTxs = 0; @@ -523,13 +623,44 @@ void CBlockPolicyEstimator::processBlock(unsigned int nBlockHeight, CFeeRate CBlockPolicyEstimator::estimateFee(int confTarget) const { + // It's not possible to get reasonable estimates for confTarget of 1 + if (confTarget <= 1) + return CFeeRate(0); + + return estimateRawFee(confTarget, DOUBLE_SUCCESS_PCT, FeeEstimateHorizon::MED_HALFLIFE); +} + +CFeeRate CBlockPolicyEstimator::estimateRawFee(int confTarget, double successThreshold, FeeEstimateHorizon horizon, EstimationResult* result) const +{ + TxConfirmStats* stats; + double sufficientTxs = SUFFICIENT_FEETXS; + switch (horizon) { + case FeeEstimateHorizon::SHORT_HALFLIFE: { + stats = shortStats; + sufficientTxs = SUFFICIENT_TXS_SHORT; + break; + } + case FeeEstimateHorizon::MED_HALFLIFE: { + stats = feeStats; + break; + } + case FeeEstimateHorizon::LONG_HALFLIFE: { + stats = longStats; + break; + } + default: { + return CFeeRate(0); + } + } + LOCK(cs_feeEstimator); // Return failure if trying to analyze a target we're not tracking - // It's not possible to get reasonable estimates for confTarget of 1 - if (confTarget <= 1 || (unsigned int)confTarget > feeStats->GetMaxConfirms()) + if (confTarget <= 0 || (unsigned int)confTarget > stats->GetMaxConfirms()) + return CFeeRate(0); + if (successThreshold > 1) return CFeeRate(0); - double median = feeStats->EstimateMedianVal(confTarget, SUFFICIENT_FEETXS, MIN_SUCCESS_PCT, true, nBestSeenHeight); + double median = stats->EstimateMedianVal(confTarget, sufficientTxs, successThreshold, true, nBestSeenHeight, result); if (median < 0) return CFeeRate(0); @@ -537,31 +668,148 @@ CFeeRate CBlockPolicyEstimator::estimateFee(int confTarget) const return CFeeRate(median); } -CFeeRate CBlockPolicyEstimator::estimateSmartFee(int confTarget, int *answerFoundAtTarget, const CTxMemPool& pool) const +unsigned int CBlockPolicyEstimator::BlockSpan() const +{ + if (firstRecordedHeight == 0) return 0; + assert(nBestSeenHeight >= firstRecordedHeight); + + return nBestSeenHeight - firstRecordedHeight; +} + +unsigned int CBlockPolicyEstimator::HistoricalBlockSpan() const +{ + if (historicalFirst == 0) return 0; + assert(historicalBest >= historicalFirst); + + if (nBestSeenHeight - historicalBest > OLDEST_ESTIMATE_HISTORY) return 0; + + return historicalBest - historicalFirst; +} + +unsigned int CBlockPolicyEstimator::MaxUsableEstimate() const +{ + // Block spans are divided by 2 to make sure there are enough potential failing data points for the estimate + return std::min(longStats->GetMaxConfirms(), std::max(BlockSpan(), HistoricalBlockSpan()) / 2); +} + +/** Return a fee estimate at the required successThreshold from the shortest + * time horizon which tracks confirmations up to the desired target. If + * checkShorterHorizon is requested, also allow short time horizon estimates + * for a lower target to reduce the given answer */ +double CBlockPolicyEstimator::estimateCombinedFee(unsigned int confTarget, double successThreshold, bool checkShorterHorizon) const +{ + double estimate = -1; + if (confTarget >= 1 && confTarget <= longStats->GetMaxConfirms()) { + // Find estimate from shortest time horizon possible + if (confTarget <= shortStats->GetMaxConfirms()) { // short horizon + estimate = shortStats->EstimateMedianVal(confTarget, SUFFICIENT_TXS_SHORT, successThreshold, true, nBestSeenHeight); + } + else if (confTarget <= feeStats->GetMaxConfirms()) { // medium horizon + estimate = feeStats->EstimateMedianVal(confTarget, SUFFICIENT_FEETXS, successThreshold, true, nBestSeenHeight); + } + else { // long horizon + estimate = longStats->EstimateMedianVal(confTarget, SUFFICIENT_FEETXS, successThreshold, true, nBestSeenHeight); + } + if (checkShorterHorizon) { + // If a lower confTarget from a more recent horizon returns a lower answer use it. + if (confTarget > feeStats->GetMaxConfirms()) { + double medMax = feeStats->EstimateMedianVal(feeStats->GetMaxConfirms(), SUFFICIENT_FEETXS, successThreshold, true, nBestSeenHeight); + if (medMax > 0 && (estimate == -1 || medMax < estimate)) + estimate = medMax; + } + if (confTarget > shortStats->GetMaxConfirms()) { + double shortMax = shortStats->EstimateMedianVal(shortStats->GetMaxConfirms(), SUFFICIENT_TXS_SHORT, successThreshold, true, nBestSeenHeight); + if (shortMax > 0 && (estimate == -1 || shortMax < estimate)) + estimate = shortMax; + } + } + } + return estimate; +} + +/** Ensure that for a conservative estimate, the DOUBLE_SUCCESS_PCT is also met + * at 2 * target for any longer time horizons. + */ +double CBlockPolicyEstimator::estimateConservativeFee(unsigned int doubleTarget) const +{ + double estimate = -1; + if (doubleTarget <= shortStats->GetMaxConfirms()) { + estimate = feeStats->EstimateMedianVal(doubleTarget, SUFFICIENT_FEETXS, DOUBLE_SUCCESS_PCT, true, nBestSeenHeight); + } + if (doubleTarget <= feeStats->GetMaxConfirms()) { + double longEstimate = longStats->EstimateMedianVal(doubleTarget, SUFFICIENT_FEETXS, DOUBLE_SUCCESS_PCT, true, nBestSeenHeight); + if (longEstimate > estimate) { + estimate = longEstimate; + } + } + return estimate; +} + +/** estimateSmartFee returns the max of the feerates calculated with a 60% + * threshold required at target / 2, an 85% threshold required at target and a + * 95% threshold required at 2 * target. Each calculation is performed at the + * shortest time horizon which tracks the required target. Conservative + * estimates, however, required the 95% threshold at 2 * target be met for any + * longer time horizons also. + */ +CFeeRate CBlockPolicyEstimator::estimateSmartFee(int confTarget, int *answerFoundAtTarget, const CTxMemPool& pool, bool conservative) const { if (answerFoundAtTarget) *answerFoundAtTarget = confTarget; double median = -1; - { LOCK(cs_feeEstimator); // Return failure if trying to analyze a target we're not tracking - if (confTarget <= 0 || (unsigned int)confTarget > feeStats->GetMaxConfirms()) + if (confTarget <= 0 || (unsigned int)confTarget > longStats->GetMaxConfirms()) return CFeeRate(0); // It's not possible to get reasonable estimates for confTarget of 1 if (confTarget == 1) confTarget = 2; - while (median < 0 && (unsigned int)confTarget <= feeStats->GetMaxConfirms()) { - median = feeStats->EstimateMedianVal(confTarget++, SUFFICIENT_FEETXS, MIN_SUCCESS_PCT, true, nBestSeenHeight); + unsigned int maxUsableEstimate = MaxUsableEstimate(); + if (maxUsableEstimate <= 1) + return CFeeRate(0); + + if ((unsigned int)confTarget > maxUsableEstimate) { + confTarget = maxUsableEstimate; + } + + assert(confTarget > 0); //estimateCombinedFee and estimateConservativeFee take unsigned ints + + /** true is passed to estimateCombined fee for target/2 and target so + * that we check the max confirms for shorter time horizons as well. + * This is necessary to preserve monotonically increasing estimates. + * For non-conservative estimates we do the same thing for 2*target, but + * for conservative estimates we want to skip these shorter horizons + * checks for 2*target becuase we are taking the max over all time + * horizons so we already have monotonically increasing estimates and + * the purpose of conservative estimates is not to let short term + * fluctuations lower our estimates by too much. + */ + double halfEst = estimateCombinedFee(confTarget/2, HALF_SUCCESS_PCT, true); + double actualEst = estimateCombinedFee(confTarget, SUCCESS_PCT, true); + double doubleEst = estimateCombinedFee(2 * confTarget, DOUBLE_SUCCESS_PCT, !conservative); + median = halfEst; + if (actualEst > median) { + median = actualEst; + } + if (doubleEst > median) { + median = doubleEst; + } + + if (conservative || median == -1) { + double consEst = estimateConservativeFee(2 * confTarget); + if (consEst > median) { + median = consEst; + } } } // Must unlock cs_feeEstimator before taking mempool locks if (answerFoundAtTarget) - *answerFoundAtTarget = confTarget - 1; + *answerFoundAtTarget = confTarget; // If mempool is limiting txs , return at least the min feerate from the mempool CAmount minPoolFee = pool.GetMinFee(gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000).GetFeePerK(); @@ -574,6 +822,7 @@ CFeeRate CBlockPolicyEstimator::estimateSmartFee(int confTarget, int *answerFoun return CFeeRate(median); } + bool CBlockPolicyEstimator::Write(CAutoFile& fileout) const { try { @@ -581,7 +830,16 @@ bool CBlockPolicyEstimator::Write(CAutoFile& fileout) const fileout << 140100; // version required to read: 0.14.1 or later fileout << CLIENT_VERSION; // version that wrote the file fileout << nBestSeenHeight; + if (BlockSpan() > HistoricalBlockSpan()/2) { + fileout << firstRecordedHeight << nBestSeenHeight; + } + else { + fileout << historicalFirst << historicalBest; + } + fileout << buckets; feeStats->Write(fileout); + shortStats->Write(fileout); + longStats->Write(fileout); } catch (const std::exception&) { LogPrintf("CBlockPolicyEstimator::Write(): unable to write policy estimator data (non-fatal)\n"); @@ -594,19 +852,95 @@ bool CBlockPolicyEstimator::Read(CAutoFile& filein) { try { LOCK(cs_feeEstimator); - int nVersionRequired, nVersionThatWrote, nFileBestSeenHeight; + int nVersionRequired, nVersionThatWrote; + unsigned int nFileBestSeenHeight, nFileHistoricalFirst, nFileHistoricalBest; filein >> nVersionRequired >> nVersionThatWrote; if (nVersionRequired > CLIENT_VERSION) return error("CBlockPolicyEstimator::Read(): up-version (%d) fee estimate file", nVersionRequired); + + // Read fee estimates file into temporary variables so existing data + // structures aren't corrupted if there is an exception. filein >> nFileBestSeenHeight; - feeStats->Read(filein); - nBestSeenHeight = nFileBestSeenHeight; - // if nVersionThatWrote < 120300 then another TxConfirmStats (for priority) follows but can be ignored. + + if (nVersionThatWrote < 149900) { + // Read the old fee estimates file for temporary use, but then discard. Will start collecting data from scratch. + // decay is stored before buckets in old versions, so pre-read decay and pass into TxConfirmStats constructor + double tempDecay; + filein >> tempDecay; + if (tempDecay <= 0 || tempDecay >= 1) + throw std::runtime_error("Corrupt estimates file. Decay must be between 0 and 1 (non-inclusive)"); + + std::vector tempBuckets; + filein >> tempBuckets; + size_t tempNum = tempBuckets.size(); + if (tempNum <= 1 || tempNum > 1000) + throw std::runtime_error("Corrupt estimates file. Must have between 2 and 1000 feerate buckets"); + + std::map tempMap; + + std::unique_ptr tempFeeStats(new TxConfirmStats(tempBuckets, tempMap, MED_BLOCK_PERIODS, tempDecay, 1)); + tempFeeStats->Read(filein, nVersionThatWrote, tempNum); + // if nVersionThatWrote < 120300 then another TxConfirmStats (for priority) follows but can be ignored. + + tempMap.clear(); + for (unsigned int i = 0; i < tempBuckets.size(); i++) { + tempMap[tempBuckets[i]] = i; + } + } + else { // nVersionThatWrote >= 149900 + filein >> nFileHistoricalFirst >> nFileHistoricalBest; + if (nFileHistoricalFirst > nFileHistoricalBest || nFileHistoricalBest > nFileBestSeenHeight) { + throw std::runtime_error("Corrupt estimates file. Historical block range for estimates is invalid"); + } + std::vector fileBuckets; + filein >> fileBuckets; + size_t numBuckets = fileBuckets.size(); + if (numBuckets <= 1 || numBuckets > 1000) + throw std::runtime_error("Corrupt estimates file. Must have between 2 and 1000 feerate buckets"); + + std::unique_ptr fileFeeStats(new TxConfirmStats(buckets, bucketMap, MED_BLOCK_PERIODS, MED_DECAY, MED_SCALE)); + std::unique_ptr fileShortStats(new TxConfirmStats(buckets, bucketMap, SHORT_BLOCK_PERIODS, SHORT_DECAY, SHORT_SCALE)); + std::unique_ptr fileLongStats(new TxConfirmStats(buckets, bucketMap, LONG_BLOCK_PERIODS, LONG_DECAY, LONG_SCALE)); + fileFeeStats->Read(filein, nVersionThatWrote, numBuckets); + fileShortStats->Read(filein, nVersionThatWrote, numBuckets); + fileLongStats->Read(filein, nVersionThatWrote, numBuckets); + + // Fee estimates file parsed correctly + // Copy buckets from file and refresh our bucketmap + buckets = fileBuckets; + bucketMap.clear(); + for (unsigned int i = 0; i < buckets.size(); i++) { + bucketMap[buckets[i]] = i; + } + + // Destroy old TxConfirmStats and point to new ones that already reference buckets and bucketMap + delete feeStats; + delete shortStats; + delete longStats; + feeStats = fileFeeStats.release(); + shortStats = fileShortStats.release(); + longStats = fileLongStats.release(); + + nBestSeenHeight = nFileBestSeenHeight; + historicalFirst = nFileHistoricalFirst; + historicalBest = nFileHistoricalBest; + } } - catch (const std::exception&) { - LogPrintf("CBlockPolicyEstimator::Read(): unable to read policy estimator data (non-fatal)\n"); + catch (const std::exception& e) { + LogPrintf("CBlockPolicyEstimator::Read(): unable to read policy estimator data (non-fatal): %s\n",e.what()); return false; } return true; } +void CBlockPolicyEstimator::FlushUnconfirmed(CTxMemPool& pool) { + int64_t startclear = GetTimeMicros(); + std::vector txids; + pool.queryHashes(txids); + LOCK(cs_feeEstimator); + for (auto& txid : txids) { + removeTx(txid, false); + } + int64_t endclear = GetTimeMicros(); + LogPrint(BCLog::ESTIMATEFEE, "Recorded %u unconfirmed txs from mempool in %ld micros\n",txids.size(), endclear - startclear); +} diff --git a/src/policy/fees.h b/src/policy/fees.h index ce087c238e3e..ed77efcb6167 100644 --- a/src/policy/fees.h +++ b/src/policy/fees.h @@ -42,53 +42,57 @@ class TxConfirmStats; * within your desired 5 blocks. * * Here is a brief description of the implementation: - * When a transaction enters the mempool, we - * track the height of the block chain at entry. Whenever a block comes in, - * we count the number of transactions in each bucket and the total amount of feerate - * paid in each bucket. Then we calculate how many blocks Y it took each - * transaction to be mined and we track an array of counters in each bucket - * for how long it to took transactions to get confirmed from 1 to a max of 25 - * and we increment all the counters from Y up to 25. This is because for any - * number Z>=Y the transaction was successfully mined within Z blocks. We - * want to save a history of this information, so at any time we have a - * counter of the total number of transactions that happened in a given feerate - * bucket and the total number that were confirmed in each number 1-25 blocks - * or less for any bucket. We save this history by keeping an exponentially - * decaying moving average of each one of these stats. Furthermore we also - * keep track of the number unmined (in mempool) transactions in each bucket - * and for how many blocks they have been outstanding and use that to increase - * the number of transactions we've seen in that feerate bucket when calculating - * an estimate for any number of confirmations below the number of blocks - * they've been outstanding. + * When a transaction enters the mempool, we track the height of the block chain + * at entry. All further calculations are conducted only on this set of "seen" + * transactions. Whenever a block comes in, we count the number of transactions + * in each bucket and the total amount of feerate paid in each bucket. Then we + * calculate how many blocks Y it took each transaction to be mined. We convert + * from a number of blocks to a number of periods Y' each encompassing "scale" + * blocks. This is is tracked in 3 different data sets each up to a maximum + * number of periods. Within each data set we have an array of counters in each + * feerate bucket and we increment all the counters from Y' up to max periods + * representing that a tx was successfullly confirmed in less than or equal to + * that many periods. We want to save a history of this information, so at any + * time we have a counter of the total number of transactions that happened in a + * given feerate bucket and the total number that were confirmed in each of the + * periods or less for any bucket. We save this history by keeping an + * exponentially decaying moving average of each one of these stats. This is + * done for a different decay in each of the 3 data sets to keep relevant data + * from different time horizons. Furthermore we also keep track of the number + * unmined (in mempool or left mempool without being included in a block) + * transactions in each bucket and for how many blocks they have been + * outstanding and use both of these numbers to increase the number of transactions + * we've seen in that feerate bucket when calculating an estimate for any number + * of confirmations below the number of blocks they've been outstanding. */ -/** Track confirm delays up to 25 blocks, can't estimate beyond that */ -static const unsigned int MAX_BLOCK_CONFIRMS = 25; - -/** Decay of .998 is a half-life of 346 blocks or about 14.4 hours */ -static const double DEFAULT_DECAY = .998; - -/** Require greater than 95% of X feerate transactions to be confirmed within Y blocks for X to be big enough */ -static const double MIN_SUCCESS_PCT = .95; - -/** Require an avg of 1 tx in the combined feerate bucket per block to have stat significance */ -static const double SUFFICIENT_FEETXS = 1; +/* Identifier for each of the 3 different TxConfirmStats which will track + * history over different time horizons. */ +enum FeeEstimateHorizon { + SHORT_HALFLIFE = 0, + MED_HALFLIFE = 1, + LONG_HALFLIFE = 2 +}; -// Minimum and Maximum values for tracking feerates -// The MIN_BUCKET_FEERATE should just be set to the lowest reasonable feerate we -// might ever want to track. Historically this has been 1000 since it was -// inheriting DEFAULT_MIN_RELAY_TX_FEE and changing it is disruptive as it -// invalidates old estimates files. So leave it at 1000 unless it becomes -// necessary to lower it, and then lower it substantially. -static constexpr double MIN_BUCKET_FEERATE = 1000; -static const double MAX_BUCKET_FEERATE = 1e7; -static const double INF_FEERATE = MAX_MONEY; +/* Used to return detailed information about a feerate bucket */ +struct EstimatorBucket +{ + double start = -1; + double end = -1; + double withinTarget = 0; + double totalConfirmed = 0; + double inMempool = 0; + double leftMempool = 0; +}; -// We have to lump transactions into buckets based on feerate, but we want to be able -// to give accurate estimates over a large range of potential feerates -// Therefore it makes sense to exponentially space the buckets -/** Spacing of FeeRate buckets */ -static const double FEE_SPACING = 1.1; +/* Used to return detailed information about a fee estimate calculation */ +struct EstimationResult +{ + EstimatorBucket pass; + EstimatorBucket fail; + double decay; + unsigned int scale; +}; /** * We want to be able to estimate feerates that are needed on tx's to be included in @@ -97,6 +101,55 @@ static const double FEE_SPACING = 1.1; */ class CBlockPolicyEstimator { +private: + /** Track confirm delays up to 12 blocks for short horizon */ + static constexpr unsigned int SHORT_BLOCK_PERIODS = 12; + static constexpr unsigned int SHORT_SCALE = 1; + /** Track confirm delays up to 48 blocks for medium horizon */ + static constexpr unsigned int MED_BLOCK_PERIODS = 24; + static constexpr unsigned int MED_SCALE = 2; + /** Track confirm delays up to 1008 blocks for long horizon */ + static constexpr unsigned int LONG_BLOCK_PERIODS = 42; + static constexpr unsigned int LONG_SCALE = 24; + /** Historical estimates that are older than this aren't valid */ + static const unsigned int OLDEST_ESTIMATE_HISTORY = 6 * 1008; + + /** Decay of .962 is a half-life of 18 blocks or about 3 hours */ + static constexpr double SHORT_DECAY = .962; + /** Decay of .998 is a half-life of 144 blocks or about 1 day */ + static constexpr double MED_DECAY = .9952; + /** Decay of .9995 is a half-life of 1008 blocks or about 1 week */ + static constexpr double LONG_DECAY = .99931; + + /** Require greater than 60% of X feerate transactions to be confirmed within Y/2 blocks*/ + static constexpr double HALF_SUCCESS_PCT = .6; + /** Require greater than 85% of X feerate transactions to be confirmed within Y blocks*/ + static constexpr double SUCCESS_PCT = .85; + /** Require greater than 95% of X feerate transactions to be confirmed within 2 * Y blocks*/ + static constexpr double DOUBLE_SUCCESS_PCT = .95; + + /** Require an avg of 0.1 tx in the combined feerate bucket per block to have stat significance */ + static constexpr double SUFFICIENT_FEETXS = 0.1; + /** Require an avg of 0.5 tx when using short decay since there are fewer blocks considered*/ + static constexpr double SUFFICIENT_TXS_SHORT = 0.5; + + /** Minimum and Maximum values for tracking feerates + * The MIN_BUCKET_FEERATE should just be set to the lowest reasonable feerate we + * might ever want to track. Historically this has been 1000 since it was + * inheriting DEFAULT_MIN_RELAY_TX_FEE and changing it is disruptive as it + * invalidates old estimates files. So leave it at 1000 unless it becomes + * necessary to lower it, and then lower it substantially. + */ + static constexpr double MIN_BUCKET_FEERATE = 1000; + static constexpr double MAX_BUCKET_FEERATE = 1e7; + + /** Spacing of FeeRate buckets + * We have to lump transactions into buckets based on feerate, but we want to be able + * to give accurate estimates over a large range of potential feerates + * Therefore it makes sense to exponentially space the buckets + */ + static constexpr double FEE_SPACING = 1.05; + public: /** Create new BlockPolicyEstimator and initialize stats tracking classes with default values */ CBlockPolicyEstimator(); @@ -110,16 +163,23 @@ class CBlockPolicyEstimator void processTransaction(const CTxMemPoolEntry& entry, bool validFeeEstimate); /** Remove a transaction from the mempool tracking stats*/ - bool removeTx(uint256 hash); + bool removeTx(uint256 hash, bool inBlock); - /** Return a feerate estimate */ + /** DEPRECATED. Return a feerate estimate */ CFeeRate estimateFee(int confTarget) const; - /** Estimate feerate needed to get be included in a block within - * confTarget blocks. If no answer can be given at confTarget, return an - * estimate at the lowest target where one can be given. + /** Estimate feerate needed to get be included in a block within confTarget + * blocks. If no answer can be given at confTarget, return an estimate at + * the closest target where one can be given. 'conservative' estimates are + * valid over longer time horizons also. + */ + CFeeRate estimateSmartFee(int confTarget, int *answerFoundAtTarget, const CTxMemPool& pool, bool conservative = true) const; + + /** Return a specific fee estimate calculation with a given success + * threshold and time horizon, and optionally return detailed data about + * calculation */ - CFeeRate estimateSmartFee(int confTarget, int *answerFoundAtTarget, const CTxMemPool& pool) const; + CFeeRate estimateRawFee(int confTarget, double successThreshold, FeeEstimateHorizon horizon, EstimationResult *result = nullptr) const; /** Write estimation data to a file */ bool Write(CAutoFile& fileout) const; @@ -127,9 +187,15 @@ class CBlockPolicyEstimator /** Read estimation data from a file */ bool Read(CAutoFile& filein); + /** Empty mempool transactions on shutdown to record failure to confirm for txs still in mempool */ + void FlushUnconfirmed(CTxMemPool& pool); + private: - CFeeRate minTrackedFee; //!< Passed to constructor to avoid dependency on main unsigned int nBestSeenHeight; + unsigned int firstRecordedHeight; + unsigned int historicalFirst; + unsigned int historicalBest; + struct TxStatsInfo { unsigned int blockHeight; @@ -142,14 +208,30 @@ class CBlockPolicyEstimator /** Classes to track historical data on transaction confirmations */ TxConfirmStats* feeStats; + TxConfirmStats* shortStats; + TxConfirmStats* longStats; unsigned int trackedTxs; unsigned int untrackedTxs; + std::vector buckets; // The upper-bound of the range for the bucket (inclusive) + std::map bucketMap; // Map of bucket upper-bound to index into all vectors by bucket + mutable CCriticalSection cs_feeEstimator; /** Process a transaction confirmed in a block*/ bool processBlockTx(unsigned int nBlockHeight, const CTxMemPoolEntry* entry); + /** Helper for estimateSmartFee */ + double estimateCombinedFee(unsigned int confTarget, double successThreshold, bool checkShorterHorizon) const; + /** Helper for estimateSmartFee */ + double estimateConservativeFee(unsigned int doubleTarget) const; + /** Number of blocks of data recorded while fee estimates have been running */ + unsigned int BlockSpan() const; + /** Number of blocks of recorded fee estimate data represented in saved data file */ + unsigned int HistoricalBlockSpan() const; + /** Calculation of highest target that reasonable estimate can be provided for */ + unsigned int MaxUsableEstimate() const; }; + #endif /*BITCOIN_POLICYESTIMATOR_H */ diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index 10ca11922859..40772d08e6ba 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -137,6 +137,10 @@ static const CRPCConvertParam vRPCConvertParams[] = { "getrawmempool", 0, "verbose" }, { "estimatefee", 0, "nblocks" }, { "estimatesmartfee", 0, "nblocks" }, + { "estimatesmartfee", 1, "conservative" }, + { "estimaterawfee", 0, "nblocks" }, + { "estimaterawfee", 1, "threshold" }, + { "estimaterawfee", 2, "horizon" }, { "prioritisetransaction", 1, "fee_delta" }, { "setban", 2, "bantime" }, { "setban", 3, "absolute" }, diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index a823554267d2..978b279f1e1e 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -825,6 +825,7 @@ UniValue estimatefee(const JSONRPCRequest& request) if (request.fHelp || request.params.size() != 1) throw std::runtime_error( "estimatefee nblocks\n" + "\nDEPRECATED. Please use estimatesmartfee for more intelligent estimates." "\nEstimates the approximate fee per kilobyte needed for a transaction to begin\n" "confirmation within nblocks blocks.\n" "\nArguments:\n" @@ -855,15 +856,18 @@ UniValue estimatefee(const JSONRPCRequest& request) UniValue estimatesmartfee(const JSONRPCRequest& request) { - if (request.fHelp || request.params.size() != 1) + if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) throw std::runtime_error( - "estimatesmartfee nblocks\n" - "\nWARNING: This interface is unstable and may disappear or change!\n" + "estimatesmartfee nblocks (conservative)\n" "\nEstimates the approximate fee per kilobyte needed for a transaction to begin\n" "confirmation within nblocks blocks if possible and return the number of blocks\n" "for which the estimate is valid.\n" "\nArguments:\n" - "1. nblocks (numeric)\n" + "1. nblocks (numeric)\n" + "2. conservative (bool, optional, default=true) Whether to return a more conservative estimate which\n" + " also satisfies a longer history. A conservative estimate potentially returns a higher\n" + " feerate and is more likely to be sufficient for the desired target, but is not as\n" + " responsive to short term drops in the prevailing fee market\n" "\nResult:\n" "{\n" " \"feerate\" : x.x, (numeric) estimate fee-per-kilobyte (in " + CURRENCY_UNIT + ")\n" @@ -880,15 +884,102 @@ UniValue estimatesmartfee(const JSONRPCRequest& request) RPCTypeCheck(request.params, {UniValue::VNUM}); int nBlocks = request.params[0].get_int(); + bool conservative = true; + if (request.params.size() > 1 && !request.params[1].isNull()) { + RPCTypeCheckArgument(request.params[1], UniValue::VBOOL); + conservative = request.params[1].get_bool(); + } UniValue result(UniValue::VOBJ); int answerFound; - CFeeRate feeRate = ::feeEstimator.estimateSmartFee(nBlocks, &answerFound, ::mempool); + CFeeRate feeRate = ::feeEstimator.estimateSmartFee(nBlocks, &answerFound, ::mempool, conservative); result.push_back(Pair("feerate", feeRate == CFeeRate(0) ? -1.0 : ValueFromAmount(feeRate.GetFeePerK()))); result.push_back(Pair("blocks", answerFound)); return result; } +UniValue estimaterawfee(const JSONRPCRequest& request) +{ + if (request.fHelp || request.params.size() < 1|| request.params.size() > 3) + throw std::runtime_error( + "estimaterawfee nblocks (threshold horizon)\n" + "\nWARNING: This interface is unstable and may disappear or change!\n" + "\nWARNING: This is an advanced API call that is tightly coupled to the specific\n" + " implementation of fee estimation. The parameters it can be called with\n" + " and the results it returns will change if the internal implementation changes.\n" + "\nEstimates the approximate fee per kilobyte needed for a transaction to begin\n" + "confirmation within nblocks blocks if possible. Uses virtual transaction size as defined\n" + "in BIP 141 (witness data is discounted).\n" + "\nArguments:\n" + "1. nblocks (numeric)\n" + "2. threshold (numeric, optional) The proportion of transactions in a given feerate range that must have been\n" + " confirmed within nblocks in order to consider those feerates as high enough and proceed to check\n" + " lower buckets. Default: 0.95\n" + "3. horizon (numeric, optional) How long a history of estimates to consider. 0=short, 1=medium, 2=long.\n" + " Default: 1\n" + "\nResult:\n" + "{\n" + " \"feerate\" : x.x, (numeric) estimate fee-per-kilobyte (in BTC)\n" + " \"decay\" : x.x, (numeric) exponential decay (per block) for historical moving average of confirmation data\n" + " \"scale\" : x, (numeric) The resolution of confirmation targets at this time horizon\n" + " \"pass\" : { (json object) information about the lowest range of feerates to succeed in meeting the threshold\n" + " \"startrange\" : x.x, (numeric) start of feerate range\n" + " \"endrange\" : x.x, (numeric) end of feerate range\n" + " \"withintarget\" : x.x, (numeric) number of txs over history horizon in the feerate range that were confirmed within target\n" + " \"totalconfirmed\" : x.x, (numeric) number of txs over history horizon in the feerate range that were confirmed at any point\n" + " \"inmempool\" : x.x, (numeric) current number of txs in mempool in the feerate range unconfirmed for at least target blocks\n" + " \"leftmempool\" : x.x, (numeric) number of txs over history horizon in the feerate range that left mempool unconfirmed after target\n" + " }\n" + " \"fail\" : { ... } (json object) information about the highest range of feerates to fail to meet the threshold\n" + "}\n" + "\n" + "A negative feerate is returned if no answer can be given.\n" + "\nExample:\n" + + HelpExampleCli("estimaterawfee", "6 0.9 1") + ); + + RPCTypeCheck(request.params, boost::assign::list_of(UniValue::VNUM)(UniValue::VNUM)(UniValue::VNUM), true); + RPCTypeCheckArgument(request.params[0], UniValue::VNUM); + int nBlocks = request.params[0].get_int(); + double threshold = 0.95; + if (!request.params[1].isNull()) + threshold = request.params[1].get_real(); + FeeEstimateHorizon horizon = FeeEstimateHorizon::MED_HALFLIFE; + if (!request.params[2].isNull()) { + int horizonInt = request.params[2].get_int(); + if (horizonInt < 0 || horizonInt > 2) { + throw JSONRPCError(RPC_TYPE_ERROR, "Invalid horizon for fee estimates"); + } else { + horizon = (FeeEstimateHorizon)horizonInt; + } + } + UniValue result(UniValue::VOBJ); + CFeeRate feeRate; + EstimationResult buckets; + feeRate = ::feeEstimator.estimateRawFee(nBlocks, threshold, horizon, &buckets); + + result.push_back(Pair("feerate", feeRate == CFeeRate(0) ? -1.0 : ValueFromAmount(feeRate.GetFeePerK()))); + result.push_back(Pair("decay", buckets.decay)); + result.push_back(Pair("scale", (int)buckets.scale)); + UniValue passbucket(UniValue::VOBJ); + passbucket.push_back(Pair("startrange", round(buckets.pass.start))); + passbucket.push_back(Pair("endrange", round(buckets.pass.end))); + passbucket.push_back(Pair("withintarget", round(buckets.pass.withinTarget * 100.0) / 100.0)); + passbucket.push_back(Pair("totalconfirmed", round(buckets.pass.totalConfirmed * 100.0) / 100.0)); + passbucket.push_back(Pair("inmempool", round(buckets.pass.inMempool * 100.0) / 100.0)); + passbucket.push_back(Pair("leftmempool", round(buckets.pass.leftMempool * 100.0) / 100.0)); + result.push_back(Pair("pass", passbucket)); + UniValue failbucket(UniValue::VOBJ); + failbucket.push_back(Pair("startrange", round(buckets.fail.start))); + failbucket.push_back(Pair("endrange", round(buckets.fail.end))); + failbucket.push_back(Pair("withintarget", round(buckets.fail.withinTarget * 100.0) / 100.0)); + failbucket.push_back(Pair("totalconfirmed", round(buckets.fail.totalConfirmed * 100.0) / 100.0)); + failbucket.push_back(Pair("inmempool", round(buckets.fail.inMempool * 100.0) / 100.0)); + failbucket.push_back(Pair("leftmempool", round(buckets.fail.leftMempool * 100.0) / 100.0)); + result.push_back(Pair("fail", failbucket)); + return result; +} + static const CRPCCommand commands[] = { // category name actor (function) okSafeMode // --------------------- ------------------------ ----------------------- ---------- @@ -903,7 +994,9 @@ static const CRPCCommand commands[] = { "generating", "generatetoaddress", &generatetoaddress, true, {"nblocks","address","maxtries"} }, #endif // ENABLE_MINER { "util", "estimatefee", &estimatefee, true, {"nblocks"} }, - { "util", "estimatesmartfee", &estimatesmartfee, true, {"nblocks"} }, + { "util", "estimatesmartfee", &estimatesmartfee, true, {"nblocks", "conservative"} }, + + { "hidden", "estimaterawfee", &estimaterawfee, true, {"nblocks", "threshold", "horizon"} }, }; void RegisterMiningRPCCommands(CRPCTable &t) diff --git a/src/test/policyestimator_tests.cpp b/src/test/policyestimator_tests.cpp index 1eea585ac3ad..b1b894e42c9d 100644 --- a/src/test/policyestimator_tests.cpp +++ b/src/test/policyestimator_tests.cpp @@ -49,8 +49,8 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) int blocknum = 0; // Loop through 200 blocks - // At a decay .998 and 4 fee transactions per block - // This makes the tx count about 1.33 per bucket, above the 1 threshold + // At a decay .9952 and 4 fee transactions per block + // This makes the tx count about 2.5 per bucket, well above the 0.1 threshold while (blocknum < 200) { for (int j = 0; j < 10; j++) { // For each fee for (int k = 0; k < 4; k++) { // add 4 fee txs @@ -74,20 +74,14 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) } mpool.removeForBlock(block, ++blocknum); block.clear(); - if (blocknum == 30) { - // At this point we should need to combine 5 buckets to get enough data points - // So estimateFee(1,2,3) should fail and estimateFee(4) should return somewhere around - // 8*baserate. estimateFee(4) %'s are 100,100,100,100,90 = average 98% + // Check after just a few txs that combining buckets works as expected + if (blocknum == 3) { + // At this point we should need to combine 3 buckets to get enough data points + // So estimateFee(1) should fail and estimateFee(2) should return somewhere around + // 9*baserate. estimateFee(2) %'s are 100,100,90 = average 97% BOOST_CHECK(feeEst.estimateFee(1) == CFeeRate(0)); - BOOST_CHECK(feeEst.estimateFee(2) == CFeeRate(0)); - BOOST_CHECK(feeEst.estimateFee(3) == CFeeRate(0)); - BOOST_CHECK(feeEst.estimateFee(4).GetFeePerK() < 8*baseRate.GetFeePerK() + deltaFee); - BOOST_CHECK(feeEst.estimateFee(4).GetFeePerK() > 8*baseRate.GetFeePerK() - deltaFee); - int answerFound; - BOOST_CHECK(feeEst.estimateSmartFee(1, &answerFound, mpool) == feeEst.estimateFee(4) && answerFound == 4); - BOOST_CHECK(feeEst.estimateSmartFee(3, &answerFound, mpool) == feeEst.estimateFee(4) && answerFound == 4); - BOOST_CHECK(feeEst.estimateSmartFee(4, &answerFound, mpool) == feeEst.estimateFee(4) && answerFound == 4); - BOOST_CHECK(feeEst.estimateSmartFee(8, &answerFound, mpool) == feeEst.estimateFee(8) && answerFound == 8); + BOOST_CHECK(feeEst.estimateFee(2).GetFeePerK() < 9*baseRate.GetFeePerK() + deltaFee); + BOOST_CHECK(feeEst.estimateFee(2).GetFeePerK() > 9*baseRate.GetFeePerK() - deltaFee); } } @@ -104,13 +98,14 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) BOOST_CHECK(origFeeEst[i-1] <= origFeeEst[i-2]); } int mult = 11-i; - if (i > 1) { + if (i % 2 == 0) { //At scale 2, test logic is only correct for even targets BOOST_CHECK(origFeeEst[i-1] < mult*baseRate.GetFeePerK() + deltaFee); BOOST_CHECK(origFeeEst[i-1] > mult*baseRate.GetFeePerK() - deltaFee); } - else { - BOOST_CHECK(origFeeEst[i-1] == CFeeRate(0).GetFeePerK()); - } + } + // Fill out rest of the original estimates + for (int i = 10; i <= 48; i++) { + origFeeEst.push_back(feeEst.estimateFee(i).GetFeePerK()); } // Mine 50 more blocks with no transactions happening, estimates shouldn't change @@ -139,10 +134,8 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) mpool.removeForBlock(block, ++blocknum); } - int answerFound; for (int i = 1; i < 10;i++) { BOOST_CHECK(feeEst.estimateFee(i) == CFeeRate(0) || feeEst.estimateFee(i).GetFeePerK() > origFeeEst[i-1] - deltaFee); - BOOST_CHECK(feeEst.estimateSmartFee(i, &answerFound, mpool).GetFeePerK() > origFeeEst[answerFound-1] - deltaFee); } // Mine all those transactions @@ -155,16 +148,16 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) txHashes[j].pop_back(); } } - mpool.removeForBlock(block, 265); + mpool.removeForBlock(block, 266); block.clear(); BOOST_CHECK(feeEst.estimateFee(1) == CFeeRate(0)); for (int i = 2; i < 10;i++) { - BOOST_CHECK(feeEst.estimateFee(i).GetFeePerK() > origFeeEst[i-1] - deltaFee); + BOOST_CHECK(feeEst.estimateFee(i) == CFeeRate(0) || feeEst.estimateFee(i).GetFeePerK() > origFeeEst[i-1] - deltaFee); } - // Mine 200 more blocks where everything is mined every block + // Mine 400 more blocks where everything is mined every block // Estimates should be below original estimates - while (blocknum < 465) { + while (blocknum < 665) { for (int j = 0; j < 10; j++) { // For each fee multiple for (int k = 0; k < 4; k++) { // add 4 fee txs tx.vin[0].prevout.n = 10000*blocknum+100*j+k; @@ -180,7 +173,7 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) block.clear(); } BOOST_CHECK(feeEst.estimateFee(1) == CFeeRate(0)); - for (int i = 2; i < 10; i++) { + for (int i = 2; i < 9; i++) { // At 9, the original estimate was already at the bottom (b/c scale = 2) BOOST_CHECK(feeEst.estimateFee(i).GetFeePerK() < origFeeEst[i-1] - deltaFee); } @@ -190,7 +183,7 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) mpool.TrimToSize(1); BOOST_CHECK(mpool.GetMinFee(1).GetFeePerK() > feeV[5]); for (int i = 1; i < 10; i++) { - BOOST_CHECK(feeEst.estimateSmartFee(i, NULL, mpool).GetFeePerK() >= feeEst.estimateFee(i).GetFeePerK()); + BOOST_CHECK(feeEst.estimateSmartFee(i, NULL, mpool).GetFeePerK() >= feeEst.estimateRawFee(i, 0.85, FeeEstimateHorizon::MED_HALFLIFE).GetFeePerK()); BOOST_CHECK(feeEst.estimateSmartFee(i, NULL, mpool).GetFeePerK() >= mpool.GetMinFee(1).GetFeePerK()); } } diff --git a/src/txmempool.cpp b/src/txmempool.cpp index a0a1393d8637..55590b4cd081 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -693,7 +693,7 @@ void CTxMemPool::removeUnchecked(txiter it, MemPoolRemovalReason reason) mapLinks.erase(it); mapTx.erase(it); nTransactionsUpdated++; - if (minerPolicyEstimator) {minerPolicyEstimator->removeTx(hash);} + if (minerPolicyEstimator) {minerPolicyEstimator->removeTx(hash, false);} removeAddressIndex(hash); removeSpentIndex(hash); } diff --git a/test/functional/smartfees.py b/test/functional/smartfees.py index e541e8ad54cb..baf9d94480d9 100755 --- a/test/functional/smartfees.py +++ b/test/functional/smartfees.py @@ -116,8 +116,8 @@ def check_estimates(node, fees_seen, max_invalid, print_estimates = True): for i,e in enumerate(all_estimates): # estimate is for i+1 if e >= 0: valid_estimate = True - # estimatesmartfee should return the same result - assert_equal(node.estimatesmartfee(i+1)["feerate"], e) + if i >= 13: # for n>=14 estimatesmartfee(n/2) should be at least as high as estimatefee(n) + assert(node.estimatesmartfee((i+1)//2)["feerate"] > float(e) - delta) else: invalid_estimates += 1 From d76cd903e4f4e48fd411593ac838980c8bd775e6 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Fri, 26 May 2017 11:55:00 -0700 Subject: [PATCH 23/36] Merge #10419: [trivial] Fix three recently introduced typos efc2e3302 [trivial] Fix three recently introduced typos (practicalswift) Tree-SHA512: 99e97f3c1350299dfce9c0c35547d480f25c0b877da311d9120f113afd3089eda31b88b2378e2370f288b0c41bb69cee0fd3abca661cd93d5a56982f90709f91 --- src/policy/fees.cpp | 2 +- src/policy/fees.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/policy/fees.cpp b/src/policy/fees.cpp index 58c17792dceb..56e5fd2be04d 100644 --- a/src/policy/fees.cpp +++ b/src/policy/fees.cpp @@ -784,7 +784,7 @@ CFeeRate CBlockPolicyEstimator::estimateSmartFee(int confTarget, int *answerFoun * This is necessary to preserve monotonically increasing estimates. * For non-conservative estimates we do the same thing for 2*target, but * for conservative estimates we want to skip these shorter horizons - * checks for 2*target becuase we are taking the max over all time + * checks for 2*target because we are taking the max over all time * horizons so we already have monotonically increasing estimates and * the purpose of conservative estimates is not to let short term * fluctuations lower our estimates by too much. diff --git a/src/policy/fees.h b/src/policy/fees.h index ed77efcb6167..8fef145459a5 100644 --- a/src/policy/fees.h +++ b/src/policy/fees.h @@ -51,7 +51,7 @@ class TxConfirmStats; * blocks. This is is tracked in 3 different data sets each up to a maximum * number of periods. Within each data set we have an array of counters in each * feerate bucket and we increment all the counters from Y' up to max periods - * representing that a tx was successfullly confirmed in less than or equal to + * representing that a tx was successfully confirmed in less than or equal to * that many periods. We want to save a history of this information, so at any * time we have a counter of the total number of transactions that happened in a * given feerate bucket and the total number that were confirmed in each of the From ae1beffe00cc40b1f0689f6bf1baed6330652a91 Mon Sep 17 00:00:00 2001 From: Pasta Date: Wed, 17 Jul 2019 14:20:39 -0500 Subject: [PATCH 24/36] remove boost list_of Signed-off-by: Pasta --- src/rpc/mining.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index 978b279f1e1e..bcae9293f0f0 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -938,7 +938,7 @@ UniValue estimaterawfee(const JSONRPCRequest& request) + HelpExampleCli("estimaterawfee", "6 0.9 1") ); - RPCTypeCheck(request.params, boost::assign::list_of(UniValue::VNUM)(UniValue::VNUM)(UniValue::VNUM), true); + RPCTypeCheck(request.params, {UniValue::VNUM, UniValue::VNUM, UniValue::VNUM}, true); RPCTypeCheckArgument(request.params[0], UniValue::VNUM); int nBlocks = request.params[0].get_int(); double threshold = 0.95; From 83af2d8d506354b65c353cef74455abe1399c24a Mon Sep 17 00:00:00 2001 From: Pasta Date: Wed, 24 Jul 2019 12:02:28 -0500 Subject: [PATCH 25/36] remove redundant wait_node Signed-off-by: Pasta --- test/functional/fundrawtransaction.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/functional/fundrawtransaction.py b/test/functional/fundrawtransaction.py index 8f99eb5ef2eb..1a8094020a09 100755 --- a/test/functional/fundrawtransaction.py +++ b/test/functional/fundrawtransaction.py @@ -454,7 +454,6 @@ def run_test(self): self.stop_node(3) self.nodes[1].encryptwallet("test") bitcoind_processes[1].wait(timeout=BITCOIND_PROC_WAIT_TIMEOUT) - wait_node(1) self.nodes = self.start_nodes(self.num_nodes, self.options.tmpdir, [['-usehd=0']] * self.num_nodes) # This test is not meant to test fee estimation and we'd like From c34ec30b61c5c2c92285aaeb23a0ce335c6670d6 Mon Sep 17 00:00:00 2001 From: Pasta Date: Wed, 24 Jul 2019 12:03:39 -0500 Subject: [PATCH 26/36] s/bitcoin/dash Signed-off-by: Pasta --- test/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/README.md b/test/README.md index bce08f81bffe..c87b129a8b9a 100644 --- a/test/README.md +++ b/test/README.md @@ -155,7 +155,7 @@ import pdb; pdb.set_trace() anywhere in the test. You will then be able to inspect variables, as well as call methods that interact with the dashd nodes-under-test. -If further introspection of the bitcoind instances themselves becomes +If further introspection of the dashd instances themselves becomes necessary, this can be accomplished by first setting a pdb breakpoint at an appropriate location, running the test to that point, then using `gdb` to attach to the process and debug. @@ -169,15 +169,15 @@ For instance, to attach to `self.node[1]` during a run: use the directory path to get the pid from the pid file: ```bash -cat /tmp/user/1000/testo9vsdjo3/node1/regtest/bitcoind.pid -gdb /home/example/bitcoind +cat /tmp/user/1000/testo9vsdjo3/node1/regtest/dashd.pid +gdb /home/example/dashd ``` Note: gdb attach step may require `sudo` ### Util tests -Util tests can be run locally by running `test/util/bitcoin-util-test.py`. +Util tests can be run locally by running `test/util/dash-util-test.py`. Use the `-v` option for verbose output. # Writing functional tests From 6ce278f56396767a9bbd1db1881eb6563b45c03d Mon Sep 17 00:00:00 2001 From: Pasta Date: Wed, 24 Jul 2019 12:04:54 -0500 Subject: [PATCH 27/36] s/149900/140100 Signed-off-by: Pasta --- src/policy/fees.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/policy/fees.cpp b/src/policy/fees.cpp index 56e5fd2be04d..b8dede4f838a 100644 --- a/src/policy/fees.cpp +++ b/src/policy/fees.cpp @@ -364,7 +364,7 @@ void TxConfirmStats::Read(CAutoFile& filein, int nFileVersion, size_t numBuckets size_t maxConfirms, maxPeriods; // The current version will store the decay with each individual TxConfirmStats and also keep a scale factor - if (nFileVersion >= 149900) { + if (nFileVersion >= 140100) { filein >> decay; if (decay <= 0 || decay >= 1) { throw std::runtime_error("Corrupt estimates file. Decay must be between 0 and 1 (non-inclusive)"); @@ -393,7 +393,7 @@ void TxConfirmStats::Read(CAutoFile& filein, int nFileVersion, size_t numBuckets } } - if (nFileVersion >= 149900) { + if (nFileVersion >= 140100) { filein >> failAvg; if (maxPeriods != failAvg.size()) { throw std::runtime_error("Corrupt estimates file. Mismatch in confirms tracked for failures"); @@ -862,7 +862,7 @@ bool CBlockPolicyEstimator::Read(CAutoFile& filein) // structures aren't corrupted if there is an exception. filein >> nFileBestSeenHeight; - if (nVersionThatWrote < 149900) { + if (nVersionThatWrote < 140100) { // Read the old fee estimates file for temporary use, but then discard. Will start collecting data from scratch. // decay is stored before buckets in old versions, so pre-read decay and pass into TxConfirmStats constructor double tempDecay; @@ -887,7 +887,7 @@ bool CBlockPolicyEstimator::Read(CAutoFile& filein) tempMap[tempBuckets[i]] = i; } } - else { // nVersionThatWrote >= 149900 + else { // nVersionThatWrote >= 140100 filein >> nFileHistoricalFirst >> nFileHistoricalBest; if (nFileHistoricalFirst > nFileHistoricalBest || nFileHistoricalBest > nFileBestSeenHeight) { throw std::runtime_error("Corrupt estimates file. Historical block range for estimates is invalid"); From c84636a4736e983fd5b23ab02644a0b71fc64e5f Mon Sep 17 00:00:00 2001 From: Pasta Date: Wed, 24 Jul 2019 12:07:03 -0500 Subject: [PATCH 28/36] update commented time estimates for fees Signed-off-by: Pasta --- src/policy/fees.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/policy/fees.h b/src/policy/fees.h index 8fef145459a5..d2ee816a1998 100644 --- a/src/policy/fees.h +++ b/src/policy/fees.h @@ -114,11 +114,11 @@ class CBlockPolicyEstimator /** Historical estimates that are older than this aren't valid */ static const unsigned int OLDEST_ESTIMATE_HISTORY = 6 * 1008; - /** Decay of .962 is a half-life of 18 blocks or about 3 hours */ + /** Decay of .962 is a half-life of 18 blocks or about 45 minutes */ static constexpr double SHORT_DECAY = .962; - /** Decay of .998 is a half-life of 144 blocks or about 1 day */ + /** Decay of .998 is a half-life of 144 blocks or about 6 hours */ static constexpr double MED_DECAY = .9952; - /** Decay of .9995 is a half-life of 1008 blocks or about 1 week */ + /** Decay of .9995 is a half-life of 1008 blocks or about 2 days */ static constexpr double LONG_DECAY = .99931; /** Require greater than 60% of X feerate transactions to be confirmed within Y/2 blocks*/ From 64c195a03d6dec6fa0fef175fdba8cfd326da0ba Mon Sep 17 00:00:00 2001 From: Pasta Date: Wed, 24 Jul 2019 12:08:08 -0500 Subject: [PATCH 29/36] remove witness/segwit based text Signed-off-by: Pasta --- src/rpc/mining.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index bcae9293f0f0..05478c1cc71a 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -908,8 +908,7 @@ UniValue estimaterawfee(const JSONRPCRequest& request) " implementation of fee estimation. The parameters it can be called with\n" " and the results it returns will change if the internal implementation changes.\n" "\nEstimates the approximate fee per kilobyte needed for a transaction to begin\n" - "confirmation within nblocks blocks if possible. Uses virtual transaction size as defined\n" - "in BIP 141 (witness data is discounted).\n" + "confirmation within nblocks blocks if possible.\n" "\nArguments:\n" "1. nblocks (numeric)\n" "2. threshold (numeric, optional) The proportion of transactions in a given feerate range that must have been\n" From f38ed3c71b6c295666fcec3d0f0797401254d8d2 Mon Sep 17 00:00:00 2001 From: Pasta Date: Wed, 24 Jul 2019 12:12:30 -0500 Subject: [PATCH 30/36] #10821 continued Signed-off-by: Pasta --- src/bench/bench_dash.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/bench/bench_dash.cpp b/src/bench/bench_dash.cpp index 2cb94af3cff2..b5a899ad8d15 100644 --- a/src/bench/bench_dash.cpp +++ b/src/bench/bench_dash.cpp @@ -4,6 +4,7 @@ #include "bench.h" +#include "crypto/sha256.h" #include "key.h" #include "stacktraces.h" #include "validation.h" @@ -18,6 +19,8 @@ void CleanupBLSDkgTests(); int main(int argc, char** argv) { + SHA256AutoDetect(); + RegisterPrettySignalHandlers(); RegisterPrettyTerminateHander(); From 316fa18593096ed3bc1610f83aa59298bc269cb4 Mon Sep 17 00:00:00 2001 From: Pasta Date: Wed, 24 Jul 2019 12:13:09 -0500 Subject: [PATCH 31/36] s/bitcoind/dashd Signed-off-by: Pasta --- test/functional/zapwallettxes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/functional/zapwallettxes.py b/test/functional/zapwallettxes.py index af867d7a52c7..963f229b4287 100755 --- a/test/functional/zapwallettxes.py +++ b/test/functional/zapwallettxes.py @@ -4,7 +4,7 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the zapwallettxes functionality. -- start two bitcoind nodes +- start two dashd nodes - create two transactions on node 0 - one is confirmed and one is unconfirmed. - restart node 0 and verify that both the confirmed and the unconfirmed transactions are still available. From e0424c1a0b89bd7f4af250c3dfb848218d9474ec Mon Sep 17 00:00:00 2001 From: PastaPastaPasta Date: Wed, 24 Jul 2019 12:15:22 -0500 Subject: [PATCH 32/36] s/libbitcoin/libdash Co-Authored-By: UdjinM6 --- src/Makefile.am | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Makefile.am b/src/Makefile.am index 3cb1cba7373d..23af507b6e87 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -386,7 +386,7 @@ crypto_libdash_crypto_a_SOURCES = \ crypto/sha512.h if EXPERIMENTAL_ASM -crypto_libbitcoin_crypto_a_SOURCES += crypto/sha256_sse4.cpp +crypto_libdash_crypto_a_SOURCES += crypto/sha256_sse4.cpp endif # x11 From 4a22fb78f981f8fa9b6c5dc445b187f3d9cd9569 Mon Sep 17 00:00:00 2001 From: Pasta Date: Wed, 24 Jul 2019 12:52:32 -0500 Subject: [PATCH 33/36] s/dash-util-test.py/bitcoin-util-test.py Signed-off-by: Pasta --- test/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/README.md b/test/README.md index c87b129a8b9a..52907ddb1608 100644 --- a/test/README.md +++ b/test/README.md @@ -177,7 +177,7 @@ Note: gdb attach step may require `sudo` ### Util tests -Util tests can be run locally by running `test/util/dash-util-test.py`. +Util tests can be run locally by running `test/util/bitcoin-util-test.py`. Use the `-v` option for verbose output. # Writing functional tests From f0c73f5cecd9453c84d1b4c39daf1c665fc50645 Mon Sep 17 00:00:00 2001 From: Pasta Date: Wed, 24 Jul 2019 12:55:09 -0500 Subject: [PATCH 34/36] Revert "Skip mempool.dat when wallet is starting in "zap" mode (#2782)" This reverts commit 60a91848a49f069748ec5141cd369359a4f7955e. --- src/validation.cpp | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/validation.cpp b/src/validation.cpp index 461616ae25de..5a478a04f94b 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -4358,11 +4358,6 @@ static const uint64_t MEMPOOL_DUMP_VERSION = 1; bool LoadMempool(void) { - if (gArgs.GetBoolArg("-zapwallettxes", false)) { - LogPrintf("Skipping mempool.dat because of zapwallettxes\n"); - return true; - } - const CChainParams& chainparams = Params(); int64_t nExpiryTimeout = gArgs.GetArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY) * 60 * 60; FILE* filestr = fsbridge::fopen(GetDataDir() / "mempool.dat", "rb"); From d6633b5fbaf37c4b9b44110b476252d4e15b4aeb Mon Sep 17 00:00:00 2001 From: Pasta Date: Wed, 24 Jul 2019 12:56:48 -0500 Subject: [PATCH 35/36] apply rpcconsole.cpp patch Signed-off-by: Pasta --- src/qt/rpcconsole.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp index 0a17126766a5..e74bb95bfbc5 100644 --- a/src/qt/rpcconsole.cpp +++ b/src/qt/rpcconsole.cpp @@ -57,8 +57,8 @@ const TrafficGraphData::GraphRange INITIAL_TRAFFIC_GRAPH_SETTING = TrafficGraphD // Repair parameters const QString SALVAGEWALLET("-salvagewallet"); const QString RESCAN("-rescan"); -const QString ZAPTXES1("-zapwallettxes=1"); -const QString ZAPTXES2("-zapwallettxes=2"); +const QString ZAPTXES1("-zapwallettxes=1 -persistmempool=0"); +const QString ZAPTXES2("-zapwallettxes=2 -persistmempool=0"); const QString UPGRADEWALLET("-upgradewallet"); const QString REINDEX("-reindex"); From d97fd1ca760261c20d763c140a3a416ebb15bc57 Mon Sep 17 00:00:00 2001 From: Pasta Date: Thu, 25 Jul 2019 11:20:48 -0500 Subject: [PATCH 36/36] state that getinfo will be deprecated in a future version Signed-off-by: Pasta --- src/rpc/misc.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/rpc/misc.cpp b/src/rpc/misc.cpp index 1ee189f426cd..29da7022cc47 100644 --- a/src/rpc/misc.cpp +++ b/src/rpc/misc.cpp @@ -58,7 +58,7 @@ UniValue getinfo(const JSONRPCRequest& request) "\nDEPRECATED. Returns an object containing various state info.\n" "\nResult:\n" "{\n" - " \"deprecation-warning\": \"...\" (string) warning that the getinfo command is deprecated and will be removed in 0.16\n" + " \"deprecation-warning\": \"...\" (string) warning that the getinfo command is deprecated and will be removed in a future version\n" " \"version\": xxxxx, (numeric) the server version\n" " \"protocolversion\": xxxxx, (numeric) the protocol version\n" " \"walletversion\": xxxxx, (numeric) the wallet version\n" @@ -94,8 +94,8 @@ UniValue getinfo(const JSONRPCRequest& request) GetProxy(NET_IPV4, proxy); UniValue obj(UniValue::VOBJ); - obj.push_back(Pair("deprecation-warning", "WARNING: getinfo is deprecated and will be fully removed in 0.16." - " Projects should transition to using getblockchaininfo, getnetworkinfo, and getwalletinfo before upgrading to 0.16")); + obj.push_back(Pair("deprecation-warning", "WARNING: getinfo is deprecated and will be fully removed in a future version." + " Projects should transition to using getblockchaininfo, getnetworkinfo, and getwalletinfo.")); obj.push_back(Pair("version", CLIENT_VERSION)); obj.push_back(Pair("protocolversion", PROTOCOL_VERSION)); #ifdef ENABLE_WALLET