Merge pull request #2795 from CounterpartyXCP/moretests

More tests and fixes

counterparty-core/counterpartycore/lib/api/dbbuilder.py

@@ -122,9 +122,10 @@ def rollback_state_db(state_db, block_index):
     logger.info(f"Rolling back State DB to block index {block_index}...")
     start_time = time.time()
 
-    with log.Spinner("Rolling back State DB tables..."):
-        rollback_tables(state_db, block_index)
-    with log.Spinner("Re-applying migrations..."):
-        reapply_migrations(state_db, MIGRATIONS_AFTER_ROLLBACK)
+    with state_db:
+        with log.Spinner("Rolling back State DB tables..."):
+            rollback_tables(state_db, block_index)
+        with log.Spinner("Re-applying migrations..."):
+            reapply_migrations(state_db, MIGRATIONS_AFTER_ROLLBACK)
 
     logger.info(f"State DB rolled back in {time.time() - start_time:.2f} seconds")

counterparty-core/counterpartycore/lib/messages/versions/enhanced_send.py

@@ -142,7 +142,10 @@ def compose(
     if problems and not skip_validation:
         raise exceptions.ComposeError(problems)
 
-    asset_id = ledger.get_asset_id(db, asset, block_index)
+    if not skip_validation:
+        asset_id = ledger.get_asset_id(db, asset, block_index)
+    else:
+        asset_id = ledger.generate_asset_id(asset, block_index)
 
     short_address_bytes = address.pack(destination)
 

counterparty-core/counterpartycore/test/regtest/apidoc/blueprint-template.md

@@ -167,4 +167,9 @@ Returns server information and the list of documented routes in JSON format.
     }
     ```
 
+## Mempool API
+
+The `/v2/mempool/events` route allows to retrieve events generated by transactions that have not yet been confirmed.
+Unconfirmed transactions are considered unordered and each one is parsed independently of the others. Therefore some transactions can be invalid once parsed in the mempool and valid once confirmed. And vice versa. For example, if a utxo move is chained after an attach, it will be invalid in the mempool but valid once confirmed.
+
 <API_BLUEPRINT>

counterparty-core/counterpartycore/test/regtest/regtestnode.py

@@ -4,17 +4,24 @@
 import json
 import os
 import signal
+import struct
 import sys
 import threading
 import time
 import urllib.parse
 from io import StringIO
 
 import sh
-from counterpartycore.lib import arc4, database
+from bitcoinutils.keys import P2wpkhAddress
+from bitcoinutils.script import Script, b_to_h
+from bitcoinutils.setup import setup
+from bitcoinutils.transactions import Transaction, TxInput, TxOutput
+from counterpartycore.lib import arc4, config, database
 
 WALLET_NAME = "xcpwallet"
 
+setup("regtest")
+
 
 class ServerNotReady(Exception):
     pass
@@ -115,6 +122,47 @@ def broadcast_transaction(
         self.wait_for_counterparty_server()
         return tx_hash, block_hash, block_time
 
+    def compose_and_send_transaction(
+        self, source, messsage_id=None, data=None, no_confirmation=False, dont_wait_mempool=False
+    ):
+        list_unspent = json.loads(
+            self.bitcoin_cli("listunspent", 0, 9999999, json.dumps([source])).strip()
+        )
+        sorted(list_unspent, key=lambda x: -x["amount"])
+        utxo = list_unspent[0]
+
+        tx_inputs = [TxInput(utxo["txid"], utxo["vout"])]
+        tx_outputs = []
+
+        if data is not None:
+            tx_data = b"CNTRPRTY"
+            if messsage_id is not None:
+                tx_data += struct.pack(config.SHORT_TXTYPE_FORMAT, messsage_id)
+            tx_data += data
+            key = arc4.init_arc4(binascii.unhexlify(utxo["txid"]))
+            encrypted_data = key.encrypt(data)
+            tx_outputs.append(TxOutput(0, Script(["OP_RETURN", b_to_h(encrypted_data)])))
+
+        tx_outputs.append(
+            TxOutput(int(utxo["amount"] * 1e8), P2wpkhAddress(source).to_script_pub_key())
+        )
+
+        tx = Transaction(tx_inputs, tx_outputs)
+        tx_hex = tx.to_hex()
+
+        signed_transaction_json = self.bitcoin_wallet(
+            "signrawtransactionwithwallet", tx_hex
+        ).strip()
+        signed_transaction = json.loads(signed_transaction_json)["hex"]
+
+        tx_hash, _block_hash, _block_time = self.broadcast_transaction(
+            signed_transaction, no_confirmation, dont_wait_mempool
+        )
+
+        print(f"Transaction sent: {source} {data} ({tx_hash})")
+
+        return f"{utxo['txid']}:{utxo['vout']}", tx_hash
+
     def send_transaction(
         self,
         source,

counterparty-core/counterpartycore/test/regtest/utxosupport_propertytest.py

@@ -4,6 +4,33 @@
 from hypothesis import settings
 from properytestnode import PropertyTestNode
 
+MESSAGE_IDS = [
+    0,
+    2,
+    3,
+    4,
+    10,
+    11,
+    12,
+    13,
+    20,
+    21,
+    22,
+    23,
+    30,
+    40,
+    50,
+    60,
+    70,
+    80,
+    90,
+    91,
+    100,
+    101,
+    102,
+    110,
+]
+
 
 class UTXOSupportPropertyTest(PropertyTestNode):
     def run_tests(self):
@@ -12,16 +39,16 @@ def run_tests(self):
             self.issue_assets,
             hypothesis.strategies.sampled_from(self.addresses),
             hypothesis.strategies.text(
-                alphabet="BCDEFGHIJKLMNOPQRSTUVWXYZ", min_size=5, max_size=10
+                alphabet="BCDEFGHIJKLMNOPQRSTUVWXYZ", min_size=5, max_size=8
             ),
-            hypothesis.strategies.integers(min_value=10 * 10e8, max_value=1000 * 10e8),
+            hypothesis.strategies.integers(min_value=10 * 1e8, max_value=1000 * 1e8),
         )
 
         # attach assets
         self.test_with_given_data(
             self.attach_asset,
             hypothesis.strategies.sampled_from(self.balances),
-            hypothesis.strategies.integers(min_value=1 * 10e8, max_value=10 * 10e8),
+            hypothesis.strategies.integers(min_value=1 * 1e8, max_value=10 * 1e8),
         )
 
         # move assets
@@ -38,6 +65,25 @@ def run_tests(self):
             hypothesis.strategies.sampled_from(self.addresses + [None]),
         )
 
+        # random invalid transactions
+        self.test_with_given_data(
+            self.invalid_transaction,
+            hypothesis.strategies.sampled_from(self.addresses),
+            hypothesis.strategies.sampled_from(MESSAGE_IDS),
+            hypothesis.strategies.binary(min_size=20, max_size=30),
+        )
+
+        # issue, attach, move and detach assets chaining
+        self.test_with_given_data(
+            self.issue_attach_move_detach_send,
+            hypothesis.strategies.sampled_from(self.addresses),
+            hypothesis.strategies.text(
+                alphabet="BCDEFGHIJKLMNOPQRSTUVWXYZ", min_size=9, max_size=11
+            ),
+            hypothesis.strategies.integers(min_value=10 * 1e8, max_value=1000 * 1e8),
+            hypothesis.strategies.sampled_from(self.addresses),
+        )
+
     @settings(deadline=None)
     def issue_assets(self, source, asset_name, quantity):
         # don't issue the same asset twice
@@ -109,6 +155,95 @@ def detach_asset(self, balance, destination):
         self.upsert_balance(source, asset, -quantity, utxo_address)
         self.upsert_balance(destination or utxo_address, asset, quantity, None)
 
+    @settings(deadline=None)
+    def invalid_transaction(self, source, message_id, data):
+        utxo, tx_hash = self.node.compose_and_send_transaction(
+            source, message_id, data, no_confirmation=True, dont_wait_mempool=True
+        )
+        upserts = []
+        # check if no utxo moves
+        for address_or_utxo, asset, quantity, utxo_address in self.balances:
+            if utxo == address_or_utxo:
+                upserts.append([utxo, asset, -quantity, utxo_address])
+                upserts.append([f"{tx_hash}:1", asset, quantity, utxo_address])
+        for upsert in upserts:
+            self.upsert_balance(*upsert)
+
+    @settings(deadline=None, max_examples=20)
+    def issue_attach_move_detach_send(self, source, asset_name, quantity, destination):
+        # don't issue the same asset twice
+        # and don't issue twice from the same source
+        for balance in self.balances:
+            if balance[1] == asset_name or (balance[0] == source and len(balance[1]) > 8):
+                return
+        tx_hash = self.send_transaction(
+            source,
+            "issuance",
+            {
+                "asset": asset_name,
+                "quantity": quantity,
+                "exact_fee": 0,
+            },
+        )
+        tx_hash = self.send_transaction(
+            source,
+            "attach",
+            {
+                "asset": asset_name,
+                "quantity": quantity,
+                "exact_fee": 0,
+                "validate": False,
+                "inputs_set": f"{tx_hash}:1",
+            },
+        )
+        tx_hash = self.send_transaction(
+            f"{tx_hash}:1",
+            "movetoutxo",
+            {
+                "destination": source,
+                "exact_fee": 0,
+                "validate": False,
+                "inputs_set": f"{tx_hash}:1",
+            },
+        )
+        tx_hash = self.send_transaction(
+            f"{tx_hash}:0",
+            "movetoutxo",
+            {
+                "destination": source,
+                "exact_fee": 0,
+                "validate": False,
+                "inputs_set": f"{tx_hash}:0",
+            },
+        )
+        tx_hash = self.send_transaction(
+            f"{tx_hash}:0",
+            "detach",
+            {
+                "exact_fee": 0,
+                "validate": False,
+                "inputs_set": f"{tx_hash}:0",
+            },
+        )
+        if destination != source:
+            print(f"source: {source}, destination: {destination}")
+            tx_hash = self.send_transaction(
+                source,
+                "send",
+                {
+                    "destination": destination,
+                    "asset": asset_name,
+                    "quantity": int(10 * 1e8),
+                    "exact_fee": 0,
+                    "validate": False,
+                    "inputs_set": f"{tx_hash}:1",
+                },
+            )
+            self.upsert_balance(source, asset_name, quantity - 10 * 1e8)
+            self.upsert_balance(destination, asset_name, 10 * 1e8)
+        else:
+            self.upsert_balance(source, asset_name, quantity)
+
 
 if __name__ == "__main__":
     UTXOSupportPropertyTest()

counterparty-core/requirements.txt

@@ -35,4 +35,5 @@ yoyo-migrations==8.2.0
 gunicorn==23.0.0
 waitress==3.0.1
 hypothesis==6.116.0
+bitcoin-utils==0.7.1
 counterparty-rs==10.7.3

release-notes/release-notes-v10.7.4.md

@@ -53,6 +53,7 @@
 - Add the ability to retrieve balances by asset long name
 - When composing an attach / move, use 10,000 sats for the value, rather than 546
 - Add `send_type` filter for `sends` table (`send`, `attach`, `move` or `detach`)
+- Document how mempool events work
 
 ## CLI