Merge branch 'develop' into fixreorg

counterparty-core/counterpartycore/lib/api/compose.py

@@ -111,7 +111,7 @@
     "inputs_set": (
         str,
         None,
-        "A comma-separated list of UTXOs (`<txid>:<vout>`) to use as inputs for the transaction being created",
+        "A comma-separated list of UTXOs (`<txid>:<vout>`) to use as inputs for the transaction being created. To speed up the composition you can also use the following format for utxos: `<txid>:<vout>:<value>`.",
     ),
     "return_psbt": (
         bool,

counterparty-core/counterpartycore/lib/api/migrations/0001.create_and_populate_address_events.py

@@ -18,7 +18,7 @@ def dict_factory(cursor, row):
 
 def apply(db):
     start_time = time.time()
-    logger.debug("Populating `address_events` table...")
+    logger.debug("Populating the `address_events` table...")
 
     if hasattr(db, "row_factory"):
         db.row_factory = dict_factory

counterparty-core/counterpartycore/lib/api/migrations/0004.create_and_populate_assets_info.py

@@ -116,6 +116,43 @@ def apply(db):
         },
     )
 
+    start_time_supply = time.time()
+    logger.debug("Updating the `supply` field...")
+
+    db.execute("""
+        CREATE TEMP TABLE issuances_quantity AS
+        SELECT asset, SUM(quantity) AS quantity FROM issuances WHERE status = 'valid' GROUP BY asset
+    """)
+    db.execute("""
+        CREATE TEMP TABLE destructions_quantity AS
+        SELECT asset, SUM(quantity) AS quantity FROM destructions WHERE status = 'valid' GROUP BY asset
+    """)
+
+    db.execute("""
+        CREATE TEMP TABLE supplies AS
+        SELECT 
+            issuances_quantity.asset, 
+            issuances_quantity.quantity - COALESCE(destructions_quantity.quantity, 0) AS supply 
+        FROM issuances_quantity
+        LEFT JOIN destructions_quantity ON issuances_quantity.asset = destructions_quantity.asset
+        WHERE issuances_quantity.asset = destructions_quantity.asset
+    """)
+
+    db.execute("""
+        CREATE INDEX temp.supplies_asset_idx ON supplies(asset)
+    """)
+
+    db.execute("""
+        UPDATE assets_info SET
+        supply = COALESCE((SELECT supplies.supply FROM supplies WHERE assets_info.asset = supplies.asset), supply)
+    """)
+
+    db.execute("DROP TABLE issuances_quantity")
+    db.execute("DROP TABLE destructions_quantity")
+    db.execute("DROP TABLE supplies")
+
+    logger.debug(f"Updated the `supply` field in {time.time() - start_time_supply:.2f} seconds")
+
     db.execute("CREATE UNIQUE INDEX assets_info_asset_idx ON assets_info (asset)")
     db.execute("CREATE UNIQUE INDEX assets_info_asset_id_idx ON assets_info (asset_id)")
     db.execute("CREATE INDEX assets_info_asset_longname_idx ON assets_info (asset_longname)")

counterparty-core/counterpartycore/lib/api/queries.py

@@ -206,6 +206,8 @@ def select_rows(
                 bindings += value
             elif key.endswith("__notnull"):
                 where_field.append(f"{key[:-9]} IS NOT NULL")
+            elif key.endswith("__null"):
+                where_field.append(f"{key[:-6]} IS NULL")
             else:
                 if key in ADDRESS_FIELDS and len(value.split(",")) > 1:
                     where_field.append(f"{key} IN ({','.join(['?'] * len(value.split(',')))})")
@@ -1179,15 +1181,7 @@ def prepare_sends_where(send_type: SendType, other_conditions=None):
             where = [other_conditions] if other_conditions else []
             break
         if type_send in typing.get_args(SendType):
-            where_send = {}
-            if type_send == "send":
-                where_send = {"source__notlike": "%:%", "destination__notlike": "%:%"}
-            elif type_send == "move":
-                where_send = {"source__like": "%:%", "destination__like": "%:%"}
-            elif type_send == "attach":
-                where_send = {"source__notlike": "%:%", "destination__like": "%:%"}
-            elif type_send == "detach":
-                where_send = {"source__like": "%:%", "destination__notlike": "%:%"}
+            where_send = {"send_type": type_send}
             if other_conditions:
                 where_send.update(other_conditions)
             if where_send:
@@ -2196,7 +2190,7 @@ def get_receive_by_address_and_asset(
     )
 
 
-def prepare_dispenser_where(status, other_conditions=None):
+def prepare_dispenser_where(status, other_conditions=None, exclude_with_oracle=False):
     where = []
     statuses = status.split(",")
     for s in statuses:
@@ -2205,12 +2199,16 @@ def prepare_dispenser_where(status, other_conditions=None):
 
         if s == "all":
             where = other_conditions or {}
+            if exclude_with_oracle:
+                where["oracle_address__null"] = True
             break
 
         if s in DispenserStatusNumber:
             where_status = {"status": DispenserStatusNumber[s]}
             if other_conditions:
                 where_status.update(other_conditions)
+            if exclude_with_oracle:
+                where_status["oracle_address__null"] = True
 
             where.append(where_status)
 
@@ -2223,6 +2221,7 @@ def prepare_dispenser_where(status, other_conditions=None):
 def get_dispensers(
     state_db,
     status: DispenserStatus = "all",
+    exclude_with_oracle: bool = False,
     cursor: str = None,
     limit: int = 100,
     offset: int = None,
@@ -2231,6 +2230,7 @@ def get_dispensers(
     """
     Returns all dispensers
     :param str status: The status of the dispensers to return
+    :param bool exclude_with_oracle: Whether to exclude dispensers with an oracle
     :param str cursor: The last index of the dispensers to return
     :param int limit: The maximum number of dispensers to return (e.g. 5)
     :param int offset: The number of lines to skip before returning results (overrides the `cursor` parameter)
@@ -2240,7 +2240,7 @@ def get_dispensers(
     return select_rows(
         state_db,
         "dispensers",
-        where=prepare_dispenser_where(status),
+        where=prepare_dispenser_where(status, exclude_with_oracle=exclude_with_oracle),
         last_cursor=cursor,
         limit=limit,
         offset=offset,
@@ -2253,6 +2253,7 @@ def get_dispensers_by_address(
     state_db,
     address: str,
     status: DispenserStatus = "all",
+    exclude_with_oracle: bool = False,
     cursor: str = None,
     limit: int = 100,
     offset: int = None,
@@ -2262,6 +2263,7 @@ def get_dispensers_by_address(
     Returns the dispensers of an address
     :param str address: The address to return (e.g. $ADDRESS_1)
     :param str status: The status of the dispensers to return
+    :param bool exclude_with_oracle: Whether to exclude dispensers with an oracle
     :param str cursor: The last index of the dispensers to return
     :param int limit: The maximum number of dispensers to return (e.g. 5)
     :param int offset: The number of lines to skip before returning results (overrides the `cursor` parameter)
@@ -2270,7 +2272,9 @@ def get_dispensers_by_address(
     return select_rows(
         state_db,
         "dispensers",
-        where=prepare_dispenser_where(status, {"source": address}),
+        where=prepare_dispenser_where(
+            status, {"source": address}, exclude_with_oracle=exclude_with_oracle
+        ),
         last_cursor=cursor,
         limit=limit,
         offset=offset,
@@ -2283,6 +2287,7 @@ def get_dispensers_by_asset(
     state_db,
     asset: str,
     status: DispenserStatus = "all",
+    exclude_with_oracle: bool = False,
     cursor: str = None,
     limit: int = 100,
     offset: int = None,
@@ -2292,6 +2297,7 @@ def get_dispensers_by_asset(
     Returns the dispensers of an asset
     :param str asset: The asset to return (e.g. XCP)
     :param str status: The status of the dispensers to return
+    :param bool exclude_with_oracle: Whether to exclude dispensers with an oracle
     :param str cursor: The last index of the dispensers to return
     :param int limit: The maximum number of dispensers to return (e.g. 5)
     :param int offset: The number of lines to skip before returning results (overrides the `cursor` parameter)
@@ -2300,7 +2306,9 @@ def get_dispensers_by_asset(
     return select_rows(
         state_db,
         "dispensers",
-        where=prepare_dispenser_where(status, {"asset": asset.upper()}),
+        where=prepare_dispenser_where(
+            status, {"asset": asset.upper()}, exclude_with_oracle=exclude_with_oracle
+        ),
         last_cursor=cursor,
         limit=limit,
         offset=offset,

counterparty-core/counterpartycore/lib/api/util.py

@@ -423,9 +423,15 @@ def inject_normalized_quantity(item, field_name, asset_info):
         return item
 
     if item[field_name] is not None:
-        if field_name in ["give_price", "get_price", "forward_price", "backward_price", "price"]:
+        if field_name in ["give_price", "get_price", "forward_price", "backward_price"]:
             # use 16 decimal places for prices
             item[field_name + "_normalized"] = normalize_price(item[field_name])
+        elif field_name == "price":
+            if "satoshirate" in item:
+                price = D(item["satoshirate_normalized"]) / D(item["give_quantity_normalized"])
+                item[field_name + "_normalized"] = normalize_price(price)
+            else:
+                item[field_name + "_normalized"] = normalize_price(item[field_name])
         else:
             item[field_name + "_normalized"] = (
                 divide(item[field_name], 10**8)

counterparty-core/counterpartycore/lib/backend/electrs.py

@@ -5,11 +5,11 @@
 
 def electr_query(url):
     if config.ELECTRS_URL is None:
-        raise exceptions.ElectrError("Electrs server not configured")
+        raise exceptions.ElectrsError("Electrs server not configured")
     try:
         return requests.get(f"{config.ELECTRS_URL}/{url}", timeout=10).json()
     except requests.exceptions.RequestException as e:
-        raise exceptions.ElectrError(f"Electrs error: {e}") from e
+        raise exceptions.ElectrsError(f"Electrs error: {e}") from e
 
 
 def get_utxos(address, unconfirmed: bool = False, unspent_tx_hash: str = None):
@@ -26,6 +26,7 @@ def get_utxos(address, unconfirmed: bool = False, unspent_tx_hash: str = None):
             continue
         if unspent_tx_hash and utxo["txid"] != unspent_tx_hash:
             continue
+        utxo["amount"] = utxo["value"] / 10**8
         result.append(utxo)
     return result
 
@@ -35,7 +36,7 @@ def get_history(address: str, unconfirmed: bool = False):
     Returns all transactions involving a given address
     :param address: The address to search for (e.g. $ADDRESS_3)
     """
-    tx_list = electr_query(f"address/{address}/history")
+    tx_list = electr_query(f"address/{address}/txs")
     result = []
     for tx in tx_list:
         if tx["status"]["confirmed"] or unconfirmed:

counterparty-core/counterpartycore/lib/blocks.py

@@ -772,26 +772,6 @@ def initialise(db):
         ],
     )
 
-    # Consolidated
-    send.initialise(db)
-    destroy.initialise(db)
-    order.initialise(db)
-    btcpay.initialise(db)
-    issuance.initialise(db)
-    broadcast.initialise(db)
-    bet.initialise(db)
-    dividend.initialise(db)
-    burn.initialise(db)
-    cancel.initialise(db)
-    rps.initialise(db)
-    rpsresolve.initialise(db)
-    sweep.initialise(db)
-    dispenser.initialise(db)
-    fairminter.initialise(db)
-    fairmint.initialise(db)
-
-    gas.initialise(db)
-
     # Messages
     cursor.execute(
         """CREATE TABLE IF NOT EXISTS messages(
@@ -859,6 +839,25 @@ def initialise(db):
     if "addresses" not in columns:
         cursor.execute("""ALTER TABLE mempool ADD COLUMN addresses TEXT""")
 
+    # Consolidated
+    send.initialise(db)
+    destroy.initialise(db)
+    order.initialise(db)
+    btcpay.initialise(db)
+    issuance.initialise(db)
+    broadcast.initialise(db)
+    bet.initialise(db)
+    dividend.initialise(db)
+    burn.initialise(db)
+    cancel.initialise(db)
+    rps.initialise(db)
+    rpsresolve.initialise(db)
+    sweep.initialise(db)
+    dispenser.initialise(db)
+    fairminter.initialise(db)
+    fairmint.initialise(db)
+    gas.initialise(db)
+
     create_views(db)
 
     # Lock UPDATE on all tables

counterparty-core/counterpartycore/lib/ledger.py

@@ -865,14 +865,27 @@ def get_asset_issuances_quantity(db, asset):
 
 
 def get_assets_last_issuance(state_db, asset_list):
+    assets_info = []
     cursor = state_db.cursor()
     fields = ["asset", "asset_longname", "description", "issuer", "divisible", "locked"]
-    query = f"""
-        SELECT {", ".join(fields)} FROM assets_info
-        WHERE asset IN ({",".join(["?"] * len(asset_list))})
-    """  # nosec B608  # noqa: S608
-    cursor.execute(query, asset_list)
-    assets_info = cursor.fetchall()
+
+    asset_name_list = [asset for asset in asset_list if asset and "." not in asset]
+    if len(asset_name_list) > 0:
+        query = f"""
+            SELECT {", ".join(fields)} FROM assets_info
+            WHERE asset IN ({",".join(["?"] * len(asset_name_list))})
+        """  # nosec B608  # noqa: S608
+        cursor.execute(query, asset_name_list)
+        assets_info += cursor.fetchall()
+
+    asset_longname_list = [asset for asset in asset_list if asset and "." in asset]
+    if len(asset_longname_list) > 0:
+        query = f"""
+            SELECT {", ".join(fields)} FROM assets_info
+            WHERE asset_longname IN ({",".join(["?"] * len(asset_longname_list))})
+        """  # nosec B608  # noqa: S608
+        cursor.execute(query, asset_longname_list)
+        assets_info += cursor.fetchall()
 
     result = {
         "BTC": {
@@ -891,9 +904,14 @@ def get_assets_last_issuance(state_db, asset_list):
         },
     }
     for asset_info in assets_info:
-        asset = asset_info["asset"]
-        del asset_info["asset"]
-        result[asset] = asset_info
+        if asset_info["asset_longname"] and asset_info["asset_longname"] in asset_list:
+            result[asset_info["asset_longname"]] = asset_info
+            result[asset_info["asset"]] = asset_info
+        else:
+            asset = asset_info["asset"]
+            del asset_info["asset"]
+            result[asset] = asset_info
+
     return result
 
 

counterparty-core/counterpartycore/lib/messages/attach.py

@@ -188,6 +188,7 @@ def parse(db, tx, message):
             "msg_index": ledger.get_send_msg_index(db, tx["tx_hash"]),
             "block_index": tx["block_index"],
             "status": status,
+            "send_type": "attach",
         }
         ledger.insert_record(db, "sends", bindings, "ATTACH_TO_UTXO")
         # return here to avoid further processing
@@ -223,6 +224,7 @@ def parse(db, tx, message):
         "asset": asset,
         "quantity": quantity,
         "fee_paid": fee,
+        "send_type": "attach",
     }
     ledger.insert_record(db, "sends", bindings, "ATTACH_TO_UTXO")
 

counterparty-core/counterpartycore/lib/messages/detach.py

@@ -71,6 +71,7 @@ def detach_assets(db, tx, source, destination=None):
             "msg_index": ledger.get_send_msg_index(db, tx["tx_hash"]),
             "block_index": tx["block_index"],
             "status": status,
+            "send_type": "detach",
         }
         ledger.insert_record(db, "sends", bindings, "DETACH_FROM_UTXO")
         # stop here to avoid further processing
@@ -125,6 +126,7 @@ def detach_assets(db, tx, source, destination=None):
             "asset": balance["asset"],
             "quantity": balance["quantity"],
             "fee_paid": 0,
+            "send_type": "detach",
         }
         ledger.insert_record(db, "sends", bindings, "DETACH_FROM_UTXO")