diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 8660d2f..41909bc 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -13,6 +13,13 @@ jobs: with: submodules: recursive + - name: Checkout master of submodules + run: | + cd lib/auction-keeper && git checkout master + cd ../pygasprice-client && git checkout master + cd ../pymaker && git checkout master + cd ../.. + - name: setup python uses: actions/setup-python@v4 with: diff --git a/DOCKER.md b/DOCKER.md index a84a1a9..9d39f57 100644 --- a/DOCKER.md +++ b/DOCKER.md @@ -13,6 +13,7 @@ git clone git@github.com:makerdao/cage-keeper.git cd cage-keeper git submodule update --init --recursive ``` +Note: To get gas prices from Etherscan or Blocknative, checkout the master branch code for the auction-keeper, and pygasprice_client submodules ## Configure, Build and Run: @@ -31,11 +32,8 @@ BLOCKCHAIN_NETWORK= # Account used to pay for gas ETH_FROM_ADDRESS= -# URL of Vulcanize instance to use -VULCANIZE_URL= - # ETH Gas Station API key -ETH_GASSTATION_API_KEY= +ETHERSCAN_API_KEY= # For ease of use, do not change the location of ETH account keys, note that account files should always be placed in the secrets directory of the cage-keeper, and files named as indicated. ETH_ACCOUNT_KEY='key_file=/opt/keeper/cage-keeper/secrets/keystore.json,pass_file=/opt/keeper/cage-keeper/secrets/password.txt' diff --git a/Dockerfile b/Dockerfile index 782f690..f3a644f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,13 +5,16 @@ RUN groupadd -r keeper && useradd -d /home/keeper -m --no-log-init -r -g keeper apt-get -y install jq bc && \ apt-get clean && rm -rf /var/lib/apt/lists/* -WORKDIR /opt/keeper +COPY bin /opt/keeper/cage-keeper/bin +COPY lib /opt/keeper/cage-keeper/lib +COPY src /opt/keeper/cage-keeper/src +COPY install.sh /opt/keeper/cage-keeper/install.sh +COPY run-cage-keeper.sh /opt/keeper/cage-keeper/run-cage-keeper.sh +COPY requirements.txt /opt/keeper/cage-keeper/requirements.txt -RUN git clone https://github.com/makerdao/cage-keeper.git && \ - cd cage-keeper && \ - git submodule update --init --recursive && \ - pip3 install virtualenv && \ +WORKDIR /opt/keeper/cage-keeper +RUN pip3 install virtualenv && \ ./install.sh -WORKDIR /opt/keeper/cage-keeper + CMD ["./run-cage-keeper.sh"] diff --git a/README.md b/README.md index 8fa0a5d..0c513f3 100644 --- a/README.md +++ b/README.md @@ -76,6 +76,7 @@ cd cage-keeper git submodule update --init --recursive ./install.sh ``` +Note: To get gas prices from Etherscan or Blocknative, checkout the master branch code for the auction-keeper, and pygasprice_client submodules For some known Ubuntu and macOS issues see the [pymaker](https://github.com/makerdao/pymaker) README. diff --git a/requirements-dev.txt b/requirements-dev.txt index 265a2e5..9d8dad3 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,5 +1,5 @@ attrs == 19.1.0 -codecov == 2.0.9 +codecov == 2.1.13 mock == 2.0.0 pytest == 3.3.0 pytest-asyncio == 0.8.0 diff --git a/run-cage-keeper.sh b/run-cage-keeper.sh index a06cc46..979076c 100755 --- a/run-cage-keeper.sh +++ b/run-cage-keeper.sh @@ -34,12 +34,11 @@ then fi - +# remove the --smart-gas-price flag to get gas prices from the node exec $dir/bin/cage-keeper \ --rpc-host "${SERVER_ETH_RPC_HOST}" \ --network "${BLOCKCHAIN_NETWORK}" \ --eth-from "${ETH_FROM_ADDRESS}" \ --eth-key "${ETH_ACCOUNT_KEY}" \ - --vulcanize-endpoint "${VULCANIZE_URL}" \ - --vulcanize-key "${VULCANIZE_KEY}" \ - --ethgasstation-api-key "${ETH_GASSTATION_API_KEY}" + --etherscan-api-key "${ETHERSCAN_API_KEY}" \ + --smart-gas-price diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/cage_keeper.py b/src/cage_keeper.py index fc36a0f..405b3a5 100644 --- a/src/cage_keeper.py +++ b/src/cage_keeper.py @@ -27,7 +27,6 @@ from web3 import Web3 from pymaker import Address, web3_via_http -from pymaker.gas import DefaultGasPrice, FixedGasPrice from pymaker.auctions import Flipper, Flapper, Flopper from pymaker.keys import register_keys from pymaker.lifecycle import Lifecycle @@ -36,8 +35,9 @@ from pymaker.deployment import DssDeployment from pymaker.dss import Ilk, Urn -from auction_keeper.urn_history import UrnHistory -from auction_keeper.gas import DynamicGasPrice +from auction_keeper.urn_history import ChainUrnHistoryProvider + +from src.gas_factory import GasPriceFactory class CageKeeper: """Keeper to facilitate Emergency Shutdown""" @@ -73,25 +73,19 @@ def __init__(self, args: list, **kwargs): parser.add_argument("--vat-deployment-block", type=int, required=False, default=0, help=" Block that the Vat from dss-deployment-file was deployed at (e.g. 8836668") - parser.add_argument("--vulcanize-endpoint", type=str, - help="When specified, frob history will be queried from a VulcanizeDB lite node, " - "reducing load on the Ethereum node for Vault query") - - parser.add_argument("--vulcanize-key", type=str, - help="API key for the Vulcanize endpoint") - parser.add_argument("--max-errors", type=int, default=100, help="Maximum number of allowed errors before the keeper terminates (default: 100)") parser.add_argument("--debug", dest='debug', action='store_true', help="Enable debug output") - parser.add_argument("--ethgasstation-api-key", type=str, default=None, help="ethgasstation API key") + parser.add_argument("--etherscan-api-key", type=str, default=None, help="etherscan API key") - parser.add_argument("--gas-initial-multiplier", type=str, default=1.0, help="ethgasstation API key") - parser.add_argument("--gas-reactive-multiplier", type=str, default=2.25, help="gas strategy tuning") - parser.add_argument("--gas-maximum", type=str, default=5000, help="gas strategy tuning") + parser.add_argument('--gas-price', type=float, default=None, + help="Uses a fixed value (in Gwei) instead of an external API to determine initial gas") + parser.add_argument("--smart-gas-price", dest='smart_gas_price', action='store_true', + help="Use smart gas pricing strategy, based on the ethgasstation.info feed") parser.set_defaults(cageFacilitated=False) @@ -118,11 +112,7 @@ def __init__(self, args: list, **kwargs): self.confirmations = 0 - # Create gas strategy - if self.arguments.ethgasstation_api_key: - self.gas_price = DynamicGasPrice(self.arguments, self.web3) - else: - self.gas_price = DefaultGasPrice() + self.gas_price = GasPriceFactory().create_gas_price(self.arguments, self.web3) logging.basicConfig(format='%(asctime)-15s %(levelname)-8s %(message)s', @@ -154,6 +144,7 @@ def check_deployment(self): self.logger.info(f'Jug: {self.dss.jug.address}') self.logger.info(f'End: {self.dss.end.address}') self.logger.info('') + self.logger.info(f'gas price is: {self.gas_price}') def process_block(self): @@ -278,12 +269,10 @@ def get_underwater_urns(self, ilks: List) -> List[Urn]: for ilk in ilks: - urn_history = UrnHistory(self.web3, + urn_history = ChainUrnHistoryProvider(self.web3, self.dss, ilk, - self.deployment_block, - self.arguments.vulcanize_endpoint, - self.arguments.vulcanize_key) + self.deployment_block) urns = urn_history.get_urns() @@ -356,4 +345,5 @@ def yank_auctions(self, flapBids: List, flopBids: List): if __name__ == '__main__': + logging.basicConfig(format='%(asctime)-15s %(levelname)-8s %(message)s', level=logging.INFO) CageKeeper(sys.argv[1:]).main() diff --git a/src/gas_factory.py b/src/gas_factory.py new file mode 100644 index 0000000..5df482c --- /dev/null +++ b/src/gas_factory.py @@ -0,0 +1,78 @@ +# gas.py +# Copyright (C) 2020 Maker Ecosystem Growth Holdings, INC. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from typing import Optional + +from pygasprice_client.aggregator import Aggregator +from src.gas_strategies import GasStrategy, GeometricGasPrice, FixedGasPrice, DefaultGasPrice + + +class SmartGasPrice(GasStrategy): + """Simple and smart gas price scenario. + + Uses an EtherscanAPI feed. start with safe low, move to standard after 120 seconds + then just do standard * 2. Falls back to a default scenario + (incremental as well) if the EtherscanAPI feed unavailable for more than 10 minutes. + """ + + GWEI = 1000000000 + + def __init__(self, api_key: None, blocknative_api_key=None): + self.etherscan = Aggregator(refresh_interval=60, expiry=600, + etherscan_api_key=api_key, + blocknative_api_key=blocknative_api_key) + + # if etherscan retruns None 3x in a row, try the next api + + + def get_gas_price(self, time_elapsed: int) -> Optional[int]: + # start with standard price plus backup in case EtherscanAPI is down, then do fast + if 0 <= time_elapsed <= 240: + standard_price = self.etherscan.standard_price() + if standard_price is not None: + return int(standard_price*1.1) + else: + return self.default_gas_pricing(time_elapsed) + + # move to fast after 240 seconds + if time_elapsed > 240: + fast_price = self.etherscan.fast_price() + if fast_price is not None: + return int(fast_price*1.1) + else: + return self.default_gas_pricing(time_elapsed) + + # default gas pricing when EtherscanAPI feed is down + def default_gas_pricing(self, time_elapsed: int): + return GeometricGasPrice(initial_price=5*self.GWEI, + increase_by=10*self.GWEI, + every_secs=60, + max_price=100*self.GWEI).get_gas_price(time_elapsed) + + +class GasPriceFactory: + @staticmethod + def create_gas_price(arguments, web3) -> GasStrategy: + if arguments.smart_gas_price: # --smart-gas-price + print("Executing smart_gas_price option from gas factory") + return SmartGasPrice(arguments.etherscan_api_key) + elif arguments.gas_price: # --gas-price + print("Executing fixed gas price option from gas strategies") + return FixedGasPrice(arguments.gas_price) + else: + print("Executing GeometricGasPrice gas price option from gas strategies") + # returns max_price, tip for new transactions + return GeometricGasPrice(web3, initial_price=None, initial_tip=1000000000, every_secs=60).get_gas_fees(120) diff --git a/src/gas_strategies.py b/src/gas_strategies.py new file mode 100644 index 0000000..f9e2e1d --- /dev/null +++ b/src/gas_strategies.py @@ -0,0 +1,266 @@ +# This file is part of Maker Keeper Framework. +# +# Copyright (C) 2017-2021 reverendus, EdNoepel +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import math +from pprint import pformat +from typing import Optional, Tuple +from web3 import Web3 + + +class GasStrategy(object): + GWEI = 1000000000 + + """Abstract class, which can be inherited for implementing different gas price strategies. + + To build custom gas price strategies, override methods within such that gas fees returned + increase over time. The piece of code responsible for sending Ethereum transactions + (please see :py:class:`pymaker.Transact`) will in this case overwrite the transaction + with another one, using the same `nonce` but increasing gas price. If the value returned + by `get_gas_price` does not go up, no new transaction gets submitted to the network. + + An example custom gas price strategy my be: start with 10 GWei. If transaction has not been + confirmed within 10 minutes, try again with 15 GWei. If still no confirmation, increase + to 30 GWei and then wait indefinitely for confirmation. + """ + + def get_gas_price(self, time_elapsed: int) -> Optional[int]: + """Return gas price applicable for type 0 transactions. + + Bear in mind that Geth requires the gas price for overwritten transactions to increase by at + least 10%, while OpenEthereum requires a gas price bump of 12.5%. Also, you may return + `None` which will make the node use the default gas price, but once you returned + a numeric value (gas price in Wei), you shouldn't switch back to `None` as such + transaction will likely not get overwritten. + + Args: + time_elapsed: Number of seconds since this specific Ethereum transaction + has been originally sent for the first time. + + Returns: + Gas price in Wei, or `None` if default gas price should be used. Default gas price + means it's the Ethereum node the keeper is connected to will decide on the gas price. + """ + raise NotImplementedError("Please implement this method") + + def get_gas_fees(self, time_elapsed: int) -> Tuple[int, int]: + """Return max fee (fee cap) and priority fee (tip) for type 2 (EIP-1559) transactions. + + Note that Web3 currently requires specifying both `maxFeePerGas` and `maxPriorityFeePerGas` on a type 2 + transaction. This is inconsistent with the EIP-1559 spec. + + Args: + time_elapsed: Number of seconds since this specific Ethereum transaction + has been originally sent for the first time. + + Returns: + Gas price in Wei, or `None` if default gas price should be used. Default gas price + means it's the Ethereum node the keeper is connected to will decide on the gas price. + """ + raise NotImplementedError("Please implement this method") + + def __repr__(self): + return f"{__name__}({pformat(vars(self))})" + + +class DefaultGasPrice(GasStrategy): + """Default gas price. + + Uses the default gas price i.e. gas price will be decided by the Ethereum node + the keeper is connected to. + """ + + def get_gas_price(self, time_elapsed: int) -> Optional[int]: + return None + + def get_gas_fees(self, time_elapsed: int) -> Optional[Tuple[int, int]]: + return None, None + + +class NodeAwareGasStrategy(GasStrategy): + """Abstract baseclass which is Web3-aware. + + Retrieves the default gas price provided by the Ethereum node to be consumed by subclasses. + """ + + def __init__(self, web3: Web3): + assert isinstance(web3, Web3) + if self.__class__ == NodeAwareGasStrategy: + raise NotImplementedError('This class is not intended to be used directly') + self.web3 = web3 + + def get_gas_price(self, time_elapsed: int) -> Optional[int]: + """If user wants node to choose gas price, they should use DefaultGasPrice for the same functionality + without an additional HTTP request. This baseclass exists to let a subclass manipulate the node price.""" + raise NotImplementedError("Please implement this method") + + def get_gas_fees(self, time_elapsed: int) -> Optional[Tuple[int, int]]: + """Implementation of tip is subjective. For August 2021, the following implementation is a reasonable example: + return int(self.get_next_base_fee(self)*1.5), 2 * self.GWEI""" + raise NotImplementedError("Please implement this method") + + def get_node_gas_price(self) -> int: + return max(self.web3.manager.request_blocking("eth_gasPrice", []), 1) + + def get_base_fee(self) -> Optional[int]: + """Useful for calculating maxfee; a multiple of this value is suggested""" + pending_block = self.web3.eth.getBlock('pending') + + if 'baseFeePerGas' in pending_block: + return max(int(pending_block['baseFeePerGas'], 0), 1) + else: + return None + + +class FixedGasPrice(GasStrategy): + """Fixed gas price. + + Uses specified gas price instead of the default price suggested by the Ethereum + node the keeper is connected to. The gas price may be later changed (while the transaction + is still in progress) by calling the `update_gas_price` method. + + Attributes: + gas_price: Gas price to be used (in Wei) for legacy transactions + max_fee: Maximum fee (in Wei) for EIP-1559 transactions, should be >= (base_fee + tip) + tip: Priority fee (in Wei) for EIP-1559 transactions + """ + def __init__(self, gas_price: Optional[int], max_fee: Optional[int] = None, tip: Optional[int] = None): + assert isinstance(gas_price, int) or gas_price is None + assert isinstance(max_fee, int) or max_fee is None + assert isinstance(tip, int) or tip is None + assert gas_price or (max_fee and tip) + self.gas_price = gas_price + self.max_fee = max_fee + self.tip = tip + + def update_gas_price(self, new_gas_price: int, new_max_fee: int, new_tip: int): + """Changes the initial gas price to a higher value, preferably higher. + + The only reason when calling this function makes sense is when an async transaction is in progress. + In this case, the loop waiting for the transaction to be mined (see :py:class:`pymaker.Transact`) + will resend the pending transaction again with the new gas price. + + As OpenEthereum excepts the gas price to rise by at least 12.5% in replacement transactions, the price + argument supplied to this method should be accordingly higher. + + Args: + new_gas_price: New gas price to be set (in Wei). + new_max_fee: New maximum fee (in Wei) appropriate for subsequent block(s). + new_tip: New prioritization fee (in Wei). + """ + assert isinstance(new_gas_price, int) or new_gas_price is None + assert isinstance(new_max_fee, int) or new_max_fee is None + assert isinstance(new_tip, int) or new_tip is None + assert new_gas_price or (new_max_fee and new_tip) + self.gas_price = new_gas_price + self.max_fee = new_max_fee + self.tip = new_tip + + def get_gas_price(self, time_elapsed: int) -> Optional[int]: + return self.gas_price + + def get_gas_fees(self, time_elapsed: int) -> Optional[Tuple[int, int]]: + return self.max_fee, self.tip + + +class GeometricGasPrice(NodeAwareGasStrategy): + """Geometrically increasing gas price. + + Start with `initial_price`, then increase it every 'every_secs' seconds by a fixed coefficient. + Coefficient defaults to 1.125 (12.5%), the minimum increase for OpenEthereum to replace a transaction. + Coefficient can be adjusted, and there is an optional upper limit. + + To disable legacy (type 0) transactions, set initial_price None. + To disable EIP-1559 (type 2) transactions, set initial_tip None. + Other parameters apply to both transaction types. + + Attributes: + initial_price: The initial gas price in Wei, used only for legacy transactions. + initial_tip: Initial priority fee paid on top of a base fee (recommend 1 GWEI minimum). + every_secs: Gas increase interval (in seconds). + coefficient: Gas price and tip multiplier, defaults to 1.125. + max_price: Optional upper limit and fee cap, defaults to None. + """ + def __init__(self, web3: Web3, initial_price: Optional[int], initial_tip: Optional[int], + every_secs: int, coefficient=1.125, max_price: Optional[int] = None): + assert isinstance(web3, Web3) + assert (isinstance(initial_price, int) and initial_price > 0) or initial_price is None + assert isinstance(initial_tip, int) or initial_tip is None + assert initial_price or (initial_tip is not None and initial_tip > 0) + assert isinstance(every_secs, int) + assert isinstance(coefficient, float) + assert (isinstance(max_price, int) and max_price > 0) or max_price is None + assert every_secs > 0 + assert coefficient > 1 + if initial_price and max_price: + assert initial_price <= max_price + if initial_tip and max_price: + assert initial_tip < max_price + super().__init__(web3) + + self.initial_price = initial_price + self.initial_tip = initial_tip + self.every_secs = every_secs + self.coefficient = coefficient + self.max_price = max_price + + def scale_by_time(self, value: int, time_elapsed: int) -> int: + assert isinstance(value, int) + assert isinstance(time_elapsed, int) + result = value + if time_elapsed >= self.every_secs: + for second in range(math.floor(time_elapsed / self.every_secs)): + result *= self.coefficient + return math.ceil(result) + + def get_gas_price(self, time_elapsed: int) -> Optional[int]: + assert isinstance(time_elapsed, int) + if not self.initial_price: + return None + + result = self.scale_by_time(self.initial_price, time_elapsed) + if self.max_price is not None: + result = min(result, self.max_price) + + return result + + def get_gas_fees(self, time_elapsed: int) -> Optional[Tuple[int, int]]: + assert isinstance(time_elapsed, int) + if not self.initial_tip: + return None, None + + base_fee = self.get_base_fee() + if not base_fee: + raise RuntimeError("Node does not provide baseFeePerGas; type 2 transactions are not available") + + tip = self.scale_by_time(self.initial_tip, time_elapsed) + + # This is how it should work, but doesn't; read more here: https://github.com/ethereum/go-ethereum/issues/23311 + if self.max_price: + # If the scaled tip would exceed our fee cap, reduce tip to largest possible + if base_fee + tip > self.max_price: + tip = max(0, self.max_price - base_fee) + # Honor the max_price, even if it does not exceed base fee + return self.max_price, tip + else: + # If not limited by user, set a standard fee cap of twice the base fee with tip included + return (base_fee * 2) + tip, tip + + # # HACK: Ensure both feecap and tip are scaled, satisfying geth's current replacement logic. + # feecap = self.scale_by_time(int(base_fee * 1.2), time_elapsed) + tip + # if self.max_price and feecap > self.max_price: + # feecap = self.max_price + # return feecap, min(tip, feecap) diff --git a/tests/conftest.py b/tests/conftest.py index 83950a7..4326834 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -107,7 +107,7 @@ def mcd(web3) -> DssDeployment: @pytest.fixture(scope="session") def keeper(mcd: DssDeployment, keeper_address: Address) -> CageKeeper: - keeper = CageKeeper(args=args(f"--eth-from {keeper_address} --network testnet --vat-deployment-block {1}"), web3=mcd.web3) + keeper = CageKeeper(args=args(f"--eth-from {keeper_address} --network testnet --vat-deployment-block {1} --smart-gas-price"), web3=mcd.web3) assert isinstance(keeper, CageKeeper) return keeper diff --git a/tests/test_cageKeeper.py b/tests/test_cageKeeper.py index fec5b9f..e522946 100644 --- a/tests/test_cageKeeper.py +++ b/tests/test_cageKeeper.py @@ -31,7 +31,8 @@ from pymaker.approval import directly, hope_directly from pymaker.auctions import Flapper, Flopper, Flipper from pymaker.deployment import DssDeployment -from pymaker.dss import Collateral, Ilk, Urn +from pymaker.dss import Ilk, Urn +from pymaker.collateral import Collateral from pymaker.numeric import Wad, Ray, Rad from pymaker.shutdown import ShutdownModule, End