Tech 1461

.github/workflows/tests.yaml

@@ -13,6 +13,13 @@ jobs:
       with:
         submodules: recursive
 
+    - name: Checkout master of submodules
+      run: |
+        cd lib/auction-keeper && git checkout master
+        cd ../pygasprice-client && git checkout master
+        cd ../pymaker && git checkout master
+        cd ../..
+
     - name: setup python
       uses: actions/setup-python@v4
       with:

DOCKER.md

@@ -13,6 +13,7 @@ git clone [email protected]:makerdao/cage-keeper.git
 cd cage-keeper
 git submodule update --init --recursive
 ```
+Note: To get gas prices from Etherscan or Blocknative, checkout the master branch code for the auction-keeper, and pygasprice_client submodules
 
 ## Configure, Build and Run:
 
@@ -31,11 +32,8 @@ BLOCKCHAIN_NETWORK=
 # Account used to pay for gas
 ETH_FROM_ADDRESS=
 
-# URL of Vulcanize instance to use
-VULCANIZE_URL=
-
 # ETH Gas Station API key
-ETH_GASSTATION_API_KEY=
+ETHERSCAN_API_KEY=
 
 # For ease of use, do not change the location of ETH account keys, note that account files should always be placed in the secrets directory of the cage-keeper, and files named as indicated.
 ETH_ACCOUNT_KEY='key_file=/opt/keeper/cage-keeper/secrets/keystore.json,pass_file=/opt/keeper/cage-keeper/secrets/password.txt'

Dockerfile

@@ -5,13 +5,16 @@ RUN groupadd -r keeper && useradd -d /home/keeper -m --no-log-init -r -g keeper
     apt-get -y install jq bc && \
     apt-get clean && rm -rf /var/lib/apt/lists/*
 
-WORKDIR /opt/keeper
+COPY bin /opt/keeper/cage-keeper/bin
+COPY lib /opt/keeper/cage-keeper/lib
+COPY src /opt/keeper/cage-keeper/src
+COPY install.sh /opt/keeper/cage-keeper/install.sh
+COPY run-cage-keeper.sh /opt/keeper/cage-keeper/run-cage-keeper.sh
+COPY requirements.txt /opt/keeper/cage-keeper/requirements.txt
 
-RUN git clone https://github.com/makerdao/cage-keeper.git && \
-    cd cage-keeper && \
-    git submodule update --init --recursive && \
-    pip3 install virtualenv && \
+WORKDIR /opt/keeper/cage-keeper
+RUN pip3 install virtualenv && \
     ./install.sh
 
-WORKDIR /opt/keeper/cage-keeper
+
 CMD ["./run-cage-keeper.sh"]

README.md

@@ -76,6 +76,7 @@ cd cage-keeper
 git submodule update --init --recursive
 ./install.sh
 ```
+Note: To get gas prices from Etherscan or Blocknative, checkout the master branch code for the auction-keeper, and pygasprice_client submodules
 
 For some known Ubuntu and macOS issues see the [pymaker](https://github.com/makerdao/pymaker) README.
 

requirements-dev.txt

@@ -1,5 +1,5 @@
 attrs == 19.1.0
-codecov == 2.0.9
+codecov == 2.1.13
 mock == 2.0.0
 pytest == 3.3.0
 pytest-asyncio == 0.8.0

run-cage-keeper.sh

@@ -34,12 +34,11 @@ then
 
 fi
 
-
+# remove the --smart-gas-price flag to get gas prices from the node
 exec $dir/bin/cage-keeper \
   --rpc-host "${SERVER_ETH_RPC_HOST}" \
   --network "${BLOCKCHAIN_NETWORK}" \
   --eth-from "${ETH_FROM_ADDRESS}" \
   --eth-key "${ETH_ACCOUNT_KEY}" \
-  --vulcanize-endpoint "${VULCANIZE_URL}" \
-  --vulcanize-key "${VULCANIZE_KEY}" \
-  --ethgasstation-api-key "${ETH_GASSTATION_API_KEY}"
+  --etherscan-api-key "${ETHERSCAN_API_KEY}" \
+  --smart-gas-price

src/cage_keeper.py

@@ -27,7 +27,6 @@
 from web3 import Web3
 
 from pymaker import Address, web3_via_http
-from pymaker.gas import DefaultGasPrice, FixedGasPrice
 from pymaker.auctions import Flipper, Flapper, Flopper
 from pymaker.keys import register_keys
 from pymaker.lifecycle import Lifecycle
@@ -36,8 +35,9 @@
 from pymaker.deployment import DssDeployment
 from pymaker.dss import Ilk, Urn
 
-from auction_keeper.urn_history import UrnHistory
-from auction_keeper.gas import DynamicGasPrice
+from auction_keeper.urn_history import ChainUrnHistoryProvider
+
+from src.gas_factory import GasPriceFactory
 
 class CageKeeper:
     """Keeper to facilitate Emergency Shutdown"""
@@ -73,25 +73,19 @@ def __init__(self, args: list, **kwargs):
         parser.add_argument("--vat-deployment-block", type=int, required=False, default=0,
                             help=" Block that the Vat from dss-deployment-file was deployed at (e.g. 8836668")
 
-        parser.add_argument("--vulcanize-endpoint", type=str,
-                            help="When specified, frob history will be queried from a VulcanizeDB lite node, "
-                                 "reducing load on the Ethereum node for Vault query")
-
-        parser.add_argument("--vulcanize-key", type=str,
-                            help="API key for the Vulcanize endpoint")
-
         parser.add_argument("--max-errors", type=int, default=100,
                             help="Maximum number of allowed errors before the keeper terminates (default: 100)")
 
         parser.add_argument("--debug", dest='debug', action='store_true',
                             help="Enable debug output")
 
-        parser.add_argument("--ethgasstation-api-key", type=str, default=None, help="ethgasstation API key")
+        parser.add_argument("--etherscan-api-key", type=str, default=None, help="etherscan API key")
 
-        parser.add_argument("--gas-initial-multiplier", type=str, default=1.0, help="ethgasstation API key")
-        parser.add_argument("--gas-reactive-multiplier", type=str, default=2.25, help="gas strategy tuning")
-        parser.add_argument("--gas-maximum", type=str, default=5000, help="gas strategy tuning")
+        parser.add_argument('--gas-price', type=float, default=None,
+                            help="Uses a fixed value (in Gwei) instead of an external API to determine initial gas")
 
+        parser.add_argument("--smart-gas-price", dest='smart_gas_price', action='store_true',
+                            help="Use smart gas pricing strategy, based on the ethgasstation.info feed")
 
 
         parser.set_defaults(cageFacilitated=False)
@@ -118,11 +112,7 @@ def __init__(self, args: list, **kwargs):
 
         self.confirmations = 0
 
-        # Create gas strategy
-        if self.arguments.ethgasstation_api_key:
-            self.gas_price = DynamicGasPrice(self.arguments, self.web3)
-        else:
-            self.gas_price = DefaultGasPrice()
+        self.gas_price = GasPriceFactory().create_gas_price(self.arguments, self.web3)
 
 
         logging.basicConfig(format='%(asctime)-15s %(levelname)-8s %(message)s',
@@ -154,6 +144,7 @@ def check_deployment(self):
         self.logger.info(f'Jug: {self.dss.jug.address}')
         self.logger.info(f'End: {self.dss.end.address}')
         self.logger.info('')
+        self.logger.info(f'gas price is: {self.gas_price}')
 
 
     def process_block(self):
@@ -278,12 +269,10 @@ def get_underwater_urns(self, ilks: List) -> List[Urn]:
 
         for ilk in ilks:
 
-            urn_history = UrnHistory(self.web3,
+            urn_history = ChainUrnHistoryProvider(self.web3,
                                      self.dss,
                                      ilk,
-                                     self.deployment_block,
-                                     self.arguments.vulcanize_endpoint,
-                                     self.arguments.vulcanize_key)
+                                     self.deployment_block)
 
             urns = urn_history.get_urns()
 
@@ -356,4 +345,5 @@ def yank_auctions(self, flapBids: List, flopBids: List):
 
 
 if __name__ == '__main__':
+    logging.basicConfig(format='%(asctime)-15s %(levelname)-8s %(message)s', level=logging.INFO)
     CageKeeper(sys.argv[1:]).main()

src/gas_factory.py

@@ -0,0 +1,78 @@
+# gas.py
+# Copyright (C) 2020 Maker Ecosystem Growth Holdings, INC.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+from typing import Optional
+
+from pygasprice_client.aggregator import Aggregator
+from src.gas_strategies import GasStrategy, GeometricGasPrice, FixedGasPrice, DefaultGasPrice
+
+
+class SmartGasPrice(GasStrategy):
+    """Simple and smart gas price scenario.
+
+    Uses an EtherscanAPI feed. start with safe low, move to standard after 120 seconds
+    then just do standard * 2. Falls back to a default scenario
+    (incremental as well) if the EtherscanAPI feed unavailable for more than 10 minutes.
+    """
+
+    GWEI = 1000000000
+
+    def __init__(self, api_key: None, blocknative_api_key=None):
+        self.etherscan = Aggregator(refresh_interval=60, expiry=600,
+                                      etherscan_api_key=api_key,
+                                      blocknative_api_key=blocknative_api_key)
+
+    # if etherscan retruns None 3x in a row, try the next api
+
+
+    def get_gas_price(self, time_elapsed: int) -> Optional[int]:
+        # start with standard price plus backup in case EtherscanAPI is down, then do fast
+        if 0 <= time_elapsed <= 240:
+            standard_price = self.etherscan.standard_price()
+            if standard_price is not None:
+                return int(standard_price*1.1)
+            else:
+                return self.default_gas_pricing(time_elapsed)
+
+        # move to fast after 240 seconds
+        if time_elapsed > 240:
+            fast_price = self.etherscan.fast_price()
+            if fast_price is not None:
+                return int(fast_price*1.1)
+            else:
+                return self.default_gas_pricing(time_elapsed)
+
+    # default gas pricing when EtherscanAPI feed is down
+    def default_gas_pricing(self, time_elapsed: int):
+        return GeometricGasPrice(initial_price=5*self.GWEI,
+                                  increase_by=10*self.GWEI,
+                                  every_secs=60,
+                                  max_price=100*self.GWEI).get_gas_price(time_elapsed)
+
+
+class GasPriceFactory:
+    @staticmethod
+    def create_gas_price(arguments, web3) -> GasStrategy:
+        if arguments.smart_gas_price: # --smart-gas-price
+            print("Executing smart_gas_price option from gas factory")
+            return SmartGasPrice(arguments.etherscan_api_key)
+        elif arguments.gas_price: # --gas-price
+            print("Executing fixed gas price option from gas strategies")
+            return FixedGasPrice(arguments.gas_price)
+        else:
+            print("Executing GeometricGasPrice gas price option from gas strategies")
+            # returns max_price, tip for new transactions
+            return GeometricGasPrice(web3, initial_price=None, initial_tip=1000000000, every_secs=60).get_gas_fees(120)