adding test "test_store_not_include_data" (#76)

* adding test "test_store_not_include_data"

* adding test case for wrong peer address formatting

* try restore message with wrong peer id

* send peer address with wrong protocol

* Fixing review comments & add test for wrong topic

* adding test for topic content positive scenario

* adding test for different wrong content topic

* adding content topic & pubsubtopic tests

* adding test cases for wrong encoding 7 no encoding of pubsub topic

* adding test case test_time_filter_start_time_after_end_time

* adding more time tests

* adding tests with wrong start & end time

* adding invalid ascending test

* adding page_size tests

* adding invalid ascending test

* fix minor issues and adding comments

* fixing review comments

* minor change in test name

* Adding review comments

src/node/api_clients/rest.py

@@ -93,7 +93,20 @@ def get_filter_messages(self, content_topic, pubsub_topic=None):
         return get_messages_response.json()
 
     def get_store_messages(
-        self, peer_addr, include_data, pubsub_topic, content_topics, start_time, end_time, hashes, cursor, page_size, ascending, store_v, **kwargs
+        self,
+        peer_addr,
+        include_data,
+        pubsub_topic,
+        content_topics,
+        start_time,
+        end_time,
+        hashes,
+        cursor,
+        page_size,
+        ascending,
+        store_v,
+        encode_pubsubtopic=True,
+        **kwargs,
     ):
         base_url = f"store/{store_v}/messages"
         params = []
@@ -103,7 +116,10 @@ def get_store_messages(
         if include_data is not None:
             params.append(f"includeData={include_data}")
         if pubsub_topic is not None:
-            params.append(f"pubsubTopic={quote(pubsub_topic, safe='')}")
+            if encode_pubsubtopic:
+                params.append(f"pubsubTopic={quote(pubsub_topic, safe='')}")
+            else:
+                params.append(f"pubsubTopic={pubsub_topic}")
         if content_topics is not None:
             params.append(f"contentTopics={quote(content_topics, safe='')}")
         if start_time is not None:

src/node/store_response.py

@@ -59,6 +59,16 @@ def message_hash(self, index):
         else:
             return None
 
+    def message_content(self, index):
+        try:
+            if self.messages is not None:
+                payload = self.messages[index]["message"]["contentTopic"]
+                return payload
+            else:
+                return None
+        except IndexError:
+            return None
+
     def message_payload(self, index):
         try:
             if self.messages is not None:

tests/store/test_api_flags.py

@@ -1,8 +1,14 @@
 import pytest
+from time import time
+from src.libs.custom_logger import get_custom_logger
 from src.libs.common import to_base64
 from src.node.waku_message import WakuMessage
+from src.node.store_response import StoreResponse
 from src.steps.store import StepsStore
 from src.test_data import SAMPLE_INPUTS
+from src.test_data import PUBSUB_TOPICS_STORE
+
+logger = get_custom_logger(__name__)
 
 
 @pytest.mark.usefixtures("node_setup")
@@ -11,6 +17,36 @@ def test_store_with_peerAddr(self):
         self.publish_message()
         self.check_published_message_is_stored(store_node=self.store_node1, peer_addr=self.multiaddr_list[0])
 
+    def test_store_with_wrongPeerAddr(self):
+        self.publish_message()
+        wrong_peer_addr = self.multiaddr_list[0][1:]
+        logger.debug(f"Running test with wrong_peer_addr: {wrong_peer_addr}")
+        try:
+            self.check_published_message_is_stored(store_node=self.store_node1, peer_addr=wrong_peer_addr)
+            raise Exception("Message stored with wrong peer address")
+        except Exception as ex:
+            logger.debug(f" Response with wrong peer address is { ex.args[0]}")
+            assert ex.args[0].find("Invalid MultiAddress") != -1
+        # try to send wrong peer id
+        wrong_peer_id = self.multiaddr_list[0][:-1]
+        logger.debug(f"Running test with wrong_peer_addr {wrong_peer_id}")
+        try:
+            self.check_published_message_is_stored(store_node=self.store_node1, peer_addr=wrong_peer_id)
+            raise Exception("Message restored with wrong peer id")
+        except Exception as ex:
+            logger.debug(f" Response with wrong peer id is { ex.args[0]}")
+            assert ex.args[0].find("Failed parsing remote peer info") != -1
+
+        # send address without /tcp number
+        wrong_peer_addr = self.multiaddr_list[0].replace("/tcp", "")
+        logger.debug(f"logger is {wrong_peer_addr}")
+        try:
+            self.check_published_message_is_stored(store_node=self.store_node1, peer_addr=wrong_peer_addr)
+            raise Exception("Message stored with wrong peer address")
+        except Exception as ex:
+            logger.debug(f" Response with wrong peer address is { ex.args[0]}")
+            assert ex.args[0].find("Unsupported protocol") != -1
+
     def test_store_include_data(self):
         message_list = []
         for payload in SAMPLE_INPUTS:
@@ -25,3 +61,24 @@ def test_store_include_data(self):
                 assert store_response.message_pubsub_topic(index) == self.test_pubsub_topic
                 waku_message = WakuMessage([store_response.message_at(index)])
                 waku_message.assert_received_message(message_list[index])
+
+    def test_store_not_include_data(self):
+        self.publish_message(message=self.create_message())
+        store_response = self.get_messages_from_store(self.store_node1, include_data="false")
+        logger.debug(f" Message restored with hash only is {store_response.messages} ")
+        assert "message" not in store_response.messages
+
+    def test_get_store_messages_with_wrong_typo_content_topic(self):
+        # positive scenario
+        content_topic = "/myapp/1/latest/protoo"
+        logger.debug(f"Trying to publish msg with content topic {content_topic}")
+        msg = self.publish_message(message=self.create_message(contentTopic=content_topic))
+        store_response = self.get_messages_from_store(self.store_node1, include_data="true", content_topics=content_topic)
+        try:
+            if store_response.messages is not None:
+                stored_contentTopic = store_response.message_content(0)
+                logger.debug(f"stored content topic is {stored_contentTopic}")
+                assert stored_contentTopic == content_topic, "content topics don't match"
+
+        except Exception as e:
+            raise Exception(f"can't get message with content topic {content_topic}")

tests/store/test_page_size.py

@@ -1,6 +1,9 @@
 import pytest
 from src.libs.common import to_base64
 from src.steps.store import StepsStore
+from src.libs.custom_logger import get_custom_logger
+
+logger = get_custom_logger(__name__)
 
 
 @pytest.mark.usefixtures("node_setup")
@@ -33,3 +36,31 @@ def test_different_page_size(self, page_size):
         for node in self.store_nodes:
             store_response = self.get_messages_from_store(node, page_size=page_size)
             assert len(store_response.messages) == page_size, "Message count mismatch"
+
+    def test_extreme_number_page_size(self):
+        for i in range(150):
+            self.publish_message(message=self.create_message(payload=to_base64(f"Message_{i}")))
+        for node in self.store_nodes:
+            store_response = self.get_messages_from_store(node, page_size=1000000)
+            assert len(store_response.messages) == 100, "Message count mismatch"
+
+    def test_negative_number_page_size(self):
+        page_size = -1
+        for i in range(10):
+            self.publish_message(message=self.create_message(payload=to_base64(f"Message_{i}")))
+        logger.debug(f"requesting stored message with wrong page_size = {page_size}")
+        for node in self.store_nodes:
+            store_response = self.get_messages_from_store(node, page_size=page_size)
+            assert len(store_response.messages) == 10, "Message count mismatch"
+
+    def test_invalid_page_size(self):
+        page_size = "$2"
+        for i in range(10):
+            self.publish_message(message=self.create_message(payload=to_base64(f"Message_{i}")))
+        for node in self.store_nodes:
+            try:
+                store_response = self.get_messages_from_store(node, page_size=page_size)
+                raise Exception(f"invalid page number {page_size} was acepted !!")
+            except Exception as e:
+                logger.debug(f"the invalid page_size {page_size} wasn't accepted ")
+                assert e.args[0].find("400 Client Error: Bad Request for url"), "Error logged isn't for bad url"

tests/store/test_sorting.py

@@ -1,6 +1,9 @@
 import pytest
 from src.libs.common import to_base64
 from src.steps.store import StepsStore
+from src.libs.custom_logger import get_custom_logger
+
+logger = get_custom_logger(__name__)
 
 
 @pytest.mark.usefixtures("node_setup")
@@ -21,3 +24,18 @@ def test_store_sort_ascending(self, ascending):
                 assert response_message_hash_list == expected_message_hash_list[:5], "Message hash mismatch for acending order"
             else:
                 assert response_message_hash_list == expected_message_hash_list[5:], "Message hash mismatch for descending order"
+
+    def test_store_invalid_ascending(self):
+        expected_message_hash_list = []
+        ascending = "##"
+        for i in range(4):
+            message = self.create_message(payload=to_base64(f"Message_{i}"))
+            self.publish_message(message=message)
+            expected_message_hash_list.append(self.compute_message_hash(self.test_pubsub_topic, message))
+        logger.debug(f"requesting stored messages with invalid ascending ={ascending}")
+        for node in self.store_nodes:
+            store_response = self.get_messages_from_store(node, ascending=ascending, page_size=2)
+            response_message_hash_list = []
+            for index in range(len(store_response.messages)):
+                response_message_hash_list.append(store_response.message_hash(index))
+            assert response_message_hash_list == expected_message_hash_list[:2], "pages aren't forward as expected"

tests/store/test_time_filter.py

@@ -1,3 +1,5 @@
+import time
+
 import pytest
 from datetime import timedelta, datetime
 from src.libs.custom_logger import get_custom_logger
@@ -112,3 +114,135 @@ def test_time_filter_start_time_equals_end_time(self):
             )
             assert len(store_response.messages) == 1, "Message count mismatch"
             assert store_response.message_hash(0) == message_hash_list[0], "Incorrect messaged filtered based on time"
+
+    def test_time_filter_start_time_after_end_time(self):
+        start_time = self.ts_pass[4]["value"]  # 2 sec Future
+        end_time = self.ts_pass[0]["value"]  # 3 sec past
+        for timestamp in self.ts_pass:
+            message = self.create_message(timestamp=timestamp["value"])
+            self.publish_message(message=message)
+        logger.debug(f"inquering stored messages with start time {start_time} after end time {end_time}")
+        for node in self.store_nodes:
+            store_response = self.get_messages_from_store(
+                node,
+                page_size=20,
+                start_time=start_time,
+                end_time=end_time,
+            )
+            logger.debug(f"response for wrong time message is {store_response.response}")
+            assert len(store_response.messages) == 0, "got messages with start time after end time !"
+
+        def test_time_filter_negative_start_time(self):
+            for timestamp in self.ts_pass:
+                message = self.create_message(timestamp=timestamp["value"])
+                self.publish_message(message=message)
+
+        start_time = -10000
+        logger.debug(f"inquering stored messages with start time {start_time}")
+        for node in self.store_nodes:
+            store_response = self.get_messages_from_store(node, page_size=20, start_time=start_time, include_data=True)
+            logger.debug(f"number of messages stored for  " f"start time = {start_time} is  {len(store_response.messages)}")
+
+            assert len(store_response.messages) == 6, "number of messages retrieved doesn't match time filter "
+
+    def test_time_filter_zero_start_time(self):
+        for timestamp in self.ts_pass:
+            message = self.create_message(timestamp=timestamp["value"])
+            self.publish_message(message=message)
+        start_time = 0
+        logger.debug(f"inquering stored messages with start time {start_time}")
+        for node in self.store_nodes:
+            store_response = self.get_messages_from_store(node, page_size=20, start_time=start_time, include_data=True)
+            logger.debug(f"number of messages stored for  " f"start time = {start_time} is  {len(store_response.messages)}")
+
+            assert len(store_response.messages) == 6, "number of messages retrieved doesn't match time filter "
+
+    def test_time_filter_zero_start_end_time(self):
+        for timestamp in self.ts_pass:
+            message = self.create_message(timestamp=timestamp["value"])
+            self.publish_message(message=message)
+        start_time = 0
+        end_time = 0
+        logger.debug(f"inquering stored messages with start time {start_time}")
+        for node in self.store_nodes:
+            store_response = self.get_messages_from_store(node, page_size=20, start_time=start_time, end_time=end_time, include_data=True)
+            logger.debug(f"number of messages stored for  " f"start time = {start_time} is  {len(store_response.messages)}")
+
+            assert len(store_response.messages) == 6, "number of messages retrieved doesn't match time filter "
+
+    def test_time_filter_invalid_start_time(self):
+        for timestamp in self.ts_pass:
+            message = self.create_message(timestamp=timestamp["value"])
+            self.publish_message(message=message)
+        start_time = "abc"
+        logger.debug(f"inquering stored messages with start time {start_time}")
+        try:
+            for node in self.store_nodes:
+                store_response = self.get_messages_from_store(node, page_size=20, start_time=start_time, include_data=True)
+            raise Exception(f" request for stored messages with invalid start time {start_time} is successful")
+        except Exception as e:
+            logger.debug(f"invalid start_time cause error {e}")
+            assert e.args[0].find("Bad Request for url"), "url with wrong start_time is accepted"
+
+    def test_time_filter_end_time_now(self):
+        self.ts_pass[3]["value"] = int((datetime.now() + timedelta(seconds=4)).timestamp() * 1e9)
+        start_time = self.ts_pass[0]["value"]
+        i = 0
+        for timestamp in self.ts_pass:
+            message = self.create_message(timestamp=timestamp["value"])
+            self.publish_message(message=message)
+            i += 1
+        end_time = int(datetime.now().timestamp() * 1e9)
+        logger.debug(f"inquering stored messages with start time {start_time} after end time {end_time}")
+        for node in self.store_nodes:
+            store_response = self.get_messages_from_store(node, page_size=20, start_time=start_time, end_time=end_time, include_data=True)
+            logger.debug(f"number of messages stored for start time {start_time} and " f"end time = {end_time} is  {len(store_response.messages)}")
+            assert len(store_response.messages) == 4, "number of messages retrieved doesn't match time filter "
+
+    def test_time_filter_big_timestamp(self):
+        start_time = self.ts_pass[0]["value"]
+        for timestamp in self.ts_pass:
+            message = self.create_message(timestamp=timestamp["value"])
+            self.publish_message(message=message)
+        end_time = int((datetime.now() + timedelta(days=8000)).timestamp() * 1e9)
+        logger.debug(f"inquering stored messages with start time {start_time} after end time {end_time}")
+        for node in self.store_nodes:
+            store_response = self.get_messages_from_store(node, page_size=20, start_time=start_time, end_time=end_time, include_data=True)
+            logger.debug(f"number of messages stored for start time {start_time} and " f"end time = {end_time} is  {len(store_response.messages)}")
+            assert len(store_response.messages) == 6, "number of messages retrieved doesn't match time filter "
+
+    def test_time_filter_small_timestamp(self):
+        start_time = self.ts_pass[0]["value"]
+        for timestamp in self.ts_pass:
+            message = self.create_message(timestamp=timestamp["value"])
+            self.publish_message(message=message)
+        end_time = self.ts_pass[5]["value"] + 1
+        logger.debug(f"inquering stored messages with start time {start_time} after end time {end_time}")
+        for node in self.store_nodes:
+            store_response = self.get_messages_from_store(node, page_size=20, start_time=start_time, end_time=end_time, include_data=True)
+            logger.debug(f"number of messages stored for start time {start_time} and " f"end time = {end_time} is  {len(store_response.messages)}")
+
+            assert len(store_response.messages) == 6, "number of messages retrieved doesn't match time filter "
+
+    def test_time_filter_negative_end_time(self):
+        for timestamp in self.ts_pass:
+            message = self.create_message(timestamp=timestamp["value"])
+            self.publish_message(message=message)
+        end_time = -10000
+        logger.debug(f"inquering stored messages with end time {end_time}")
+        for node in self.store_nodes:
+            store_response = self.get_messages_from_store(node, page_size=20, end_time=end_time, include_data=True)
+            logger.debug(f"number of messages stored for  " f"end time = {end_time} is  {len(store_response.messages)}")
+
+            assert len(store_response.messages) == 6, "number of messages retrieved doesn't match time filter "
+
+    def test_time_filter_zero_end_time(self):
+        for timestamp in self.ts_pass:
+            message = self.create_message(timestamp=timestamp["value"])
+            self.publish_message(message=message)
+        end_time = 0
+        logger.debug(f"inquering stored messages with end time {end_time}")
+        for node in self.store_nodes:
+            store_response = self.get_messages_from_store(node, page_size=20, end_time=end_time, include_data=True)
+            logger.debug(f"number of messages stored for  " f"end time = {end_time} is  {len(store_response.messages)}")
+            assert len(store_response.messages) == 6, "number of messages retrieved doesn't match time filter "