diff --git a/mail_client/mail_client/doctype/dmarc_report/__init__.py b/mail_client/mail_client/doctype/dmarc_report/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/mail_client/mail_client/doctype/dmarc_report/dmarc_report.js b/mail_client/mail_client/doctype/dmarc_report/dmarc_report.js new file mode 100644 index 00000000..7e7dfba7 --- /dev/null +++ b/mail_client/mail_client/doctype/dmarc_report/dmarc_report.js @@ -0,0 +1,8 @@ +// Copyright (c) 2024, Frappe Technologies Pvt. Ltd. and contributors +// For license information, please see license.txt + +// frappe.ui.form.on("DMARC Report", { +// refresh(frm) { + +// }, +// }); diff --git a/mail_client/mail_client/doctype/dmarc_report/dmarc_report.json b/mail_client/mail_client/doctype/dmarc_report/dmarc_report.json new file mode 100644 index 00000000..4c2f56c1 --- /dev/null +++ b/mail_client/mail_client/doctype/dmarc_report/dmarc_report.json @@ -0,0 +1,147 @@ +{ + "actions": [], + "autoname": "hash", + "creation": "2024-11-19 11:38:22.846485", + "doctype": "DocType", + "engine": "InnoDB", + "field_order": [ + "section_break_nb25", + "domain_name", + "policy_published", + "column_break_kyq2", + "report_id", + "organization", + "email", + "extra_contact_info", + "section_break_bgcw", + "from_date", + "column_break_lztn", + "to_date", + "section_break_prwz", + "records" + ], + "fields": [ + { + "fieldname": "section_break_nb25", + "fieldtype": "Section Break" + }, + { + "fieldname": "organization", + "fieldtype": "Data", + "in_list_view": 1, + "in_standard_filter": 1, + "label": "Organization", + "no_copy": 1, + "read_only": 1 + }, + { + "fieldname": "email", + "fieldtype": "Data", + "in_list_view": 1, + "in_standard_filter": 1, + "label": "Email", + "no_copy": 1, + "read_only": 1 + }, + { + "fieldname": "from_date", + "fieldtype": "Datetime", + "in_list_view": 1, + "in_standard_filter": 1, + "label": "From Date", + "no_copy": 1, + "read_only": 1 + }, + { + "fieldname": "to_date", + "fieldtype": "Datetime", + "in_list_view": 1, + "in_standard_filter": 1, + "label": "To Date", + "no_copy": 1, + "read_only": 1 + }, + { + "fieldname": "domain_name", + "fieldtype": "Link", + "in_list_view": 1, + "in_standard_filter": 1, + "label": "Domain Name", + "no_copy": 1, + "options": "Mail Domain", + "read_only": 1, + "search_index": 1 + }, + { + "fieldname": "column_break_kyq2", + "fieldtype": "Column Break" + }, + { + "fieldname": "section_break_bgcw", + "fieldtype": "Section Break" + }, + { + "fieldname": "column_break_lztn", + "fieldtype": "Column Break" + }, + { + "fieldname": "extra_contact_info", + "fieldtype": "Small Text", + "label": "Extra Contact Info", + "no_copy": 1, + "read_only": 1 + }, + { + "fieldname": "policy_published", + "fieldtype": "Small Text", + "label": "Policy Published", + "no_copy": 1, + "read_only": 1 + }, + { + "fieldname": "section_break_prwz", + "fieldtype": "Section Break" + }, + { + "fieldname": "records", + "fieldtype": "Table", + "label": "Records", + "no_copy": 1, + "options": "DMARC Report Detail", + "read_only": 1 + }, + { + "fieldname": "report_id", + "fieldtype": "Data", + "label": "Report ID", + "length": 255, + "unique": 1 + } + ], + "in_create": 1, + "index_web_pages_for_search": 1, + "links": [], + "modified": "2024-11-21 18:16:14.062611", + "modified_by": "Administrator", + "module": "Mail Client", + "name": "DMARC Report", + "naming_rule": "Random", + "owner": "Administrator", + "permissions": [ + { + "create": 1, + "delete": 1, + "email": 1, + "export": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "System Manager", + "share": 1, + "write": 1 + } + ], + "sort_field": "creation", + "sort_order": "DESC", + "states": [] +} \ No newline at end of file diff --git a/mail_client/mail_client/doctype/dmarc_report/dmarc_report.py b/mail_client/mail_client/doctype/dmarc_report/dmarc_report.py new file mode 100644 index 00000000..b2c1d8bb --- /dev/null +++ b/mail_client/mail_client/doctype/dmarc_report/dmarc_report.py @@ -0,0 +1,104 @@ +# Copyright (c) 2024, Frappe Technologies Pvt. Ltd. and contributors +# For license information, please see license.txt + +import json +from datetime import datetime, timezone + +import frappe +import xmltodict +from frappe.model.document import Document +from frappe.utils import cint, convert_utc_to_system_timezone, get_datetime_str + + +class DMARCReport(Document): + pass + + +def create_dmarc_report(xml_content: str) -> "DMARCReport": + """Create a DMARC Report from the given XML content.""" + + root = xmltodict.parse(xml_content) + feedback = root["feedback"] + + report_metadata = feedback["report_metadata"] + policy_published = feedback["policy_published"] + records = feedback["record"] + + doc = frappe.new_doc("DMARC Report") + doc.organization = report_metadata["org_name"] + doc.email = report_metadata["email"] + doc.report_id = report_metadata["report_id"] + doc.extra_contact_info = report_metadata.get("extra_contact_info", "") # Optional + + date_range = report_metadata["date_range"] + doc.from_date = get_datetime_str( + convert_utc_to_system_timezone(datetime.fromtimestamp(int(date_range["begin"]), tz=timezone.utc)) + ) + doc.to_date = get_datetime_str( + convert_utc_to_system_timezone(datetime.fromtimestamp(int(date_range["end"]), tz=timezone.utc)) + ) + + doc.domain_name = policy_published["domain"] + doc.policy_published = json.dumps( + { + "adkim": policy_published["adkim"], + "aspf": policy_published["aspf"], + "p": policy_published["p"], + "sp": policy_published.get("sp", ""), # Optional + "pct": policy_published.get("pct", ""), # Optional + "np": policy_published.get("np", ""), # Optional + "fo": policy_published.get("fo", ""), # Optional + }, + indent=4, + ) + + if isinstance(records, dict): + records = [records] + + for record in records: + row = record["row"] + policy_evaluated = row["policy_evaluated"] + identifiers = record["identifiers"] + auth_results = record["auth_results"] + + source_ip = row["source_ip"] + count = row["count"] + disposition = policy_evaluated["disposition"] + dkim_result = policy_evaluated["dkim"].upper() + spf_result = policy_evaluated["spf"].upper() + header_from = identifiers["header_from"] + + results = [] + for auth_type, auth_result in auth_results.items(): + if isinstance(auth_result, dict): + auth_result = [auth_result] + + for result in auth_result: + result["auth_type"] = auth_type.upper() + result["result"] = result["result"].upper() + results.append(result) + + doc.append( + "records", + { + "source_ip": source_ip, + "count": cint(count), + "disposition": disposition, + "dkim_result": dkim_result, + "spf_result": spf_result, + "header_from": header_from, + "auth_results": json.dumps(results, indent=4), + }, + ) + + doc.flags.ignore_links = True + + try: + doc.insert(ignore_permissions=True, ignore_if_duplicate=True) + return doc + except frappe.UniqueValidationError: + frappe.log_error( + title="Duplicate DMARC Report", + message=frappe.get_traceback(with_context=True), + ) + return frappe.get_doc("DMARC Report", {"report_id": doc.report_id}) diff --git a/mail_client/mail_client/doctype/dmarc_report/test_dmarc_report.py b/mail_client/mail_client/doctype/dmarc_report/test_dmarc_report.py new file mode 100644 index 00000000..cf39a522 --- /dev/null +++ b/mail_client/mail_client/doctype/dmarc_report/test_dmarc_report.py @@ -0,0 +1,9 @@ +# Copyright (c) 2024, Frappe Technologies Pvt. Ltd. and Contributors +# See license.txt + +# import frappe +from frappe.tests.utils import FrappeTestCase + + +class TestDMARCReport(FrappeTestCase): + pass diff --git a/mail_client/mail_client/doctype/dmarc_report_detail/__init__.py b/mail_client/mail_client/doctype/dmarc_report_detail/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/mail_client/mail_client/doctype/dmarc_report_detail/dmarc_report_detail.json b/mail_client/mail_client/doctype/dmarc_report_detail/dmarc_report_detail.json new file mode 100644 index 00000000..0964bf03 --- /dev/null +++ b/mail_client/mail_client/doctype/dmarc_report_detail/dmarc_report_detail.json @@ -0,0 +1,98 @@ +{ + "actions": [], + "allow_rename": 1, + "creation": "2024-11-19 12:51:48.671062", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "section_break_nmh9", + "source_ip", + "column_break_lsuy", + "count", + "section_break_bqnt", + "disposition", + "header_from", + "column_break_q04b", + "dkim_result", + "spf_result", + "section_break_0qf4", + "auth_results" + ], + "fields": [ + { + "fieldname": "source_ip", + "fieldtype": "Data", + "in_list_view": 1, + "label": "Source IP" + }, + { + "fieldname": "count", + "fieldtype": "Int", + "in_list_view": 1, + "label": "Count" + }, + { + "fieldname": "disposition", + "fieldtype": "Data", + "label": "Disposition" + }, + { + "fieldname": "dkim_result", + "fieldtype": "Select", + "in_list_view": 1, + "label": "DKIM Result", + "options": "\nPASS\nFAIL" + }, + { + "fieldname": "spf_result", + "fieldtype": "Select", + "in_list_view": 1, + "label": "SPF Result", + "options": "\nPASS\nFAIL" + }, + { + "fieldname": "header_from", + "fieldtype": "Data", + "in_list_view": 1, + "label": "Header From" + }, + { + "fieldname": "auth_results", + "fieldtype": "JSON", + "label": "Auth Results" + }, + { + "fieldname": "section_break_nmh9", + "fieldtype": "Section Break" + }, + { + "fieldname": "column_break_lsuy", + "fieldtype": "Column Break" + }, + { + "fieldname": "section_break_bqnt", + "fieldtype": "Section Break" + }, + { + "fieldname": "column_break_q04b", + "fieldtype": "Column Break" + }, + { + "fieldname": "section_break_0qf4", + "fieldtype": "Section Break" + } + ], + "index_web_pages_for_search": 1, + "istable": 1, + "links": [], + "modified": "2024-11-21 20:57:24.194305", + "modified_by": "Administrator", + "module": "Mail Client", + "name": "DMARC Report Detail", + "owner": "Administrator", + "permissions": [], + "sort_field": "creation", + "sort_order": "DESC", + "states": [] +} \ No newline at end of file diff --git a/mail_client/mail_client/doctype/dmarc_report_detail/dmarc_report_detail.py b/mail_client/mail_client/doctype/dmarc_report_detail/dmarc_report_detail.py new file mode 100644 index 00000000..d9b81106 --- /dev/null +++ b/mail_client/mail_client/doctype/dmarc_report_detail/dmarc_report_detail.py @@ -0,0 +1,9 @@ +# Copyright (c) 2024, Frappe Technologies Pvt. Ltd. and contributors +# For license information, please see license.txt + +# import frappe +from frappe.model.document import Document + + +class DMARCReportDetail(Document): + pass diff --git a/mail_client/mail_client/report/dmarc_viewer/__init__.py b/mail_client/mail_client/report/dmarc_viewer/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/mail_client/mail_client/report/dmarc_viewer/dmarc_viewer.js b/mail_client/mail_client/report/dmarc_viewer/dmarc_viewer.js new file mode 100644 index 00000000..4fbbe5ea --- /dev/null +++ b/mail_client/mail_client/report/dmarc_viewer/dmarc_viewer.js @@ -0,0 +1,57 @@ +// Copyright (c) 2024, Frappe Technologies Pvt. Ltd. and contributors +// For license information, please see license.txt + +frappe.query_reports["DMARC Viewer"] = { + formatter(value, row, column, data, default_formatter) { + value = default_formatter(value, row, column, data); + + if (["spf_result", "dkim_result", "result"].includes(column.fieldname)) { + if (data[column.fieldname] == "PASS") { + value = "" + value + ""; + } else { + value = "" + value + ""; + } + } + + return value; + }, + + filters: [ + { + fieldname: "from_date", + label: __("From Date"), + fieldtype: "Date", + default: frappe.datetime.add_days(frappe.datetime.get_today(), -7), + reqd: 1, + }, + { + fieldname: "to_date", + label: __("To Date"), + fieldtype: "Date", + default: frappe.datetime.get_today(), + reqd: 1, + }, + { + fieldname: "name", + label: __("DMARC Report"), + fieldtype: "Link", + options: "DMARC Report", + }, + { + fieldname: "domain_name", + label: __("Domain Name"), + fieldtype: "Link", + options: "Mail Domain", + }, + { + fieldname: "organization", + label: __("Organization"), + fieldtype: "Data", + }, + { + fieldname: "report_id", + label: __("Report ID"), + fieldtype: "Data", + }, + ], +}; diff --git a/mail_client/mail_client/report/dmarc_viewer/dmarc_viewer.json b/mail_client/mail_client/report/dmarc_viewer/dmarc_viewer.json new file mode 100644 index 00000000..f09b7130 --- /dev/null +++ b/mail_client/mail_client/report/dmarc_viewer/dmarc_viewer.json @@ -0,0 +1,27 @@ +{ + "add_total_row": 0, + "columns": [], + "creation": "2024-11-21 17:31:42.807210", + "disabled": 0, + "docstatus": 0, + "doctype": "Report", + "filters": [], + "idx": 0, + "is_standard": "Yes", + "letterhead": null, + "modified": "2024-11-21 17:31:42.807210", + "modified_by": "Administrator", + "module": "Mail Client", + "name": "DMARC Viewer", + "owner": "Administrator", + "prepared_report": 0, + "ref_doctype": "DMARC Report", + "report_name": "DMARC Viewer", + "report_type": "Script Report", + "roles": [ + { + "role": "System Manager" + } + ], + "timeout": 0 +} \ No newline at end of file diff --git a/mail_client/mail_client/report/dmarc_viewer/dmarc_viewer.py b/mail_client/mail_client/report/dmarc_viewer/dmarc_viewer.py new file mode 100644 index 00000000..0d464505 --- /dev/null +++ b/mail_client/mail_client/report/dmarc_viewer/dmarc_viewer.py @@ -0,0 +1,190 @@ +# Copyright (c) 2024, Frappe Technologies Pvt. Ltd. and contributors +# For license information, please see license.txt + +import json + +import frappe +from frappe import _ +from frappe.query_builder import Order +from frappe.query_builder.functions import Date + + +def execute(filters: dict | None = None) -> tuple: + columns = get_columns() + data = get_data(filters) + + return columns, data + + +def get_columns() -> list[dict]: + return [ + { + "label": _("Name"), + "fieldname": "name", + "fieldtype": "Link", + "options": "DMARC Report", + "width": 120, + }, + { + "label": _("From Date"), + "fieldname": "from_date", + "fieldtype": "Datetime", + "width": 180, + }, + { + "label": _("To Date"), + "fieldname": "to_date", + "fieldtype": "Datetime", + "width": 180, + }, + { + "label": _("Domain Name"), + "fieldname": "domain_name", + "fieldtype": "Link", + "options": "Mail Domain", + "width": 150, + }, + { + "label": _("Organization"), + "fieldname": "organization", + "fieldtype": "Data", + "width": 150, + }, + { + "label": _("Report ID"), + "fieldname": "report_id", + "fieldtype": "Data", + "width": 150, + }, + { + "label": _("Source IP"), + "fieldname": "source_ip", + "fieldtype": "Data", + "width": 150, + }, + { + "label": _("Count"), + "fieldname": "count", + "fieldtype": "Int", + "width": 70, + }, + { + "label": _("Disposition"), + "fieldname": "disposition", + "fieldtype": "Data", + "width": 150, + }, + { + "label": _("Header From"), + "fieldname": "header_from", + "fieldtype": "Data", + "width": 150, + }, + { + "label": _("SPF Result"), + "fieldname": "spf_result", + "fieldtype": "Data", + "width": 150, + }, + { + "label": _("DKIM Result"), + "fieldname": "dkim_result", + "fieldtype": "Data", + "width": 150, + }, + { + "label": _("Auth Type"), + "fieldname": "auth_type", + "fieldtype": "Data", + "width": 150, + }, + { + "label": _("Selector / Scope"), + "fieldname": "selector_or_scope", + "fieldtype": "Data", + "width": 150, + }, + { + "label": _("Domain"), + "fieldname": "domain", + "fieldtype": "Data", + "width": 150, + }, + { + "label": _("Result"), + "fieldname": "result", + "fieldtype": "Data", + "width": 150, + }, + ] + + +def get_data(filters: dict | None = None) -> list[list]: + filters = filters or {} + + DR = frappe.qb.DocType("DMARC Report") + + query = ( + frappe.qb.from_(DR) + .select( + DR.name, + DR.from_date, + DR.to_date, + DR.domain_name, + DR.organization, + DR.report_id, + ) + .orderby(DR.from_date, order=Order.desc) + ) + + if not filters.get("name"): + query = query.where( + (Date(DR.from_date) >= Date(filters.get("from_date"))) + & (Date(DR.to_date) <= Date(filters.get("to_date"))) + ) + + for field in [ + "name", + "domain_name", + "organization", + "report_id", + ]: + if filters.get(field): + query = query.where(DR[field] == filters.get(field)) + + data = query.run(as_dict=True) + + formated_data = [] + for d in data: + records = frappe.db.get_all( + "DMARC Report Detail", + filters={"parenttype": "DMARC Report", "parent": d.name}, + fields=[ + "source_ip", + "count", + "disposition", + "header_from", + "spf_result", + "dkim_result", + "auth_results", + ], + ) + + d["indent"] = 0 + formated_data.append(d) + for record in records: + record["indent"] = 1 + formated_data.append(record) + + auth_results = json.loads(record.auth_results) + for auth_result in auth_results: + auth_result["indent"] = 2 + + if auth_result["auth_type"] == "DKIM": + auth_result["selector_or_scope"] = auth_result.get("selector") + else: + auth_result["selector_or_scope"] = auth_result.get("scope") + + formated_data.append(auth_result) + + return formated_data diff --git a/mail_client/mail_client/report/outbound_delay/outbound_delay.py b/mail_client/mail_client/report/outbound_delay/outbound_delay.py index 706b0b0b..1651dd4d 100644 --- a/mail_client/mail_client/report/outbound_delay/outbound_delay.py +++ b/mail_client/mail_client/report/outbound_delay/outbound_delay.py @@ -11,7 +11,7 @@ from mail_client.utils.user import get_user_mailboxes, has_role, is_system_manager -def execute(filters: dict | None = None) -> tuple[list, list]: +def execute(filters: dict | None = None) -> tuple: columns = get_columns() data = get_data(filters) summary = get_summary(data) diff --git a/mail_client/utils/__init__.py b/mail_client/utils/__init__.py index bc8f6fc5..84c894e6 100644 --- a/mail_client/utils/__init__.py +++ b/mail_client/utils/__init__.py @@ -1,7 +1,10 @@ +import gzip import re +import zipfile from collections.abc import Callable from datetime import datetime from email.utils import parsedate_to_datetime as parsedate +from io import BytesIO import frappe import pytz @@ -108,3 +111,38 @@ def add_or_update_tzinfo(date_time: datetime | str, timezone: str | None = None) date_time = date_time.astimezone(target_tz) return str(date_time) + + +def load_compressed_file(file_path: str | None = None, file_data: bytes | None = None) -> str | None: + """Load content from a compressed file or bytes object.""" + + if not file_path and not file_data: + frappe.throw(_("Either file path or file data is required.")) + + if file_path: + if zipfile.is_zipfile(file_path): + with zipfile.ZipFile(file_path, "r") as zip_file: + file_name = zip_file.namelist()[0] + with zip_file.open(file_name) as file: + content = file.read().decode() + return content + else: + with gzip.open(file_path, "rt") as gz_file: + return gz_file.read() + + elif file_data: + try: + with zipfile.ZipFile(BytesIO(file_data), "r") as zip_file: + file_name = zip_file.namelist()[0] + with zip_file.open(file_name) as file: + return file.read().decode() + except zipfile.BadZipFile: + pass + + try: + with gzip.open(BytesIO(file_data), "rt") as gz_file: + return gz_file.read() + except OSError: + pass + + frappe.throw(_("Failed to load content from the compressed file.")) diff --git a/pyproject.toml b/pyproject.toml index c8a46656..d2128b63 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,8 +9,9 @@ readme = "README.md" dynamic = ["version"] dependencies = [ # "frappe~=15.0.0" # Installed and managed by bench. - "uuid-utils~=0.6.1", "dkimpy~=1.1.5", + "uuid-utils~=0.6.1", + "xmltodict~=0.14.2", ] [build-system]