Avoid generating duplicate headers in ezmlm-send

When processing a message's headers:

 - Make note if Reply-To was seen.
 - Save off and do not copy the Cc header to the output immediately.
 - Save off and do not copy the From header to the output immediately.

At the end of the message headers, call rewrite_from().

When flagrewritefrom is set, either by the list's configuration or
because a DMARC reject policy applies to the message sender's address:

 - The From header is rewritten as before.
 - If replytolist is not enabled, a Reply-To header is only added
   if the original message did not have one. Rationale is that the
   original Reply-To header indicated where the post author is
   interested in receiving responses, not the From header.
 - When replytolist is enabled and the original message had a Cc header,
   the From address is appended to the existing Cc header.
 - When replytolist is enabled and the original message did not have a
   Cc header, generate a new one using the From address.

After rewrite_from() performs any of the applicable manipulations
described above, it copies the resulting From and Cc headers to the
output.

Update tests/551-ezmlm-send-rewritefrom:

 - Confirm Reply-To and Cc headers are not duplicated.
 - Validate behavior when replytolist is configured.

bin/ezmlm-send.c

@@ -81,6 +81,7 @@ static stralloc lines = {0};
 static stralloc subject = {0};
 static stralloc from = {0};
 static stralloc received = {0};
+static stralloc cc = {0};
 static stralloc prefix = {0};
 static stralloc content = {0};
 stralloc boundary = {0};
@@ -111,6 +112,7 @@ static int flagfoundokpart;		/* Found something to pass on. If multipart */
 					/* we set to 0 and then set to 1 for any */
 					/* acceptable mime part. If 0 -> reject */
 static int flagsawreceived;
+static int flaghavereplyto;
 static int flagprefixed;
 static unsigned int serial = 0;
 static int flagarchived;
@@ -318,6 +320,8 @@ static void rewrite_from()
   unsigned int at;
   int r;
 
+  stralloc_copyb(&line,"",0);
+
   /* If not unconditionally rewriting headers, turn it on for this
    * message if DMARC would prevent us from sending as-is. */
   if (!flagrewritefrom) {
@@ -347,9 +351,27 @@ static void rewrite_from()
     stralloc_catb(&line,"@",1);
     stralloc_catb(&line,outhost.s,outhost.len);
     stralloc_catb(&line,">\n",2);
-    stralloc_cats(&line,flagreplytolist ? "Cc:" : "Reply-To:");
+    if (flagreplytolist) {
+      if (cc.s) {
+        --cc.len;	/* remove '\n' */
+        stralloc_catb(&cc,",\n",2);
+      }
+      stralloc_catb(&cc,from.s,from.len);
+    } else if (!flaghavereplyto) {
+      stralloc_catb(&line,"Reply-To:",9);
+      stralloc_catb(&line,from.s,from.len);
+    }
+  } else {
+    stralloc_copyb(&line,"From:",5);
     stralloc_catb(&line,from.s,from.len);
   }
+
+  if (cc.s) {
+    stralloc_catb(&line,"Cc:",3);
+    stralloc_catb(&line,cc.s,cc.len);
+  }
+
+  stralloc_catb(&line,"\n",1);
 }
 
 int main(int argc,char **argv)
@@ -519,6 +541,7 @@ int main(int argc,char **argv)
       strerr_die2sys(111,FATAL,MSG(ERR_READ_INPUT));
     if (flaginheader && match) {
       if (line.len == 1) {		/* end of header */
+        rewrite_from();
 	flaginheader = 0;
         if (flagindexed)		/* std entry */
           r = idx_copy_insertsubject();	/* all indexed lists */
@@ -624,9 +647,14 @@ int main(int argc,char **argv)
           else if (case_startb(cp,cpafter-cp,"Quoted-Printable")) encin = 'Q';
         } else if (flaglistid && case_startb(line.s,line.len,"list-id:"))
 	  flagbadfield = 1;		/* suppress if we added our own */
-	else if (case_startb(line.s,line.len,"From:")) {
+        else if (!flagbadfield && case_startb(line.s,line.len,"Reply-To:"))
+          flaghavereplyto = 1;
+        else if (case_startb(line.s,line.len,"Cc:")) {
+          stralloc_copyb(&cc,line.s+3,line.len-3);
+          flagbadfield = 1;		/* written/adjusted by rewrite_from() */
+        } else if (case_startb(line.s,line.len,"From:")) {
 	  stralloc_copyb(&from,line.s+5,line.len-5);
-	  rewrite_from();
+          flagbadfield = 1;		/* written/adjusted by rewrite_from() */
         } else if (line.len == mydtline.len)
 	  if (!byte_diff(line.s,line.len,mydtline.s))
             strerr_die2x(100,FATAL,MSG(ERR_LOOPING));

tests/02-functions

@@ -22,6 +22,16 @@ fatal() {
   exit 100;
 }
 
+unfoldhdrs() {
+  (cat "$QQHDR"; echo) |
+  sed -e '/^\([^ \t]\|$\)/x'            -e '# new header; swap hold/pattern'   \
+      -e '/^$/d'                        -e '# ignore initial empty hold'       \
+      -e '/^[ \t]/{' -e H -e d -e }     -e '# append continuation to hold'     \
+      -e 's/[ \t]*\n[ \t]*/ /g'         -e '# unfold concatenated header'      \
+    >"${TMP}hdr"
+  mv -f "${TMP}hdr" "$QQHDR"
+}
+
 grephdr() {
   # Search for the header line, and produce an error if it didn't match.
   egrep "^$*$" "$QQHDR" >/dev/null 2>&1 || \
@@ -36,6 +46,17 @@ grephdr() {
   mv -f "${TMP}hdr" "$QQHDR"
 }
 
+singlehdr() {
+  # Search for the header and ensure it appears exactly once.
+  count=$(grep -i -c "^$1" "$QQHDR")
+  if [ "$count" -ne 1 ]; then
+    [ "$count" -eq 0 ] && echo "Missing $1 line:"
+    [ "$count" -gt 1 ] && echo "Multiple $1 lines:" && grep -i "^$1" "$QQHDR"
+    BUG="${BUG} headers"
+    prompt "..............: "
+  fi
+}
+
 grepbody() {
   egrep "^$*$" "$QQBODY" >/dev/null 2>&1 || \
   {

tests/550-ezmlm-send

@@ -1,9 +1,11 @@
   prompt "ezmlm-send:           "
 
   sendfrom() {
-    cat >"$TMP" <<EOF
-From: $1
-Reply-To: $1
+    echo "From: $1" >"$TMP"
+    if [ -n "$2" ]; then
+        echo "Reply-To: $2" >>"$TMP"
+    fi
+    cat >>"$TMP" <<EOF
 Subject: test post
 X-Test-Header: one
 Content-Length: zip
@@ -26,7 +28,6 @@ EOF
   grephdr Subject: '\[PFX\] test post'
   grephdr X-Test-Header: one
   grephdr Sender: "<${LIST}@${HOST}>"
-  grephdr Reply-To: [email protected]
   grephdr_empty
 
   grepbody Local: "$LOCAL"
@@ -40,7 +41,7 @@ EOF
   grephdr Subject: 'test post'
 
   touch "$DIR"/replytolist
-  sendfrom [email protected]
+  sendfrom [email protected] [email protected]
 
   grephdr_list 1
   grephdr Precedence: bulk

tests/551-ezmlm-send-rewritefrom

@@ -4,46 +4,112 @@ touch "$DIR"/rewritefrom
 
 sendfrom '"My Name 1" <[email protected]>'
 grephdr From: "\"My Name 1\" via ${LIST} <${LIST}@${HOST}>$"
+singlehdr Reply-To:
 grephdr Reply-To: '"My Name 1" <[email protected]>'
 
 sendfrom '"My Name 2" [email protected]'
 grephdr From: "\"My Name 2\" via ${LIST} <${LIST}@${HOST}>$"
+singlehdr Reply-To:
 grephdr Reply-To: '"My Name 2" [email protected]'
 
 sendfrom 'My Name 3 <[email protected]>'
 grephdr From: "My Name 3 via ${LIST} <${LIST}@${HOST}>$"
+singlehdr Reply-To:
 grephdr Reply-To: 'My Name 3 <[email protected]>'
 
 sendfrom '[email protected] (My Name 4)'
 grephdr From: "\"My Name 4\" via ${LIST} <${LIST}@${HOST}>$"
+singlehdr Reply-To:
 grephdr Reply-To: '[email protected] \(My Name 4\)'
 
 sendfrom '[email protected]'
 grephdr From: "\"[email protected]\" via ${LIST} <${LIST}@${HOST}>$"
+singlehdr Reply-To:
 grephdr Reply-To: '[email protected]'
 
 rm -f "$DIR"/rewritefrom
 
 sendfrom '[email protected]'
 grephdr From: "\"[email protected]\" via ${LIST} <${LIST}@${HOST}>$"
+singlehdr Reply-To:
 grephdr Reply-To: '[email protected]'
 
 sendfrom '"My Name 7" <[email protected]>'
 grephdr From: "\"My Name 7\" via ${LIST} <${LIST}@${HOST}>$"
+singlehdr Reply-To:
 grephdr Reply-To: '"My Name 7" <[email protected]>'
 
 sendfrom 'My Name 8 <[email protected]>'
 grephdr From: "My Name 8 via ${LIST} <${LIST}@${HOST}>$"
+singlehdr Reply-To:
 grephdr Reply-To: 'My Name 8 <[email protected]>'
 
 touch "$DIR"/rewritefrom
 
 sendfrom '=?iso-8859-1?Q?My=20N=E4m=E9=209?= <[email protected]>'
 grephdr From: "=\\?iso-8859-1\\?Q\\?My=20N=E4m=E9=209\\?= via ${LIST} <${LIST}@${HOST}>$"
+singlehdr Reply-To:
 grephdr Reply-To: '=\?iso-8859-1\?Q\?My=20N=E4m=E9=209\?= <[email protected]>'
 
 sendfrom '=?utf-8?B?77yt772ZIMORw6ZtIDEw?= <[email protected]>'
 grephdr From: "=\\?utf-8\\?B\\?77yt772ZIMORw6ZtIDEw\\?= via ${LIST} <${LIST}@${HOST}>$"
+singlehdr Reply-To:
 grephdr Reply-To: '=\?utf-8\?B\?77yt772ZIMORw6ZtIDEw\?= <[email protected]>'
 
+sendfrom 'My Name 11 <[email protected]>' 'Reply To 11 <[email protected]>'
+grephdr From: "My Name 11 via ${LIST} <${LIST}@${HOST}>$"
+singlehdr Reply-To:
+grephdr Reply-To: 'Reply To 11 <[email protected]>'
+
+sendfrom [email protected] [email protected]
+grephdr From: "\"[email protected]\" via ${LIST} <${LIST}@${HOST}>$"
+singlehdr Reply-To:
+grephdr Reply-To: '[email protected]'
+
+touch "$DIR"/replytolist
+
+sendfrom '"My Name 13" <[email protected]>'
+grephdr From: "\"My Name 13\" via ${LIST} <${LIST}@${HOST}>$"
+singlehdr Reply-To:
+grephdr Reply-To: "<${LIST}@${HOST}>"
+singlehdr Cc:
+grephdr Cc: '"My Name 13" <[email protected]>'
+
+cat >"$TMP" <<EOF
+From: [email protected]
+To: ${LIST}@${HOST}
+Cc: [email protected]
+Subject: carbon-copied test post
+
+message goes here
+EOF
+${EZBIN}/ezmlm-send "$DIR" <"$TMP" >"$ERR" 2>&1 || \
+    fatal "failed to produce post"
+unfoldhdrs
+grephdr From: "\"[email protected]\" via ${LIST} <${LIST}@${HOST}>$"
+singlehdr Reply-To:
+grephdr Reply-To: "<${LIST}@${HOST}>"
+singlehdr Cc:
+grephdr Cc: '[email protected], [email protected]'
+
+cat >"$TMP" <<EOF
+From: My Test 15 <[email protected]>
+To: ${LIST}@${HOST}
+Cc: "Carbon Copy 15" <[email protected]>,
+ <[email protected]>
+Subject: carbon-copied test post
+
+message goes here
+EOF
+${EZBIN}/ezmlm-send "$DIR" <"$TMP" >"$ERR" 2>&1 || \
+    fatal "failed to produce post"
+unfoldhdrs
+grephdr From: "My Test 15 via ${LIST} <${LIST}@${HOST}>$"
+singlehdr Reply-To:
+grephdr Reply-To: "<${LIST}@${HOST}>"
+singlehdr Cc:
+grephdr Cc: '"Carbon Copy 15" <[email protected]>, <[email protected]>, My Test 15 <[email protected]>'
+
+rm -f "$DIR"/rewritefrom "$DIR"/replytolist
+
 echo OK