azkaban-aplcache

Fix email sending failures (#1270) * Fix email sending failures *

7/11/2017 7:05:47 PM

Details

diff --git a/azkaban-common/src/main/java/azkaban/utils/Emailer.java b/azkaban-common/src/main/java/azkaban/utils/Emailer.java
index 1cf24ba..f7a863f 100644
--- a/azkaban-common/src/main/java/azkaban/utils/Emailer.java
+++ b/azkaban-common/src/main/java/azkaban/utils/Emailer.java
@@ -56,10 +56,10 @@ public class Emailer extends AbstractMailer implements Alerter {
     this.mailSender = props.getString("mail.sender", "");
     this.tls = props.getString("mail.tls", "false");
 
-    final int mailTimeout = props.getInt("mail.timeout.millis", 10000);
+    final int mailTimeout = props.getInt("mail.timeout.millis", 30000);
     EmailMessage.setTimeout(mailTimeout);
     final int connectionTimeout =
-        props.getInt("mail.connection.timeout.millis", 10000);
+        props.getInt("mail.connection.timeout.millis", 30000);
     EmailMessage.setConnectionTimeout(connectionTimeout);
 
     EmailMessage.setTotalAttachmentMaxSize(getAttachmentMaxSize());
@@ -102,7 +102,7 @@ public class Emailer extends AbstractMailer implements Alerter {
         try {
           message.sendEmail();
         } catch (final MessagingException e) {
-          logger.error("Email message send failed", e);
+          logger.error("Failed to send SLA email message" + slaMessage, e);
         }
       }
     }
@@ -131,7 +131,8 @@ public class Emailer extends AbstractMailer implements Alerter {
       try {
         message.sendEmail();
       } catch (final MessagingException e) {
-        logger.error("Email message send failed", e);
+        logger.error(
+            "Failed to send first error email message for execution " + flow.getExecutionId(), e);
       }
     }
   }
@@ -158,7 +159,8 @@ public class Emailer extends AbstractMailer implements Alerter {
       try {
         message.sendEmail();
       } catch (final MessagingException e) {
-        logger.error("Email message send failed", e);
+        logger
+            .error("Failed to send error email message for execution " + flow.getExecutionId(), e);
       }
     }
   }
@@ -185,7 +187,8 @@ public class Emailer extends AbstractMailer implements Alerter {
       try {
         message.sendEmail();
       } catch (final MessagingException e) {
-        logger.error("Email message send failed", e);
+        logger.error("Failed to send success email message for execution " + flow.getExecutionId(),
+            e);
       }
     }
   }
diff --git a/azkaban-common/src/main/java/azkaban/utils/EmailMessage.java b/azkaban-common/src/main/java/azkaban/utils/EmailMessage.java
index ef54182..487397e 100644
--- a/azkaban-common/src/main/java/azkaban/utils/EmailMessage.java
+++ b/azkaban-common/src/main/java/azkaban/utils/EmailMessage.java
@@ -19,7 +19,6 @@ package azkaban.utils;
 import com.sun.mail.smtp.SMTPTransport;
 import java.io.File;
 import java.io.InputStream;
-import java.net.SocketTimeoutException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Date;
@@ -41,6 +40,7 @@ import org.apache.log4j.Logger;
 public class EmailMessage {
 
   private static final String protocol = "smtp";
+  private static final int MAX_EMAIL_RETRY_COUNT = 3;
   private static int _mailTimeout = 10000;
   private static int _connectionTimeout = 10000;
   private static long _totalAttachmentMaxSizeInByte = 1024 * 1024 * 1024; // 1
@@ -230,28 +230,10 @@ public class EmailMessage {
       message.setContent(this._body.toString(), this._mimeType);
     }
 
-    // Transport transport = session.getTransport();
-
     final SMTPTransport t = (SMTPTransport) session.getTransport(protocol);
 
-    try {
-      connectToSMTPServer(t);
-    } catch (final MessagingException ste) {
-      if (ste.getCause() instanceof SocketTimeoutException) {
-        try {
-          // retry on SocketTimeoutException
-          connectToSMTPServer(t);
-          this.logger.info("Email retry on SocketTimeoutException succeeded");
-        } catch (final MessagingException me) {
-          this.logger.error("Email retry on SocketTimeoutException failed", me);
-          throw me;
-        }
-      } else {
-        this.logger.error("Encountered issue while connecting to email server", ste);
-        throw ste;
-      }
-    }
-    t.sendMessage(message, message.getRecipients(Message.RecipientType.TO));
+    retryConnectToSMTPServer(t);
+    retrySendMessage(t, message);
     t.close();
   }
 
@@ -263,6 +245,37 @@ public class EmailMessage {
     }
   }
 
+  private void retryConnectToSMTPServer(final SMTPTransport t) throws MessagingException {
+    int attempt;
+    for (attempt = 0; attempt < MAX_EMAIL_RETRY_COUNT; attempt++) {
+      try {
+        connectToSMTPServer(t);
+        return;
+      } catch (final MessagingException ste) {
+        this.logger.error("Connecting to SMTP server failed, attempt: " + attempt, ste);
+      }
+    }
+    t.close();
+    throw new MessagingException("Failed to connect to SMTP server after "
+        + attempt + " attempts.");
+  }
+
+  private void retrySendMessage(final SMTPTransport t, final Message message)
+      throws MessagingException {
+    int attempt;
+    for (attempt = 0; attempt < MAX_EMAIL_RETRY_COUNT; attempt++) {
+      try {
+        t.sendMessage(message, message.getRecipients(Message.RecipientType.TO));
+        return;
+      } catch (final MessagingException sme) {
+        this.logger.error("Sending email messages failed, attempt: " + attempt, sme);
+      }
+    }
+    t.close();
+    throw new MessagingException("Failed to send email messages after "
+        + attempt + " attempts.");
+  }
+
   public void setBody(final String body, final String mimeType) {
     this._body = new StringBuffer(body);
     this._mimeType = mimeType;