MailHeader.C

/*

  This email filter was written by Ian Kaplan, Bear Products
  International.  It is copyrighted by Ian Kaplan, 2004,
  www.bearcave.com.

  You have permission to use this software without restriction on two
  conditions:

    1. You must preserve this copyright notice in this software and
       any software derived from it.

    2. You accept any risk entailed in using this software.  By
       using this software, you acknowledge that you have a
       sophisticated background in software engineering and 
       understand the way this software functions.  You further
       acknowledge that using this software may result in the
       irretrievable loss of important e-email and you alone
       are responsible for this loss.

  If either of these conditions are unacceptable, you may not use any
  part of this software.

 */

#include <assert.h>
#include <errno.h>
#include <string.h>
#include <ctype.h>

#include <vector>

#include "Logger.h"
#include "SpamUtil.h"
#include "MailHeader.h"

using namespace std;


MailHeader::MailHeader(SpamParameters &p,
                       HeaderInfo &info) : mParams( p ), mHeadInfo( info )
{
  log = pLogger->getLogger("MailHeader");
  pPushBack = 0;
  pushBackBuf[0] = '\0';
  boundaryStr[0] = '\0';
  foundValidAddress = false;
}


MailFilter::classification MailHeader::checkSubject(const char *buf, FILE *fp)
{
  log.log(Logger::DEBUG, "checkSubject", "enter");

  char msg[256];
  char subject[256];
  char foundStr[128];

  // convert to lower case
  SpamUtil().toLower(subject, buf, sizeof(subject));

  foundStr[0] = '\0';
  MailFilter::classification klass = SpamUtil().checkLine(subject, 
                                                          mParams, 
                                                          foundStr, 
                                                          sizeof(foundStr));
  
  if (klass == MailFilter::SUSPECT || klass == MailFilter::GARBAGE) {
    if (klass == MailFilter::SUSPECT) {
      sprintf(msg, "Found \"spam\" word \"%s\", email marked as SUSPECT", 
              foundStr );
    }
    else if (klass == MailFilter::GARBAGE) {
      mHeadInfo.reason( foundStr );
      sprintf(msg, "Found \"kill\" word \"%s\", email marked as GARBAGE", 
              foundStr );
    }
    log.log(Logger::DEBUG, "checkSubject", msg );
  }

  pPushBack = 0;
  char *pBuf;
  if ((pBuf = fgets(pushBackBuf, sizeof(pushBackBuf), fp)) != 0) {
    if (SpamUtil().isBlankLine(pBuf) || SpamUtil().findColon(pBuf) != 0) {
      pPushBack = pBuf;
    }
  }
  
  log.log(Logger::DEBUG, "checkSubject", "exit");
  return klass;
} // checkSubject


MailFilter::classification MailHeader::checkFrom(const char *buf)
{
  MailFilter::classification klass = MailFilter::UNKNOWN;
  log.log(Logger::DEBUG, "checkFrom", "enter");

  vector<const char *> fromAddrs = mParams.getSection(SpamParameters::from_address);
  vector<const char *> killAddrs = mParams.getSection(SpamParameters::from_kill);

  char msg[128];
  char from[256];

  SpamUtil().toLower(from, buf, sizeof(from));

  size_t len;
  len = killAddrs.size();
  for (size_t i = 0; i < len; i++) {
    if (strstr(from, killAddrs[i]) != 0) {
      sprintf(msg, "Found address \"%s\", email marked as GARBAGE", 
              killAddrs[i] );
      mHeadInfo.reason( msg );
      log.log(Logger::DEBUG, "checkFrom", msg );
      klass = MailFilter::GARBAGE;
      break;
    }
  }

  if (klass == MailFilter::UNKNOWN) {
    len = fromAddrs.size();
    for (size_t i = 0; i < len; i++) {
      if (strstr(from, fromAddrs[i]) != 0) {
        sprintf(msg, "Found \"from address\" \"%s\", email marked as EMAIL", 
                fromAddrs[i] );
        log.log(Logger::DEBUG, "checkFrom", msg );
        klass = MailFilter::EMAIL;
        break;
      }
    } // for
  }

  log.log(Logger::DEBUG, "checkFrom", "exit");
  return klass;
} // checkFrom



bool MailHeader::addrContinues(const char *buf)
{
  bool rslt = false;

  log.log(Logger::DEBUG, "addrContinues", "enter");
  if (buf) {
    int end = strlen( buf );

    if (end > 0) {
      end--;
      const char *endPtr;
      for (endPtr = &buf[end]; endPtr >= buf && isspace(*endPtr); endPtr--)
        /* nada */;
      if (*endPtr == ',') {
        rslt = true;
      }
      else if (*endPtr == '"') {
        if (endPtr > buf && *(endPtr-1) == '\'') {
          rslt = true;
        }
      }
    }
  }

  char msgbuf[128];
  sprintf( msgbuf, "returns %s", (rslt) ? "TRUE" : "FALSE" );
  log.log(Logger::DEBUG, "addrContinues", msgbuf );

  log.log(Logger::DEBUG, "addrContinues", "exit");
  return rslt;
} // addrContinues



MailFilter::classification MailHeader::checkDomainAddrs(const char *domainName,
                                                        const char *pBuf)
{
  log.log(Logger::DEBUG, "checkDomainAddrs", "enter");

  assert( ((domainName != 0) && (pBuf != 0)) );

  vector<const char *> validUsers = mParams.getSection(SpamParameters::valid_users);
  const size_t numUsers = validUsers.size();
  MailFilter::classification klass = MailFilter::UNKNOWN;
  
  bool done = false;
  size_t domainNameLen = strlen( domainName );
  const char *domainPtr = strstr(pBuf, domainName );
  while (domainPtr) {
    if (domainPtr > pBuf+2) {
      domainPtr--;
      if (*domainPtr == '@') {
        // find the start and end of the user name
        const char *endPtr = domainPtr;
        domainPtr--;
        const char *beginPtr = domainPtr;
        while (beginPtr >= pBuf && isalnum( *beginPtr ))
          beginPtr--;
        if (!isalnum(*beginPtr)) {
          beginPtr++;
        }

        // Now check to see if the user name is in the valid_users list
        // Note that this function is used for both To: and Cc:, so 
        // foundValidAddress could have been set in a previous call.
        bool foundInList = false;
        for (size_t i = 0; i < numUsers; i++) {
          const char *word = validUsers[i];
          if (SpamUtil().match(beginPtr, endPtr, word)) {
            foundValidAddress = true;
            foundInList = true;
          }
        } // for

        if (!foundInList) {
          char msg[128];
          char user[128];
          size_t ix = 0;
          for (const char *pCh = beginPtr; pCh < endPtr; pCh++, ix++) {
            user[ix] = *pCh;
          }
          user[ix] = '\0';
          sprintf(msg, "Non-valid user \"%s\", email marked as GARBAGE", 
                  user );
          mHeadInfo.reason( msg );
          log.log(Logger::DEBUG, "checkDomainAddrs", msg );
          klass = MailFilter::GARBAGE;
        }
        // endPtr points to the '@'
        pBuf = (endPtr + 1);
      }
    }
    if (klass == MailFilter::UNKNOWN) {
      pBuf = pBuf + domainNameLen;
      domainPtr = strstr(pBuf, domainName);
    }
    else {
      break;  // exit the while loop
    }
  } // while

  log.log(Logger::DEBUG, "checkDomainAddrs", "exit");

  return klass;
} // checkDomainAddrs



MailFilter::classification MailHeader::checkAddressSection(const char *buf, FILE *fp)
{
  log.log(Logger::DEBUG, "checkAddressSection", "enter");

  MailFilter::classification klass = MailFilter::UNKNOWN;
  vector<const char *> toAddrs = mParams.getSection(SpamParameters::to_list);
  vector<const char *> myDomain = mParams.getSection(SpamParameters::my_domain);

  const char *domainName = 0;
  if (myDomain.size() > 0)
    domainName = myDomain[0];
  
  const char *pBuf = buf;
  const size_t toAddrLen = toAddrs.size();

  char localBuf[1024];
  size_t i;
  bool done;
  do {
    done = true;

    SpamUtil().toLower(localBuf, pBuf, sizeof(localBuf));
    for (i = 0; i < toAddrLen; i++) {
      if (strstr(localBuf, toAddrs[i]) != 0) {
        const char *hit = toAddrs[i];
        char msg[128];
        sprintf(msg, "found \"%s\", marked as EMAIL", hit );
        log.log(Logger::DEBUG, "checkAddressSection", msg );
        klass = MailFilter::EMAIL;
        break;
      }
    } // for

    if (klass == MailFilter::UNKNOWN && domainName != 0) {
      klass = checkDomainAddrs( domainName, localBuf );
    }

    if (klass == MailFilter::UNKNOWN) {
      if (addrContinues(localBuf)) {
        if ((pBuf = fgets(localBuf, sizeof(localBuf), fp)) != 0) {
          done = false;
        }
      }
    }

  } while (!done);

  log.log(Logger::DEBUG, "checkAddressSection", "exit");

  return klass;
} // checkAddressSection


MailFilter::classification MailHeader::checkReceived(const char *buf,
                                                     FILE *fp,
                                                     size_t line)
{
  log.log(Logger::DEBUG, "checkReceived", "enter");
  MailFilter::classification klass = MailFilter::UNKNOWN;

  const char *pColon;
  do {
    pPushBack = fgets(pushBackBuf, sizeof(pushBackBuf), fp);
    if (pPushBack != 0) {
      if ((klass == MailFilter::UNKNOWN) && (strstr(pPushBack, "forged") != 0)) {
        klass = MailFilter::SUSPECT;
      }
      pColon = SpamUtil().findColon( pPushBack );
    }
  } while (pPushBack != 0 && !pColon);

  log.log(Logger::DEBUG, "checkReceived", "exit");
  return klass;
} // checkReceived


void MailHeader::saveBoundary(const char *pBound)
{
  log.log(Logger::DEBUG, "saveBoundary", "enter");

  if (*pBound == '=') {
    pBound++;
    // skip any white space between the "=" and the quote
    pBound = SpamUtil().skipWhiteSpace( pBound );
    if (*pBound == '"') {
      pBound++;
    }
    const size_t len = sizeof(boundaryStr) - 1;
    size_t ix = 0;
    while (*pBound && 
           ix < len && 
           (! isspace(*pBound)) &&
           *pBound != '"') {
      boundaryStr[ix] = *pBound;
      pBound++;
      ix++;
    }
    boundaryStr[ix] = '\0';

    if (ix > 0) {
      char msg[128];
      sprintf(msg, "boundary str. = \"%s\"", boundaryStr );
      log.log(Logger::DEBUG, "saveBoundary", msg);
    }
  }
  else {
    log.log(Logger::ERROR, "saveBoundary", "'=' expected");
  }

  log.log(Logger::DEBUG, "saveBoundary", "exit");
} // saveBoundary


MailFilter::classification MailHeader::parseContentType(FILE* fp,
                                                        char *contentBuf )
{


  const char *BOUNDARY = "boundary";
  static size_t BOUNDARY_LEN = strlen( BOUNDARY );
  const char *CONTENT_ENCODE = "Content-Transfer-Encoding";
  static size_t CONTENT_ENCODE_LEN = strlen(CONTENT_ENCODE);

  log.log(Logger::DEBUG, "parseContentType", "enter");

  MailFilter::classification klass = MailFilter::UNKNOWN;

  SpamUtil::contentType type = SpamUtil().classifySection( contentBuf );

  {
    char *pBound = strstr(contentBuf, BOUNDARY);
    if (pBound && type != SpamUtil::TEXT) {
      saveBoundary( pBound + BOUNDARY_LEN );
    }
  }
  
  pPushBack = 0;
  char *pBuf;
  if ((pBuf = fgets(pushBackBuf, sizeof(pushBackBuf), fp)) != 0) {
    // If the line that follows the Content-Type does not have a colon
    // (e.g., findColon() does not return a pointer) and it is not a 
    // blank line, then it may be a boundary definition or a charset
    // definition.  If it is a boundary we want to pick it up.  Otherwise
    // we want to skip it.
    if (SpamUtil().isBlankLine(pBuf) || SpamUtil().findColon(pBuf) != 0) {
      pPushBack = pBuf;
    }
    else {
      char *pBound = strstr(pBuf, BOUNDARY);
      if (pBound && type != SpamUtil::TEXT) {
        saveBoundary( pBound + BOUNDARY_LEN );
      }
      // get the next line
      pBuf = fgets(pushBackBuf, sizeof(pushBackBuf), fp);
      pPushBack = pBuf;
    }

    // Check for the Content-Transfer-Encoding line and see if it is base64
    char *pEncode;
    if ((pEncode = strstr(pBuf, CONTENT_ENCODE)) != 0) {
      pPushBack = 0;
      if (strstr(pEncode + CONTENT_ENCODE_LEN, "base64")) {
        type = SpamUtil::BASE64;
      }
    }

  }

  if (type == SpamUtil::BASE64) {
    if (mParams.hasFlag("kill_base64")) {
      mHeadInfo.reason("found base64 encoded information");
      klass = MailFilter::GARBAGE;
    }
    else {
      klass = MailFilter::SUSPECT;
    }
  }
  else if (type == SpamUtil::HTML) {
      klass = MailFilter::SUSPECT;
  }
  else if (type == SpamUtil::IMAGE || type == SpamUtil::AUDIO) {
      klass = MailFilter::SUSPECT;
  }

  const char *typeName;
  typeName = SpamUtil().typeToStr( type );

  char msg[128];
  sprintf(msg, "mail type = %s", typeName );
  log.log(Logger::DEBUG, "parseContentType", msg );

  log.log(Logger::DEBUG, "parseContentType", "exit");
  return klass;
} // parseContentType



void MailHeader::fillInSections(FILE *fp)
{
  static const char *TO = "to:";
  static const size_t TO_LEN = strlen( TO );
  static const char *FROM = "from:";
  static const size_t FROM_LEN = strlen( FROM );
  static const char *SUBJECT = "subject:";
  static const size_t SUBJECT_LEN = strlen( SUBJECT );
  static const char *DATE = "date:";
  static const size_t DATE_LEN = strlen( DATE );
  char buf[1024];
  char *pBuf = 0;
  size_t bufSize = 0;

  log.log(Logger::DEBUG, "fillInSections", "enter");

  do {
    if (pPushBack == 0) {
      pushBackBuf[0] = '0';
      bufSize = sizeof(buf);
      pBuf = fgets(buf, bufSize, fp);
    }
    else {
      pBuf = pPushBack;
      bufSize = sizeof( pushBackBuf );
      pPushBack = 0;
    }
    if (pBuf != 0) {
      if (! SpamUtil().isBlankLine( pBuf )) {
        char *pCopy = 0;
        if (SpamUtil().match(pBuf, TO_LEN, TO)) {
          pCopy = pBuf + TO_LEN;
          mHeadInfo.to( pCopy );
        }
        else if (SpamUtil().match(pBuf, FROM_LEN, FROM)) {
          pCopy = pBuf + FROM_LEN;
          mHeadInfo.from( pCopy );
        }
        else if (SpamUtil().match(pBuf, SUBJECT_LEN, SUBJECT)) {
          pCopy = pBuf + SUBJECT_LEN;
          mHeadInfo.subject( pCopy );
        }
        else if (SpamUtil().match(pBuf, DATE_LEN, DATE)) {
          pCopy = pBuf + DATE_LEN;
          mHeadInfo.date( pCopy );
        }
      }
      else {
        // found a blank line which follows the mail header
        break;
      }
    }
  } while (pBuf != 0);

  if (pBuf == 0) {
    log.log(Logger::DEBUG, "fillInSections", "end-of-file reached");
  }
  log.log(Logger::DEBUG, "fillInSections", "exit");
} // fillInSections


MailFilter::classification MailHeader::checkHeader(FILE *fp )
{
  log.log(Logger::DEBUG, "checkHeader", "enter");
  MailFilter::classification klass = MailFilter::BAD_VALUE;
  if (!feof(fp)) {
    klass = MailFilter::UNKNOWN;
    char *pBuf;

    // Skip any blank lines which start the e-mail message
    while ((pPushBack = fgets(pushBackBuf, sizeof(pushBackBuf), fp)) != 0) {
      if (! SpamUtil().isBlankLine( pPushBack )) {
        break;
      }
    } // while

    if (pPushBack != 0) {  // Loop through the e-mail header
      size_t line = 1;
      const char *RECEIVED = "received";
      static const size_t RECEIVED_LEN = strlen(RECEIVED);
      const char *SUBJECT = "subject";
      static const size_t SUBJECT_LEN = strlen(SUBJECT);
      const char *FROM = "from";
      static const size_t FROM_LEN = strlen(FROM);
      const char *CONTENT_TYPE = "content-type";
      const char *TO = "to";
      static const size_t TO_LEN = strlen(TO);
      const char *CC = "cc";
      const char *DATE = "date";
      const char *pBound = 0;
      const char *pColon = 0;
      char buf[1024];
      do { // DO
        if (pPushBack == 0) {
          pushBackBuf[0] = '\0';
          pBuf = fgets(buf, sizeof(buf), fp);
        }
        else {
          pBuf = pPushBack;
          pPushBack = 0;
        }
        if (!pBuf) {
          break;
        }

        if (! SpamUtil().isBlankLine( pBuf )) {
          pColon = SpamUtil().findColon( pBuf );
          if (SpamUtil().match(pBuf, FROM_LEN, FROM)) {
            pColon = pBuf + FROM_LEN;
            if (*pColon == ':') {
              pColon++;
              mHeadInfo.from( pColon );
            }
            else {
              mHeadInfo.fromNoColon( pColon );
            }
            klass = checkFrom( pColon );
          }
          else if (pColon != 0) {
            if (SpamUtil().match(pBuf, RECEIVED_LEN, RECEIVED)) {
              pColon = pBuf + RECEIVED_LEN + 1;
              klass = checkReceived( pColon, fp, line );
            } else if (SpamUtil().match(pBuf, SUBJECT_LEN, SUBJECT)) {
              pColon = pBuf + SUBJECT_LEN + 1;
              mHeadInfo.subject(pColon);
              klass = checkSubject( pColon, fp );
            }
            else if (SpamUtil().match(pBuf, pColon, CONTENT_TYPE)) {
              pColon++;
              klass = parseContentType(fp, pBuf);
            }
            else if (SpamUtil().match(pBuf, TO_LEN, TO)) {
              pColon = pBuf + TO_LEN + 1;
              mHeadInfo.to( pColon );
              klass = checkAddressSection(pColon, fp);
            }
            else if (SpamUtil().match(pBuf, pColon, CC)) {
              pColon++;
              klass = checkAddressSection(pColon, fp);
            }
            else if (SpamUtil().match(pBuf, pColon, DATE)) {
              pColon++;
              mHeadInfo.date(pColon);
            }
          } // has a colon (pColon != 0
        } // is blank line
        else {
          // found a blank line
          break;
        }
      } while (klass == MailFilter::UNKNOWN && pBuf != 0);

      // if we have not finished on a black line, fill in any sections that
      // have not been encountered yet.
      if (! SpamUtil().isBlankLine( pBuf )) {
        fillInSections(fp);
      }

      // If the email was not addressed to a known mailing list and
      // and address in the SpamFilterParams section my_address is
      // not found, then it is classified as SPAM.
      if (klass == MailFilter::UNKNOWN && (! foundValidAddress)) {
        log.log(Logger::DEBUG, "checkHeader", "Did not find a valid To: or Cc: address");
        klass = MailFilter::SUSPECT;
      }
    } // if pBuf
  }

  mHeadInfo.klass( klass );

  char msg[128];
  sprintf(msg, "return value = %s", SpamUtil().classificationToStr( klass ));
  log.log(Logger::DEBUG, "checkHeader", msg );
  log.log(Logger::DEBUG, "checkHeader", "exit");
  return klass;
} // checkHeader

---