Ticket #17: innfeed-headers.patch

File innfeed-headers.patch, 20.5 KB (added by eagle, 14 years ago)

Patch against INN 2.4.1

  • doc/man/innfeed.conf.5

    diff -ruN inn-2.4.1-maxartsize.ORIG/doc/man/innfeed.conf.5 inn-2.4.1-maxartsize/doc/man/innfeed.conf.5
    old new  
    459459This key requires a positive integer value. It defines the tcp/ip port
    460460number to use when connecting to the remote.
    461461.TP
     462.B headers-only
     463This key requires a boolean value. By default it is set to false. When
     464set to true, the peer will be sent a headers-only feed. If a Bytes:
     465header is not present, one will be added with as value the length of the
     466complete article. This is useful for feeding a diablo server, or an
     467overview-only INN server.
     468Note that innfeed should send a MODE HEADFEED command to the peer to
     469make sure it expects a headers-only feed, and the current code doesn't
     470do that yet, partly because INN itself doesn't understand it yet so
     471it would be impossible to feed a headers-only feed to INN.
     472Make sure that you
     473.I never
     474.I ever
     475send a headers-only feed to a peer that also receives a normal feed!
     476.TP
    462477.B drop-deferred
    463478This key requires a boolean value. By default it is set to false. When
    464479set to true, and a peer replies with code 431 or 436 (try again later) just
  • inn-2.4.1-maxartsize

    Binary files inn-2.4.1-maxartsize.ORIG/innfeed/.article.c.swp and inn-2.4.1-maxartsize/innfeed/.article.c.swp differ
    diff -ruN inn-2.4.1-maxartsize.ORIG/innfeed/article.c inn-2.4.1-maxartsize/innfeed/article.c
    old new  
    5353    char *fname ;               /* the file name of the article */
    5454    char *msgid ;               /* the msgid of the article (INN tells us) */
    5555    Buffer contents ;           /* the buffer of the actual on disk stuff */
     56    Buffer headers ;            /* the buffer of just the headers */
    5657    Buffer *nntpBuffers ;       /* list of buffers for transmisson */
     58    Buffer *nntpHeaderBuffers ; /* list of header buffers for transmission */
    5759    const void *mMapping ;      /* base of memory mapping, or NULL if none */
    5860    bool loggedMissing ;        /* true if article is missing and we logged */
    5961    bool articleOk ;            /* true until we know otherwise. */
     
    7678   * Private functions
    7779   */
    7880
    79 static Buffer artGetContents (Article article) ;  /* Return the buffer that
    80                                                      fillContents() filled
    81                                                      up. */
     81  /* Return the buffer that fillContents() filled up. */
     82static Buffer artGetContents (Article article, bool headersOnly) ;
    8283
    8384  /* Log statistics on article memory usage. */
    8485static void logArticleStats (TimeoutId id, void *data) ;
    8586
    86 static bool fillContents (Article article) ;  /* Read the article's bits
    87                                                  off the disk. */
     87  /*  Read the article's bits off the disk. */
     88static bool fillContents (Article article, bool HeadersOnly) ;
    8889
    8990  /* Append buffer B to the buffer array BUFFS. */
    9091static void appendBuffer (Buffer b, Buffer **buffs, int *newSpot, int *curLen);
    9192
    92 static bool prepareArticleForNNTP (Article article) ;  /* Do the necessary
    93                                                           CR-LF stuff */
     93  /* Do the necessary CR-LF stuff */
     94static bool prepareArticleForNNTP (Article article, bool headersOnly) ;
    9495
    9596static bool artFreeContents (Article art) ;  /* Tell the Article to release
    9697                                                its contents buffer if
     
    108109static unsigned int hashString (const char *string) ;
    109110
    110111  /* Locates the article with the given message ID, in the has table. */
    111 static Article hashFindArticle (const char *msgid) ;
     112static Article hashFindArticle (const char *msgid);
    112113
    113114  /* Puts the given article in the hash table. */
    114115static void hashAddArticle (Article article) ;
     
    184185Article newArticle (const char *filename, const char *msgid)
    185186{
    186187  Article newArt = NULL ;
    187  
     188
    188189  TMRstart(TMR_NEWARTICLE);
    189190  if (hashTable == NULL)
    190191    {                           /* first-time through initialization. */
     
    212213      newArt->msgid = xstrdup (msgid) ;
    213214     
    214215      newArt->contents = NULL ;
     216      newArt->headers  = NULL ;
    215217      newArt->mMapping = NULL ;
    216218      newArt->refCount = 1 ;
    217219      newArt->loggedMissing = false ;
     
    235237      newArt->refCount++ ;
    236238      d_printf (2,"Reusing existing article for %s\nx",msgid) ;
    237239    }
     240
    238241  TMRstop(TMR_NEWARTICLE);
    239242  return newArt ;
    240243}
     
    268271          if (article->nntpBuffers != NULL)
    269272            freeBufferArray (article->nntpBuffers) ;
    270273
     274          if (article->nntpHeaderBuffers != NULL)
     275            freeBufferArray (article->nntpHeaderBuffers) ;
     276
     277          if (article->headers) {
     278            if (article->contents != article->headers)
     279              bytesInUse -= bufferDataSize (article->headers) ;
     280            delBuffer (article->headers) ;
     281          }
     282
    271283          delBuffer (article->contents) ;
    272284        }
    273285
     
    360372}
    361373
    362374  /* return true if we have or are able to get the contents off the disk */
    363 bool artContentsOk (Article article)
     375bool artContentsOk (Article article, bool headersOnly)
    364376{
    365377  bool rval = false ;
    366378
    367   if ( prepareArticleForNNTP (article) )
     379  if ( prepareArticleForNNTP (article, headersOnly) )
    368380    rval = true ;
    369381
    370382  return rval ;
     
    393405
    394406  /* Get a NULL terminated array of Buffers that is ready for sending via NNTP */
    395407
    396 Buffer *artGetNntpBuffers (Article article)
     408Buffer *artGetNntpBuffers (Article article, bool headersOnly)
    397409{
    398   if ( !prepareArticleForNNTP (article) )
     410  if ( !prepareArticleForNNTP (article, headersOnly) )
    399411    return NULL ;
    400412
    401   return dupBufferArray (article->nntpBuffers) ;
     413  return dupBufferArray (headersOnly ? article->nntpHeaderBuffers :
     414                                        article->nntpBuffers) ;
    402415}
    403416
    404417
     
    409422}
    410423
    411424  /* return size of the article */
    412 int artSize (Article article)
     425int artSize (Article article, bool headersOnly)
    413426{
    414427  if (article == NULL || article->contents == NULL)
    415428    return (int)0 ;
     429  if (headersOnly && article->headers == NULL)
     430    return (int)0;
    416431  return (int)bufferDataSize(article->contents);
    417432}
    418433
    419434
    420435  /* return how many NNTP-ready buffers the article contains */
    421 unsigned int artNntpBufferCount (Article article)
     436unsigned int artNntpBufferCount (Article article, bool headersOnly)
    422437{
    423   if ( !prepareArticleForNNTP (article) )
     438  if ( !prepareArticleForNNTP (article, headersOnly) )
    424439    return 0 ;
    425440 
    426   return bufferArrayLen (article->nntpBuffers) ;
     441  return bufferArrayLen (headersOnly ? article->nntpHeaderBuffers :
     442                                        article->nntpBuffers) ;
    427443}
    428444
    429445
     
    451467
    452468  /* return a single buffer that contains the disk image of the article (i.e.
    453469     not fixed up for NNTP). */
    454 static Buffer artGetContents (Article article)
     470static Buffer artGetContents (Article article, bool headersOnly)
    455471{
    456472  Buffer rval = NULL ;
    457473
    458474  if (article->articleOk)
    459475    {
    460476      if (article->contents == NULL)
    461         fillContents (article) ;
     477        fillContents (article, headersOnly) ;
    462478
    463479      if (article->contents != NULL)
    464         rval = bufferTakeRef (article->contents) ;
     480        rval = bufferTakeRef (headersOnly ? article->headers :
     481                                            article->contents) ;
    465482    }
    466483
    467484  return rval ;
     
    495512  articleStatsId = prepareSleep (logArticleStats,ARTICLE_STATS_PERIOD,0) ;
    496513}
    497514
     515/* Find the start of the body, and check if there's a Bytes: header.
     516   body will point to the empty line between header and body. */
     517static void findBody (char *data, size_t size, bool inWireFormat,
     518                        char **body, char **bytes, char *control)
     519{
     520    char *p;
     521    int nlseen = 1;
     522
     523    *bytes = NULL;
     524    *control = NULL;
     525    *body  = NULL;
     526    if (inWireFormat) size--;
     527
     528    for (p = data; p < data + size; p++) {
     529
     530        if (nlseen && (*p == 'b' || *p == 'B') &&
     531                        p < data + size - 6 &&
     532                        strncasecmp(p, "Bytes:", 6) == 0)
     533            *bytes = p;
     534
     535        if (nlseen && (*p == 'c' || *p == 'C') &&
     536                        p < data + size - 8 &&
     537                        strncasecmp(p, "Control:", 8) == 0)
     538            *control = p;
     539
     540        if (inWireFormat) {
     541            if (*p != '\r' && *(p+1) == '\n') {
     542                p++;
     543                continue;
     544            }
     545            if (*p == '\r' && *(p+1) == '\n')
     546                p++;
     547        }
     548
     549        if (*p == '\n') {
     550           if (nlseen) {
     551                *body = p - (inWireFormat ? 1 : 0);
     552                break;
     553           }
     554           nlseen = 1;
     555        } else
     556           nlseen = 0;
     557    }
     558
     559    if (*bytes == NULL) *bytes = p;
     560}
     561
    498562
    499563  /* do the actual read of the article off disk into a Buffer that is stored
    500564     in the Article object. The Article will end up with its contents field
     
    504568     contents may be copied around after reading to insert a carriage
    505569     return before each newline. */
    506570
    507 static bool fillContents (Article article)
     571static bool fillContents (Article article, bool headersOnly)
    508572{
     573    Buffer body;
    509574    int fd = -1;
    510     char *p;
     575    char *p, *b;
    511576    static bool maxLimitNotified ;
    512577    bool opened;
    513578    size_t articlesize = 0;
     579    size_t bytes, hdrsize;
    514580    char *buffer = NULL ;
    515581    int amt = 0 ;
    516582    size_t idx = 0, amtToRead ;
     
    721787        }
    722788    }
    723789
    724    
     790    /* In header-only mode we need to replace article->contents with
     791     * a buffer with just the headers, followed by a Bytes: header. */
     792    if (article->articleOk && article->contents && headersOnly) {
     793
     794        bytes = bufferDataSize (article->contents);
     795        buffer = bufferBase (article->contents);
     796
     797        findBody(buffer, bytes, article->inWireFormat, &p, &bHdr, &cHdr);
     798
     799        if (cHdr)
     800            article->headers = bufferTakeRef (article->contents) ;
     801        else if ((article->headers = newBuffer (hdrsize + 24)) != NULL) {
     802            p = bufferBase(article->headers);
     803            memcpy(p, buffer, hdrsize);
     804            if (bHdr == NULL)
     805                hdrsize += sprintf(p + hdrsize, "Bytes: %d", bytes);
     806            hdrsize += sprintf(p + hdrsize, "%s",
     807                        article->inWireFormat ? "\r\n\r\n" : "\n\n");
     808            bufferSetDataSize (article->headers, hdrsize) ;
     809            bytesInUse += hdrsize;
     810            byteTotal += hdrsize;
     811        } else
     812            warn ("ME internal failed to build headfeed buffer") ;
     813    }
     814
    725815    /* If we're not useing storage api, we should close a valid file descriptor */
    726816    if (!article->arthandle && (fd >= 0))
    727817        close (fd) ;
    728818
    729819    TMRstop(TMR_READART);
    730     return (article->contents != NULL ? true : false) ;
     820    return ((headersOnly ? article->headers : article->contents) != NULL ?
     821            true : false) ;
    731822}
    732823
    733824
     
    751842  /* Takes the articles contents buffer and overlays a set of new buffers on
    752843     top of it. These buffers insert the required carriage return and dot
    753844     characters as needed */
    754 static bool prepareArticleForNNTP (Article article)
     845static bool prepareArticleForNNTP (Article article, bool headersOnly)
    755846{
    756847  static Buffer dotFirstBuffer ;
    757848  static Buffer dotBuffer ;
     
    762853  char *start, *end ;
    763854  Buffer contents ;
    764855
    765   contents = artGetContents (article) ; /* returns a reference */
     856  contents = artGetContents (article, headersOnly) ; /* returns a reference */
    766857
    767858  TMRstart(TMR_PREPART);
    768859  if (contents == NULL) {
    769860    TMRstop(TMR_PREPART);
    770861    return false ;
    771862  }
    772   else if (article->nntpBuffers != NULL)
     863  else if ((!headersOnly && article->nntpBuffers != NULL) ||
     864           ( headersOnly && article->nntpHeaderBuffers != NULL))
    773865    {
    774866      delBuffer (contents) ;
    775867      TMRstop(TMR_PREPART);
     
    821913 
    822914     
    823915  delBuffer (contents) ;    /* the article is still holding a reference */
    824   article->nntpBuffers = nntpBuffs ;
     916  if (headersOnly)
     917    article->nntpHeaderBuffers = nntpBuffs ;
     918  else
     919    article->nntpBuffers = nntpBuffs ;
    825920  TMRstop(TMR_PREPART);
    826921  return true ;
    827922}
     
    845940      }
    846941    }
    847942
     943  if (art->nntpHeaderBuffers != NULL)
     944    {
     945    if (bufferRefCount (art->nntpHeaderBuffers[0]) > 1)
     946      return false ;
     947    else
     948      {
     949        freeBufferArray (art->nntpHeaderBuffers) ;   
     950        art->nntpHeaderBuffers = NULL ;
     951      }
     952    }
     953
    848954  ASSERT (bufferRefCount (art->contents) == 1) ;
    849955
    850956  if (art->mMapping)
     
    853959    bytesInUse -= bufferDataSize (art->contents) ;
    854960 
    855961  delBuffer (art->contents) ;
    856 
    857962  art->contents = NULL ;
     963
     964  if (art->headers) delBuffer (art->headers) ;
     965  art->headers = NULL ;
    858966
    859967  return true ;
    860968}
  • inn-2.4.1-maxartsize

    diff -ruN inn-2.4.1-maxartsize.ORIG/innfeed/article.h inn-2.4.1-maxartsize/innfeed/article.h
    old new  
    4444
    4545  /* return true if we have the article's contents (calling this may trigger
    4646     the reading off the disk). */
    47 bool artContentsOk (Article article) ;
     47bool artContentsOk (Article article, bool headersOnly) ;
    4848
    4949  /* increments reference count and returns a copy of article that can be
    5050     kept (or passed off to someone else) */
     
    5555
    5656  /* return a list of buffers suitable for giving to an endpoint. The return
    5757     value can (must) be given to freeBufferArray */
    58 Buffer *artGetNntpBuffers (Article article) ;
     58Buffer *artGetNntpBuffers (Article article, bool headersOnly) ;
    5959
    6060  /* return the message id stoed in the article object */
    6161const char *artMsgId (Article article) ;
    6262
    6363  /* return size of the article */
    64 int artSize (Article article) ;
     64int artSize (Article article, bool headersOnly) ;
    6565
    6666  /* return the number of buffers that artGetNntpBuffers() would return. */
    67 unsigned int artNntpBufferCount (Article article) ;
     67unsigned int artNntpBufferCount (Article article, bool headersOnly) ;
    6868
    6969  /* tell the Article class to log (or not) missing articles as they occur. */
    7070void artLogMissingArticles (bool val) ;
  • innfeed/connection.c

    diff -ruN inn-2.4.1-maxartsize.ORIG/innfeed/connection.c inn-2.4.1-maxartsize/innfeed/connection.c
    old new  
    30623062  else
    30633063    {
    30643064      cxn->takesOkayed++ ;
    3065       cxn->takesSizeOkayed += artSize(artHolder->article);
     3065      cxn->takesSizeOkayed += artSize(artHolder->article, hostHeadersOnly(cxn->myHost));
    30663066
    30673067      remArtHolder (artHolder, &cxn->takeRespHead, &cxn->articleQTotal) ;
    30683068      if (cxn->articleQTotal == 0)
     
    31723172  else
    31733173    {
    31743174      cxn->takesRejected++ ;
    3175       cxn->takesSizeRejected += artSize(artHolder->article);
     3175      cxn->takesSizeRejected += artSize(artHolder->article, hostHeadersOnly(cxn->myHost));
    31763176
    31773177      remArtHolder (artHolder, &cxn->takeRespHead, &cxn->articleQTotal) ;
    31783178      /* Some(?) hosts return the 439 response even before we're done
     
    32383238      cxn->takeRespHead = NULL ;
    32393239      cxn->articleQTotal = 0 ;
    32403240      cxn->takesOkayed++ ;
    3241       cxn->takesSizeOkayed += artSize(artHolder->article);
     3241      cxn->takesSizeOkayed += artSize(artHolder->article, hostHeadersOnly(cxn->myHost));
    32423242     
    32433243      if (cxn->articleQTotal == 0)
    32443244        cxnIdle (cxn) ;
     
    35133513
    35143514  artHolder = cxn->takeRespHead ;
    35153515  cxn->takeRespHead = NULL ;
    3516   cxn->takesSizeRejected += artSize(artHolder->article);
     3516  cxn->takesSizeRejected += artSize(artHolder->article, hostHeadersOnly(cxn->myHost));
    35173517
    35183518  /* Some servers return the 437 response before we're done sending. */
    35193519  if (cxn->articleQTotal == 0 && !writeIsPending (cxn->myEp))
     
    40504050  ASSERT (article != NULL) ;
    40514051 
    40524052  if (cxn->state != cxnClosingS)
    4053     writeArray = artGetNntpBuffers (article) ;
     4053    writeArray = artGetNntpBuffers (article, hostHeadersOnly(cxn->myHost)) ;
    40544054  else
    40554055    writeArray = NULL ;
    40564056
     
    42664266      /* count up all the buffers we'll be writing. One extra each time for
    42674267         the TAKETHIS command buffer*/
    42684268      for (p = cxn->takeHead ; p != NULL ; p = p->next)
    4269         if (artContentsOk (p->article))
    4270             lenArray += (1 + artNntpBufferCount (p->article)) ;
     4269        if (artContentsOk (p->article, hostHeadersOnly(cxn->myHost)))
     4270            lenArray += (1 + artNntpBufferCount (p->article, hostHeadersOnly(cxn->myHost))) ;
    42714271
    42724272      /* now allocate the array for the buffers and put them all in it */
    42734273      /* 1 for the terminator */
     
    42874287          int i, nntpLen ;
    42884288
    42894289          article = p->article ;
    4290           nntpLen = artNntpBufferCount (article) ;
     4290          nntpLen = artNntpBufferCount (article, hostHeadersOnly(cxn->myHost)) ;
    42914291          msgid = artMsgId (article) ;
    42924292
    42934293          if (nntpLen == 0)
     
    43084308            }
    43094309          else
    43104310            {
    4311               articleBuffers = artGetNntpBuffers (article) ;
     4311              articleBuffers = artGetNntpBuffers (article, hostHeadersOnly(cxn->myHost)) ;
    43124312
    43134313              /* set up the buffer with the TAKETHIS command in it.
    43144314                 12 == strlen ("TAKETHIS \n\r") */
  • inn-2.4.1-maxartsize

    diff -ruN inn-2.4.1-maxartsize.ORIG/innfeed/host.c inn-2.4.1-maxartsize/innfeed/host.c
    old new  
    9393  unsigned int closePeriod;
    9494  unsigned int dynamicMethod;
    9595  bool wantStreaming;
     96  bool headersOnly;
    9697  bool dropDeferred;
    9798  bool minQueueCxn;
    9899  double lowPassLow; /* as percentages */
     
    504505      params->closePeriod=CLOSE_PERIOD;
    505506      params->dynamicMethod=METHOD_STATIC;
    506507      params->wantStreaming=STREAM;
     508      params->headersOnly=false;
    507509      params->dropDeferred=false;
    508510      params->minQueueCxn=false;
    509511      params->lowPassLow=NOCHECKLOW;
     
    12861288           host->params->initialConnections) ;
    12871289  fprintf (fp,"%s    want-streaming : %s\n",indent,
    12881290           boolToString (host->params->wantStreaming)) ;
     1291  fprintf (fp,"%s    headers-only : %s\n",indent,
     1292           boolToString (host->params->headersOnly)) ;
    12891293  fprintf (fp,"%s    drop-deferred : %s\n",indent,
    12901294           boolToString (host->params->dropDeferred)) ;
    12911295  fprintf (fp,"%s    min-queue-connection : %s\n",indent,
     
    21282132{
    21292133  const char *filename = artFileName (article) ;
    21302134  const char *msgid = artMsgId (article) ;
    2131   double len = artSize (article);
     2135  double len = artSize (article, false);
    21322136
    21332137  d_printf (5,"Article %s (%s) was transferred\n", msgid, filename) ;
    21342138 
     
    21912195{
    21922196  const char *filename = artFileName (article) ;
    21932197  const char *msgid = artMsgId (article) ;
    2194   double len = artSize (article);
     2198  double len = artSize (article, false);
    21952199
    21962200  d_printf (5,"Article %s (%s) was rejected\n", msgid, filename) ;
    21972201 
     
    24512455  return host->params->wantStreaming ;
    24522456}
    24532457
     2458bool hostHeadersOnly (Host host)
     2459{
     2460  return host->params->headersOnly ;
     2461}
     2462
    24542463unsigned int hostMaxChecks (Host host)
    24552464{
    24562465  return host->params->maxChecks ;
     
    26712680  GETINT(s,fp,"max-connections",0,LONG_MAX,REQ,p->absMaxConnections, inherit);
    26722681  GETINT(s,fp,"max-queue-size",1,LONG_MAX,REQ,p->maxChecks, inherit);
    26732682  GETBOOL(s,fp,"streaming",REQ,p->wantStreaming, inherit);
     2683  GETBOOL(s,fp,"headers-only",REQ,p->headersOnly, inherit);
    26742684  GETBOOL(s,fp,"drop-deferred",REQ,p->dropDeferred, inherit);
    26752685  GETBOOL(s,fp,"min-queue-connection",REQ,p->minQueueCxn, inherit);
    26762686  GETREAL(s,fp,"no-check-high",0.0,100.0,REQ,p->lowPassHigh, inherit);
  • inn-2.4.1-maxartsize

    diff -ruN inn-2.4.1-maxartsize.ORIG/innfeed/host.h inn-2.4.1-maxartsize/innfeed/host.h
    old new  
    173173/* return maxChecks */
    174174unsigned int hostmaxChecks (Host host);
    175175
     176/* return whether or not we're sending a headers-only feed */
     177bool hostHeadersOnly (Host host);
     178
    176179/* return if we should drop deferred articles */
    177180bool hostDropDeferred (Host host);
    178181
  • innfeed/imap_connection.c

    diff -ruN inn-2.4.1-maxartsize.ORIG/innfeed/imap_connection.c inn-2.4.1-maxartsize/innfeed/imap_connection.c
    old new  
    10451045    int t;
    10461046
    10471047    /* make sure contents ok; this also should load it into memory */
    1048     if (!artContentsOk (art)) {
     1048    if (!artContentsOk (art, false)) {
    10491049        d_printf(0, "%s:%d AddControlMsg(): "
    10501050                 "artContentsOk() said article was bad\n",
    10511051                 hostPeerName (cxn->myHost), cxn->ident);
     
    38973897    }
    38983898
    38993899    /* make sure contents ok; this also should load it into memory */
    3900     res = artContentsOk (item->data.article);
     3900    res = artContentsOk (item->data.article, false);
    39013901    if (res==false)
    39023902    {
    39033903        if (justadded == item->data.article) {
     
    39123912    }
    39133913
    39143914    /* Check if it's a control message */
    3915     bufs = artGetNntpBuffers (item->data.article);
     3915    bufs = artGetNntpBuffers (item->data.article, false);
    39163916    if (bufs == NULL)
    39173917    {
    39183918        /* tell to reject taking this message */