Ticket #17: innfeed-headers.patch

doc/man/innfeed.conf.5

@@ -459 +459 @@
 This key requires a positive integer value. It defines the tcp/ip port
 number to use when connecting to the remote.
 .TP
+.B headers-only
+This key requires a boolean value. By default it is set to false. When
+set to true, the peer will be sent a headers-only feed. If a Bytes:
+header is not present, one will be added with as value the length of the
+complete article. This is useful for feeding a diablo server, or an
+overview-only INN server.
+Note that innfeed should send a MODE HEADFEED command to the peer to
+make sure it expects a headers-only feed, and the current code doesn't
+do that yet, partly because INN itself doesn't understand it yet so
+it would be impossible to feed a headers-only feed to INN.
+Make sure that you
+.I never
+.I ever
+send a headers-only feed to a peer that also receives a normal feed!
+.TP
 .B drop-deferred
 This key requires a boolean value. By default it is set to false. When
 set to true, and a peer replies with code 431 or 436 (try again later) just

inn-2.4.1-maxartsize

@@ -53 +53 @@
     char *fname ;               /* the file name of the article */
     char *msgid ;               /* the msgid of the article (INN tells us) */
     Buffer contents ;           /* the buffer of the actual on disk stuff */
+    Buffer headers ;            /* the buffer of just the headers */
     Buffer *nntpBuffers ;       /* list of buffers for transmisson */
+    Buffer *nntpHeaderBuffers ; /* list of header buffers for transmission */
     const void *mMapping ;      /* base of memory mapping, or NULL if none */
     bool loggedMissing ;        /* true if article is missing and we logged */
     bool articleOk ;            /* true until we know otherwise. */
@@ -76 +78 @@
    * Private functions
    */
 
-static Buffer artGetContents (Article article) ;  /* Return the buffer that
-                                                     fillContents() filled
-                                                     up. */
+  /* Return the buffer that fillContents() filled up. */
+static Buffer artGetContents (Article article, bool headersOnly) ;
 
   /* Log statistics on article memory usage. */
 static void logArticleStats (TimeoutId id, void *data) ;
 
-static bool fillContents (Article article) ;  /* Read the article's bits
-                                                 off the disk. */
+  /*  Read the article's bits off the disk. */
+static bool fillContents (Article article, bool HeadersOnly) ;
 
   /* Append buffer B to the buffer array BUFFS. */
 static void appendBuffer (Buffer b, Buffer **buffs, int *newSpot, int *curLen);
 
-static bool prepareArticleForNNTP (Article article) ;  /* Do the necessary
-                                                          CR-LF stuff */
+  /* Do the necessary CR-LF stuff */
+static bool prepareArticleForNNTP (Article article, bool headersOnly) ;
 
 static bool artFreeContents (Article art) ;  /* Tell the Article to release
                                                 its contents buffer if
@@ -108 +109 @@
 static unsigned int hashString (const char *string) ;
 
   /* Locates the article with the given message ID, in the has table. */
-static Article hashFindArticle (const char *msgid) ;
+static Article hashFindArticle (const char *msgid);
 
   /* Puts the given article in the hash table. */
 static void hashAddArticle (Article article) ;
@@ -184 +185 @@
 Article newArticle (const char *filename, const char *msgid)
 {
   Article newArt = NULL ;
-  
+
   TMRstart(TMR_NEWARTICLE);
   if (hashTable == NULL)
     {                           /* first-time through initialization. */
@@ -212 +213 @@
       newArt->msgid = xstrdup (msgid) ;
       
       newArt->contents = NULL ;
+      newArt->headers  = NULL ;
       newArt->mMapping = NULL ;
       newArt->refCount = 1 ;
       newArt->loggedMissing = false ;
@@ -235 +237 @@
       newArt->refCount++ ;
       d_printf (2,"Reusing existing article for %s\nx",msgid) ;
     }
+
   TMRstop(TMR_NEWARTICLE);
   return newArt ;
 }
@@ -268 +271 @@
           if (article->nntpBuffers != NULL)
             freeBufferArray (article->nntpBuffers) ;
 
+          if (article->nntpHeaderBuffers != NULL)
+            freeBufferArray (article->nntpHeaderBuffers) ;
+
+          if (article->headers) {
+            if (article->contents != article->headers)
+              bytesInUse -= bufferDataSize (article->headers) ;
+            delBuffer (article->headers) ;
+          }
+
           delBuffer (article->contents) ;
         }
 
@@ -360 +372 @@
 }
 
   /* return true if we have or are able to get the contents off the disk */
-bool artContentsOk (Article article)
+bool artContentsOk (Article article, bool headersOnly)
 {
   bool rval = false ;
 
-  if ( prepareArticleForNNTP (article) )
+  if ( prepareArticleForNNTP (article, headersOnly) )
     rval = true ;
 
   return rval ;
@@ -393 +405 @@
 
   /* Get a NULL terminated array of Buffers that is ready for sending via NNTP */
 
-Buffer *artGetNntpBuffers (Article article)
+Buffer *artGetNntpBuffers (Article article, bool headersOnly)
 {
-  if ( !prepareArticleForNNTP (article) )
+  if ( !prepareArticleForNNTP (article, headersOnly) )
     return NULL ;
 
-  return dupBufferArray (article->nntpBuffers) ;
+  return dupBufferArray (headersOnly ? article->nntpHeaderBuffers :
+                                        article->nntpBuffers) ;
 }
 
 
@@ -409 +422 @@
 }
 
   /* return size of the article */
-int artSize (Article article)
+int artSize (Article article, bool headersOnly)
 {
   if (article == NULL || article->contents == NULL)
     return (int)0 ;
+  if (headersOnly && article->headers == NULL)
+    return (int)0;
   return (int)bufferDataSize(article->contents);
 }
 
 
   /* return how many NNTP-ready buffers the article contains */
-unsigned int artNntpBufferCount (Article article)
+unsigned int artNntpBufferCount (Article article, bool headersOnly)
 {
-  if ( !prepareArticleForNNTP (article) )
+  if ( !prepareArticleForNNTP (article, headersOnly) )
     return 0 ;
   
-  return bufferArrayLen (article->nntpBuffers) ;
+  return bufferArrayLen (headersOnly ? article->nntpHeaderBuffers :
+                                        article->nntpBuffers) ;
 }
 
 
@@ -451 +467 @@
 
   /* return a single buffer that contains the disk image of the article (i.e.
      not fixed up for NNTP). */
-static Buffer artGetContents (Article article)
+static Buffer artGetContents (Article article, bool headersOnly)
 {
   Buffer rval = NULL ;
 
   if (article->articleOk)
     {
       if (article->contents == NULL)
-        fillContents (article) ;
+        fillContents (article, headersOnly) ;
 
       if (article->contents != NULL)
-        rval = bufferTakeRef (article->contents) ;
+        rval = bufferTakeRef (headersOnly ? article->headers :
+                                            article->contents) ;
     }
 
   return rval ;
@@ -495 +512 @@
   articleStatsId = prepareSleep (logArticleStats,ARTICLE_STATS_PERIOD,0) ;
 }
 
+/* Find the start of the body, and check if there's a Bytes: header.
+   body will point to the empty line between header and body. */
+static void findBody (char *data, size_t size, bool inWireFormat,
+                        char **body, char **bytes, char *control)
+{
+    char *p;
+    int nlseen = 1;
+
+    *bytes = NULL;
+    *control = NULL;
+    *body  = NULL;
+    if (inWireFormat) size--;
+
+    for (p = data; p < data + size; p++) {
+
+        if (nlseen && (*p == 'b' || *p == 'B') &&
+                        p < data + size - 6 &&
+                        strncasecmp(p, "Bytes:", 6) == 0)
+            *bytes = p;
+
+        if (nlseen && (*p == 'c' || *p == 'C') &&
+                        p < data + size - 8 &&
+                        strncasecmp(p, "Control:", 8) == 0)
+            *control = p;
+
+        if (inWireFormat) {
+            if (*p != '\r' && *(p+1) == '\n') {
+                p++;
+                continue;
+            }
+            if (*p == '\r' && *(p+1) == '\n')
+                p++;
+        }
+
+        if (*p == '\n') {
+           if (nlseen) {
+                *body = p - (inWireFormat ? 1 : 0);
+                break;
+           }
+           nlseen = 1;
+        } else
+           nlseen = 0;
+    }
+
+    if (*bytes == NULL) *bytes = p;
+}
+
 
   /* do the actual read of the article off disk into a Buffer that is stored
      in the Article object. The Article will end up with its contents field
@@ -504 +568 @@
      contents may be copied around after reading to insert a carriage
      return before each newline. */
 
-static bool fillContents (Article article)
+static bool fillContents (Article article, bool headersOnly)
 {
+    Buffer body;
     int fd = -1;
-    char *p;
+    char *p, *b;
     static bool maxLimitNotified ;
     bool opened;
     size_t articlesize = 0;
+    size_t bytes, hdrsize;
     char *buffer = NULL ;
     int amt = 0 ;
     size_t idx = 0, amtToRead ;
@@ -721 +787 @@
         }
     }
 
-    
+    /* In header-only mode we need to replace article->contents with
+     * a buffer with just the headers, followed by a Bytes: header. */
+    if (article->articleOk && article->contents && headersOnly) {
+
+        bytes = bufferDataSize (article->contents);
+        buffer = bufferBase (article->contents);
+
+        findBody(buffer, bytes, article->inWireFormat, &p, &bHdr, &cHdr);
+
+        if (cHdr)
+            article->headers = bufferTakeRef (article->contents) ;
+        else if ((article->headers = newBuffer (hdrsize + 24)) != NULL) {
+            p = bufferBase(article->headers);
+            memcpy(p, buffer, hdrsize);
+            if (bHdr == NULL)
+                hdrsize += sprintf(p + hdrsize, "Bytes: %d", bytes);
+            hdrsize += sprintf(p + hdrsize, "%s",
+                        article->inWireFormat ? "\r\n\r\n" : "\n\n");
+            bufferSetDataSize (article->headers, hdrsize) ;
+            bytesInUse += hdrsize;
+            byteTotal += hdrsize;
+        } else
+            warn ("ME internal failed to build headfeed buffer") ;
+    }
+
     /* If we're not useing storage api, we should close a valid file descriptor */
     if (!article->arthandle && (fd >= 0))
         close (fd) ;
 
     TMRstop(TMR_READART);
-    return (article->contents != NULL ? true : false) ;
+    return ((headersOnly ? article->headers : article->contents) != NULL ?
+            true : false) ;
 }
 
 
@@ -751 +842 @@
   /* Takes the articles contents buffer and overlays a set of new buffers on
      top of it. These buffers insert the required carriage return and dot
      characters as needed */
-static bool prepareArticleForNNTP (Article article)
+static bool prepareArticleForNNTP (Article article, bool headersOnly)
 {
   static Buffer dotFirstBuffer ;
   static Buffer dotBuffer ;
@@ -762 +853 @@
   char *start, *end ;
   Buffer contents ;
 
-  contents = artGetContents (article) ; /* returns a reference */
+  contents = artGetContents (article, headersOnly) ; /* returns a reference */
 
   TMRstart(TMR_PREPART);
   if (contents == NULL) {
     TMRstop(TMR_PREPART);
     return false ;
   }
-  else if (article->nntpBuffers != NULL)
+  else if ((!headersOnly && article->nntpBuffers != NULL) ||
+           ( headersOnly && article->nntpHeaderBuffers != NULL))
     {
       delBuffer (contents) ;
       TMRstop(TMR_PREPART);
@@ -821 +913 @@
   
       
   delBuffer (contents) ;    /* the article is still holding a reference */
-  article->nntpBuffers = nntpBuffs ;
+  if (headersOnly)
+    article->nntpHeaderBuffers = nntpBuffs ;
+  else
+    article->nntpBuffers = nntpBuffs ;
   TMRstop(TMR_PREPART);
   return true ;
 }
@@ -845 +940 @@
       }
     }
 
+  if (art->nntpHeaderBuffers != NULL)
+    {
+    if (bufferRefCount (art->nntpHeaderBuffers[0]) > 1)
+      return false ;
+    else
+      {
+        freeBufferArray (art->nntpHeaderBuffers) ;    
+        art->nntpHeaderBuffers = NULL ;
+      }
+    }
+
   ASSERT (bufferRefCount (art->contents) == 1) ;
 
   if (art->mMapping)
@@ -853 +959 @@
     bytesInUse -= bufferDataSize (art->contents) ;
   
   delBuffer (art->contents) ;
-
   art->contents = NULL ;
+
+  if (art->headers) delBuffer (art->headers) ;
+  art->headers = NULL ;
 
   return true ;
 }

inn-2.4.1-maxartsize

@@ -44 +44 @@
 
   /* return true if we have the article's contents (calling this may trigger
      the reading off the disk). */
-bool artContentsOk (Article article) ;
+bool artContentsOk (Article article, bool headersOnly) ;
 
   /* increments reference count and returns a copy of article that can be
      kept (or passed off to someone else) */
@@ -55 +55 @@
 
   /* return a list of buffers suitable for giving to an endpoint. The return
      value can (must) be given to freeBufferArray */
-Buffer *artGetNntpBuffers (Article article) ;
+Buffer *artGetNntpBuffers (Article article, bool headersOnly) ;
 
   /* return the message id stoed in the article object */
 const char *artMsgId (Article article) ;
 
   /* return size of the article */
-int artSize (Article article) ;
+int artSize (Article article, bool headersOnly) ;
 
   /* return the number of buffers that artGetNntpBuffers() would return. */
-unsigned int artNntpBufferCount (Article article) ;
+unsigned int artNntpBufferCount (Article article, bool headersOnly) ;
 
   /* tell the Article class to log (or not) missing articles as they occur. */
 void artLogMissingArticles (bool val) ;

innfeed/connection.c

@@ -3062 +3062 @@
   else
     {
       cxn->takesOkayed++ ;
-      cxn->takesSizeOkayed += artSize(artHolder->article);
+      cxn->takesSizeOkayed += artSize(artHolder->article, hostHeadersOnly(cxn->myHost));
 
       remArtHolder (artHolder, &cxn->takeRespHead, &cxn->articleQTotal) ;
       if (cxn->articleQTotal == 0)
@@ -3172 +3172 @@
   else
     {
       cxn->takesRejected++ ;
-      cxn->takesSizeRejected += artSize(artHolder->article);
+      cxn->takesSizeRejected += artSize(artHolder->article, hostHeadersOnly(cxn->myHost));
 
       remArtHolder (artHolder, &cxn->takeRespHead, &cxn->articleQTotal) ;
       /* Some(?) hosts return the 439 response even before we're done
@@ -3238 +3238 @@
       cxn->takeRespHead = NULL ;
       cxn->articleQTotal = 0 ;
       cxn->takesOkayed++ ;
-      cxn->takesSizeOkayed += artSize(artHolder->article);
+      cxn->takesSizeOkayed += artSize(artHolder->article, hostHeadersOnly(cxn->myHost));
       
       if (cxn->articleQTotal == 0)
         cxnIdle (cxn) ;
@@ -3513 +3513 @@
 
   artHolder = cxn->takeRespHead ;
   cxn->takeRespHead = NULL ;
-  cxn->takesSizeRejected += artSize(artHolder->article);
+  cxn->takesSizeRejected += artSize(artHolder->article, hostHeadersOnly(cxn->myHost));
 
   /* Some servers return the 437 response before we're done sending. */
   if (cxn->articleQTotal == 0 && !writeIsPending (cxn->myEp))
@@ -4050 +4050 @@
   ASSERT (article != NULL) ;
   
   if (cxn->state != cxnClosingS)
-    writeArray = artGetNntpBuffers (article) ;
+    writeArray = artGetNntpBuffers (article, hostHeadersOnly(cxn->myHost)) ;
   else
     writeArray = NULL ;
 
@@ -4266 +4266 @@
       /* count up all the buffers we'll be writing. One extra each time for
          the TAKETHIS command buffer*/
       for (p = cxn->takeHead ; p != NULL ; p = p->next)
-        if (artContentsOk (p->article))
-            lenArray += (1 + artNntpBufferCount (p->article)) ;
+        if (artContentsOk (p->article, hostHeadersOnly(cxn->myHost)))
+            lenArray += (1 + artNntpBufferCount (p->article, hostHeadersOnly(cxn->myHost))) ;
 
       /* now allocate the array for the buffers and put them all in it */
       /* 1 for the terminator */
@@ -4287 +4287 @@
           int i, nntpLen ;
 
           article = p->article ;
-          nntpLen = artNntpBufferCount (article) ;
+          nntpLen = artNntpBufferCount (article, hostHeadersOnly(cxn->myHost)) ;
           msgid = artMsgId (article) ;
 
           if (nntpLen == 0)
@@ -4308 +4308 @@
             }
           else
             {
-              articleBuffers = artGetNntpBuffers (article) ;
+              articleBuffers = artGetNntpBuffers (article, hostHeadersOnly(cxn->myHost)) ;
 
               /* set up the buffer with the TAKETHIS command in it.
                  12 == strlen ("TAKETHIS \n\r") */

inn-2.4.1-maxartsize

@@ -93 +93 @@
   unsigned int closePeriod;
   unsigned int dynamicMethod;
   bool wantStreaming;
+  bool headersOnly;
   bool dropDeferred;
   bool minQueueCxn;
   double lowPassLow; /* as percentages */
@@ -504 +505 @@
       params->closePeriod=CLOSE_PERIOD;
       params->dynamicMethod=METHOD_STATIC;
       params->wantStreaming=STREAM;
+      params->headersOnly=false;
       params->dropDeferred=false;
       params->minQueueCxn=false;
       params->lowPassLow=NOCHECKLOW;
@@ -1286 +1288 @@
            host->params->initialConnections) ;
   fprintf (fp,"%s    want-streaming : %s\n",indent,
            boolToString (host->params->wantStreaming)) ;
+  fprintf (fp,"%s    headers-only : %s\n",indent,
+           boolToString (host->params->headersOnly)) ;
   fprintf (fp,"%s    drop-deferred : %s\n",indent,
            boolToString (host->params->dropDeferred)) ;
   fprintf (fp,"%s    min-queue-connection : %s\n",indent,
@@ -2128 +2132 @@
 {
   const char *filename = artFileName (article) ;
   const char *msgid = artMsgId (article) ;
-  double len = artSize (article);
+  double len = artSize (article, false);
 
   d_printf (5,"Article %s (%s) was transferred\n", msgid, filename) ;
   
@@ -2191 +2195 @@
 {
   const char *filename = artFileName (article) ;
   const char *msgid = artMsgId (article) ;
-  double len = artSize (article);
+  double len = artSize (article, false);
 
   d_printf (5,"Article %s (%s) was rejected\n", msgid, filename) ;
   
@@ -2451 +2455 @@
   return host->params->wantStreaming ;
 }
 
+bool hostHeadersOnly (Host host)
+{
+  return host->params->headersOnly ;
+}
+
 unsigned int hostMaxChecks (Host host)
 {
   return host->params->maxChecks ;
@@ -2671 +2680 @@
   GETINT(s,fp,"max-connections",0,LONG_MAX,REQ,p->absMaxConnections, inherit);
   GETINT(s,fp,"max-queue-size",1,LONG_MAX,REQ,p->maxChecks, inherit);
   GETBOOL(s,fp,"streaming",REQ,p->wantStreaming, inherit);
+  GETBOOL(s,fp,"headers-only",REQ,p->headersOnly, inherit);
   GETBOOL(s,fp,"drop-deferred",REQ,p->dropDeferred, inherit);
   GETBOOL(s,fp,"min-queue-connection",REQ,p->minQueueCxn, inherit);
   GETREAL(s,fp,"no-check-high",0.0,100.0,REQ,p->lowPassHigh, inherit);

inn-2.4.1-maxartsize

@@ -173 +173 @@
 /* return maxChecks */
 unsigned int hostmaxChecks (Host host);
 
+/* return whether or not we're sending a headers-only feed */
+bool hostHeadersOnly (Host host);
+
 /* return if we should drop deferred articles */
 bool hostDropDeferred (Host host);
 

innfeed/imap_connection.c

@@ -1045 +1045 @@
     int t;
 
     /* make sure contents ok; this also should load it into memory */
-    if (!artContentsOk (art)) {
+    if (!artContentsOk (art, false)) {
         d_printf(0, "%s:%d AddControlMsg(): "
                  "artContentsOk() said article was bad\n",
                  hostPeerName (cxn->myHost), cxn->ident);
@@ -3897 +3897 @@
     }
 
     /* make sure contents ok; this also should load it into memory */
-    res = artContentsOk (item->data.article);
+    res = artContentsOk (item->data.article, false);
     if (res==false)
     {
         if (justadded == item->data.article) {
@@ -3912 +3912 @@
     }
 
     /* Check if it's a control message */
-    bufs = artGetNntpBuffers (item->data.article);
+    bufs = artGetNntpBuffers (item->data.article, false);
     if (bufs == NULL)
     {
         /* tell to reject taking this message */