src/xover.c

/* [<][>]
[^][v][top][bottom][index][help] */

DEFINITIONS

This source file includes following functions.
  1. overviewFmtGen
  2. xrefRewrite
  3. xrefRewriteCopy
  4. xover_extract_xhdr
  5. xoverIsFilt
  6. filter_header
  7. xover_nocem
  8. xover_filter
  9. xover_translate
  10. xf_find
  11. xf_write_one
  12. xf_check_buf
  13. xf_acquire_reader
  14. xf_release_reader
  15. xf_acquire_writer
  16. xf_release_writer
  17. xf_close
  18. xf_add
  19. xf_destroy_all
  20. xf_release_all
  21. xover_emit_xhdr
  22. xf_grab_outbuf
  23. xf_open
  24. xover_input
  25. xover_output
  26. xover_io
  27. xover_finish
  28. xover_process
  29. CMDxover
  30. isFieldInXover
  31. CMDxhdr

/* $Id: xover.c,v 1.4 2002/03/29 09:50:58 proff Exp $
 * $Copyright$
 */

#include "nglobal.h"
#include "network.h"

#include "acc.h"
#include "article.h"
#include "filter.h"
#include "group.h"
#include "history.h"
#include "lock.h"
#include "reg.h"

#include "xover.h"

/* 
 * we used to store all the xovers in the hashed history file db but this
 * caused the drive heads to thrash (lots of prefetch/sequential/random
 * access issues here)
 *
 * now we store xovers corresponding to a certain range (mod 512)
 * of article numbers as nnnn.xover. this
 * equates to around 200k of disk space (assuming full range). we also
 * prepend an array index of 512*32 bits which forms a pointer to
 * each xover in the xover file and the length of each xover (so we can
 * optimise reads.. useless if mmaped) 20 bits of pointer and 12 bits of
 * length (max length = 4096, longer xovers are insane and are discarded).
 *
 * if the average xover length in an xover file isnt <2048 then nasty things
 * start to happen. I've never seen an xover >1500 bytes long, but theoretically
 * massively crossposted articles with Xref's may exceed it. this is why
 * we permit upto 4096 bytes for each xover, but expect the average to easily
 * be <2048 (average seems to be 200 to 1000 depending on group)
 *
 * this method lets the file system sequentialise the xover data,
 * which we may be storing in a non-sequential manner.
 *
 * we also handle all the server and client io in an async manner, to
 * maximise throughput and TCP packet size. RTT with the server was a problem
 * before if the the cache had a lot of small "holes". 
 *
 * the pointers in the index file are stored in network byte order, so
 * the xover databases should be portable accross different endian's. e.g
 * the one database being used by several machines via nfs. note that in
 * most cases nntpcache can pull its data out of the file-system and transmit
 * it over the network faster than nfs.
 *
 * as for xhdr's, when going via a cached server, they are converted to
 * xovers, if that header is in the overview.fmt for that server
 */

#define MI XOVER_INDEX_SIZE
#define C4toINT (x) (((x)[3]>>24)|((x)[2]>>16)|((x)[1]>>8)|(x)[0])
#define UC unsigned char
#define INTtoC4 (y,x) (x)[3]=(UC)((y)>>24)&0xff;(x)[2]=(UC)((y)>>16)&0xff;(x)[1]=(UC)((y)>>8)&0xff;(x)[0]=(UC)((y)&0xff)
#undef UC

#define FILT (CurrentGroupXoverIsFilt)

/*
 * we cache the first part of every xover file (the index) as well as the
 * file descriptor's involved.
 */

static struct xover_file
{
        struct xover_file *next, *prev;
        int id;
        char *name;
        char *dir;
        char *buf;
        char *outbuf;
        int outbuf_len;
        int outbuf_used;
        int size;
        int fd;
        bool reader;
        bool writer;
        time_t mtime;
} *xover_file_head = NULL, *xover_file_tail = NULL;

static int xover_file_nodes = 0; /* number in cache */

static enum xover_state
{
        listening, sending, receiving, dead
} xover_server_state, xover_client_state;

static fd_set fdset;
static int outstanding = 0;

EXPORT struct strList *overviewFmtGen(struct server_cfg *scfg, struct strList *l, n_u32 *hashp)
/* [<][>][^][v][top][bottom][index][help] */
{
    struct strList *list=NULL;
    int fmt_len=0, fmt_entries=0;
    struct strList *xref = NULL;
    n_u32 hash;
    for (hash = 0xadeadfed; l; l = l->next)
    {
            list=strXMListAdd(list, l->data);   /* place into shared memory */
            hash=strHash(hash, l->data);
            fmt_len += strlen(l->data) + 2;
            fmt_entries++;
            if (scfg && !xref && strnCaseEq(l->data, "Xref:", 5))
                xref = l;
    }
    if (list)
    {
            list = list->head;
            if (scfg && (!scfg->share->overview_fmt_hash || scfg->share->overview_fmt_hash != hash))
            {
                    struct strList *t;
                    t = scfg->share->overview_fmt;
                    scfg->share->overview_fmt = list->head;
                    scfg->share->overview_fmt_xref = xref;
                    scfg->share->overview_fmt_hash = hash;
                    scfg->share->overview_fmt_xref_full = xref? strnCaseEq(xref->data, "Xref:full", 9): FALSE;
                    if (t)
                        strXMListFree(t);
            }
    }
    if (hashp)
        *hashp = hash;
    return list;
}

/*
 * XrefRewrite
 *
 * Xref (s): is terminated by one of \0 \n \r \t
 *
 * returns length of new xref
 */

EXPORT int xrefRewrite(struct server_cfg *scfg, char *s, bool full)
/* [<][>][^][v][top][bottom][index][help] */
{
        char *p, *o, *s_orig=s;
        SKIPSPACE(s);
        if (full)
        {
                SKIPNOWHITE(s); /* Xref: */
                SKIPSPACE(s);
        }
        /* we don't modify the hostname field - it may actually be
         * useful to more intelligent newsreaders
         */
        SKIPNOSPACE(s); /* hostname */
        SKIPSPACE(s);
        for (o=s; *s && *s!='\t' && *s!='\r' && *s!='\n';)
        {
                bool keep;
                char c;
                p=s;
                while (*s && *s!=':')
                        s++;
                if (p==s)
                        break;
                c=*s;
                *s='\0';
                keep = getServerGroup(p) == scfg ? TRUE : FALSE;
                *s = c;
                SKIPNOWHITE(s);
                SKIPSPACE(s);
                if (keep)
                {
                        if (o!=p)
                        {
                                int len=s-p;
                                memmove(o, p, len);
                                o+=len;
                        } else
                                o=s;
                }
        }
        if (s!=o)
                memset(o, ' ', s-o);
        return s_orig-o;
}

/* 
 * as above, but strip and copy into dst. caller is responsibly for
 * allocating a correct sized dst. this version is optimised for
 * xover_translate.
 * returns length of chars in dst
 */

EXPORT int xrefRewriteCopy(struct server_cfg *scfg, char *s, char *dst, bool full)
/* [<][>][^][v][top][bottom][index][help] */
{
        char *p;
        char *odst = dst;
        while (*s == ' ')
                *dst++=*s++;
        if (full)
        {
                while (*s && !isspace(*s))
                        *dst++=*s++; /* Xref: */
                while (*s == ' ')
                        *dst++=*s++;
        }
        /* we don't modify the hostname field - it may actually be
         * useful to more intelligent newsreaders
         */
        while (*s && !isspace(*s))
                *dst++=*s++; /* hostname */
        while (*s == ' ')
                *dst++=*s++;
        for (; *s && !isspace(*s);)
        {
                bool keep;
                char c;
                p=s;
                while (*s && *s!=':')
                        *dst++=*s++; /* alt.suicide:1234 */
                c=*s;
                *s='\0';
                keep = getServerGroup(p) == scfg ? TRUE : FALSE;
                *s=c;
                if (keep)
                {
                        while (*s && !isspace(*s))
                                *dst++=*s++;
                } else
                {
                        dst -= s-p;
                        while (*s && !isspace(*s))
                                s++;
                }
                while (*s == ' ')
                        *dst++=*s++;
        }
        *dst='\0';
        return dst-odst;
}
        
/*
 * returns pointer to start of header in xover or null. 
 */

static char *xover_extract_xhdr (char *xover, struct strList *i_fmt, char *header)
/* [<][>][^][v][top][bottom][index][help] */
{
        for (; i_fmt; i_fmt = i_fmt->next)
        {
                if (strCaseEq (i_fmt->data, header))
                        return xover;
                xover = strchr (xover, '\t');
                if (!xover)
                        break;
                xover++;
        }
        return NULL;
}

EXPORT bool xoverIsFilt(struct authent *auth)
/* [<][>][^][v][top][bottom][index][help] */
{
        struct filter *f;
        struct filter_chain *fc;
        if (!auth || !con->xoverFilters)
                return FALSE;
        for (fc=auth->filter_chain; fc; fc=fc->next)
        {
                for (f=fc->filter; f; f=f->next)
                {
                        if (f->scope == sc_header)
                                return TRUE;
                }
        }
        return FALSE;
}

static char *filter_header (char *header, char *text)
/* [<][>][^][v][top][bottom][index][help] */
{
        struct filter *f;
        struct filter_chain *fc;
        int score;
        for (fc=CurrentGroupAuth->filter_chain; fc; fc=fc->next)
        {
                score = 0;
                for (f=fc->filter; f; f=f->next)
                {
                        if (f->scope != sc_header ||
                            !strCaseEq(header, f->scope_header))
                           
                                continue;
#ifdef USE_REGEX
                        if (nn_regexec(&f->preg, text, strlen(text), 0, 0, 0)==0)
#else
                        if (matchExp(f->pat, text, f->ignore_case, 0))
#endif
                        {
                                score+=f->weight;
                                switch (f->weight_op)
                                {
                                case '*':
                                        score*=f->weight;
                                        break;
                                case '/':
                                        score/=f->weight;
                                        break;
                                default:
                                        break;
                                }
                        }
                        if (score >= 10000)
                                return fc->name;
                        if (score <=-10000)
                                return NULL;
                }
                if (score>=100)
                                return fc->name;
        }
        return NULL;
}

static char *xover_nocem (char *xover, struct strList *i_fmt)
/* [<][>][^][v][top][bottom][index][help] */
{
        char *ret=NULL;
        for (; i_fmt; i_fmt = i_fmt->next)
        {
                char *xover2 = strchr (xover, '\t');
                if (!strCaseEq (i_fmt->data, "Message-ID:"))
                {
                         if (!xover2)
                                  return NULL;
                         xover = xover2+1;
                         continue;
                }
                if (xover2)
                        *xover2 = '\0';
                if (xover[0] == '<' &&
                    xover2[-1] == '>')
                    {
                        char *p;
                        xover2[-1] = '\0';
                        p = hisGet(xover+1);
                        if (p && strCaseEq (p, SPAM))
                            ret = SPAM;
                        xover2[-1] = '>';
                    }
                else
                        logd (("badly formed msgid '%s' seen in xover_nocem ()", xover));
                xover = xover2;
                if (!xover)
                        break;
                *xover = '\t';
                if (ret)
                        return ret;
                xover++;
        }
        return ret;
}

static char *xover_filter (char *xover, struct strList *i_fmt)
/* [<][>][^][v][top][bottom][index][help] */
{
        char *ret=NULL;
        for (; i_fmt; i_fmt = i_fmt->next)
        {
                char *xover2 = strchr (xover, '\t');
                if (xover2)
                        *xover2 = '\0';
                ret = filter_header(i_fmt->data, xover);
                xover = xover2;
                if (!xover)
                        break;
                *xover = '\t';
                if (ret)
                        return ret;
                xover++;
        }
        return ret;
}

/* 
 * convert one xover format into another (xover is minus leading artnum\t)
 * the input xover needs to have EOL stripped.
 */

static char *xover_translate (char *xover, struct server_cfg *cf)
/* [<][>][^][v][top][bottom][index][help] */
{
        static char buf[MAX_XOVER];
        struct strList *i_fmt = cf->share->overview_fmt, *o_fmt = overviewFmt;
        char *o = buf;
        int n = 0;
        for (; o_fmt; o_fmt = o_fmt->next, n++)
        {
                /*
                 * round robin for efficiency with non-diverging data sets.
                 */
                struct strList *i_fmt_orig = i_fmt;
                do
                {
                        if (strCaseEq (i_fmt->data, o_fmt->data))
                        {
                                i_fmt = i_fmt->next;
                                break;
                        }
                        i_fmt = i_fmt->next;
                        if (!i_fmt)
                                i_fmt = cf->share->overview_fmt;
                        n++;
                } while (i_fmt != i_fmt_orig);

                /*
                 * TODO: merge below into round robin
                 */

                if (i_fmt != i_fmt_orig)
                {
                        char *i = xover;
                        int tabs = 0;
                        if (!i_fmt)
                                i_fmt = i_fmt_orig;
                        do
                        {
                                if (n == tabs++)
                                {
                                        if (cf->share->overview_fmt_xref == i_fmt)
                                        {
                                                o+=xrefRewriteCopy(cf, i, o, cf->share->overview_fmt_xref_full);
                                        } else
                                        {
                                                while (*i && *i != '\t')
                                                        *o++ = *i++;
                                        }
                                        break;
                                }
                        } while (*i && (i = strchr (i, '\t')) && i++);
                }
                *o++ = '\t';
        }
        *--o = '\0';
        return buf;
}

/*
 * find node
 */

static struct xover_file *xf_find (int i)
/* [<][>][^][v][top][bottom][index][help] */
{
        /*
         * we work backwards, as newer nodes are
         * close to the end of the cache
         */
        struct xover_file *xf = xover_file_tail;
        for (; xf; xf = xf->prev)
        {
                if (xf->id == i && strEq (xf->dir, CurrentDir))
                        return xf;
        }
        return NULL;
}

static bool xf_write_one (struct xover_file *xf)
/* [<][>][^][v][top][bottom][index][help] */
{
        if (lseek (xf->fd, 0, SEEK_SET) != 0 ||
            write (xf->fd, xf->buf, 4 * MI) != 4 * MI ||
            (xf->outbuf &&
             (lseek (xf->fd, 0, SEEK_END) < 4 * MI ||
              write (xf->fd, xf->outbuf, xf->outbuf_used) != xf->outbuf_used)))
        {
                loge (("unable to write/lseek '%s/%s' correctly (...unlinking)", xf->dir, xf->name));
                unlink (xf->name);
                return FALSE;
        }
        xf->mtime = time (NULL);
        return TRUE;
}

static bool xf_check_buf (struct xover_file *xf, bool readAll)
/* [<][>][^][v][top][bottom][index][help] */
{
        struct stat st;
        int size;

        if (fstat (xf->fd, &st) < 0)
        {
                loge (("unable to fstat '%s/%s'", xf->dir, xf->name));
                return FALSE;
        }
        if (st.st_size < MI * 4)
        {
                loge (("short file, %d bytes '%s/%s'", (int)st.st_size, xf->dir, xf->name));
                return FALSE;
        }
        xf->size = st.st_size;
        if (!xf->buf || st.st_mtime > xf->mtime)
        {
                if (lseek (xf->fd, 0, SEEK_SET) != 0)
                {
                        loge (("couldn't lseek to start of '%s/%s'", xf->dir, xf->name));
                        return FALSE;
                }
                if (readAll)
                        size = xf->size;
                else
                        size = MI * 4;
                xf->buf = Srealloc (xf->buf, size);
                if (read (xf->fd, xf->buf, size) != size)
                {
                        loge (("couldn't read all of %d bytes from '%s/%s'", size, xf->dir, xf->name));
                        free (xf->buf);
                        xf->buf = 0;
                        return FALSE;
                }
        } else
        {
                if (!readAll)
                        return TRUE;
                if (lseek (xf->fd, MI * 4, SEEK_SET) != MI * 4)
                {
                        loge (("couldn't lseek to %d in '%s/%s'", MI * 4, xf->dir, xf->name));
                        return FALSE;
                }
                xf->buf = Srealloc (xf->buf, xf->size);
                if (read (xf->fd, xf->buf + MI * 4, xf->size - MI * 4) != xf->size - MI * 4)
                {
                        loge (("couldn't read all of %d bytes from '%s/%s'", xf->size - MI * 4, xf->dir, xf->name));
                        free (xf->buf);
                        xf->buf = 0;
                        return FALSE;
                }
        }
        xf->mtime = st.st_mtime;
        return TRUE;
}

#if 0           /* not needed at the moment */
static bool xf_acquire_reader (struct xover_file *xf, bool readAll)
/* [<][>][^][v][top][bottom][index][help] */
{
        if (xf->fd >= 0 && !xf->reader)
                if (locksh (xf->fd) == 0)
                        if (!(xf->reader = xf_check_buf (xf, readAll)))
                                lockun (xf->fd);
        return xf->reader;
}
#endif

static void xf_release_reader (struct xover_file *xf)
/* [<][>][^][v][top][bottom][index][help] */
{
        if (xf->fd >= 0 && xf->reader)
                lockun (xf->fd);
        xf->reader = FALSE;
}

static bool xf_acquire_writer (struct xover_file *xf, bool readAll)
/* [<][>][^][v][top][bottom][index][help] */
{
        if (xf->reader)
                xf_release_reader (xf);
        if (xf->fd >= 0 && !xf->writer)
                if (lockex (xf->fd) == 0)
                        if (!(xf->writer = xf_check_buf (xf, readAll)));
                                lockun (xf->fd);
        return xf->writer;
}

static bool xf_release_writer (struct xover_file *xf)
/* [<][>][^][v][top][bottom][index][help] */
{
        bool res = TRUE;
        if (xf->fd > 0 && xf->writer && xf->buf)
                res = xf_write_one (xf);
        if (xf->outbuf)
        {
                free (xf->outbuf);
                xf->outbuf = NULL;
                xf->outbuf_used = 0;
        }
        if (xf->fd > 0 && xf->writer)
                lockun (xf->fd);
        xf->writer = FALSE;
        return res;
}

static void xf_close (struct xover_file *xf)
/* [<][>][^][v][top][bottom][index][help] */
{
        xf_release_reader (xf);
        xf_release_writer (xf);
        if (xf->name)
        {
                free (xf->name);
                xf->name = NULL;
        }
        if (xf->dir)
        {
                free (xf->dir);
                xf->dir = NULL;
        }
        if (xf->buf)
        {
                free (xf->buf);
                xf->buf = NULL;
        }
        if (xf->fd > 0)
        {
                close (xf->fd);
                xf->fd = -1;
        }
        xf->id = -1;
}

/*
 * add node. we presume con->MaxXoverNodes > 1
 */

static struct xover_file *xf_add (int i)
/* [<][>][^][v][top][bottom][index][help] */
{
        struct xover_file *xf;
        if (xover_file_nodes>=con->maxXoverNodes)
        {
               /*
                * the cache is full. close the oldest node
                * and take over it's data space with the
                * new node. our linked list is now a
                * circular stack (i.e avoid malloc/free)
                */
                xf_close (xover_file_head);
                xover_file_tail->next = xf = xover_file_head;
                xf->next->prev = NULL;
                xover_file_head = xf->next;
                memset (xf, 0, sizeof *xf);
        } else
        {
                xf = Scalloc (1, sizeof *xf);
                if (xover_file_tail)
                        xover_file_tail->next = xf;
                else
                        xover_file_head = xf;
                xover_file_nodes++;
        }
        xf->prev = xover_file_tail;
        xover_file_tail = xf;
        xf->id = i;
        return xf;
}

#if 0
static void xf_destroy_all ()
/* [<][>][^][v][top][bottom][index][help] */
{
        struct xover_file *xf = xover_file_head;
        while (xf)
        {
                struct xover_file *xtmp;
                if (xf->id >= 0)
                        xf_close (xf);
                xtmp = xf->next;
                free (xf);
                xf = xtmp;
        }
        xover_file = xover_file_tail = NULL;
}
#endif

static void xf_release_all ()
/* [<][>][^][v][top][bottom][index][help] */
{
        struct xover_file *xf = xover_file_tail;
        for (; xf; xf = xf->prev)
                if (!xf_release_writer (xf))
                        xf_close (xf);
}

static int xover_emit_xhdr (char *xover, int xn, struct strList *i_fmt, char *xhdr)
/* [<][>][^][v][top][bottom][index][help] */
{
        int len;
        char *hdr=xover_extract_xhdr(xover, i_fmt, xhdr);
        len=emitf("%d ", xn);
        if (hdr)
        {
                char *tab = strchr(hdr, '\t');
                if (tab)
                        *tab = '\0';
                len+=emitrn(hdr);
                if (tab)
                        *tab = '\t';
        } else
                len+=emitrn("(none)");
        return len;
}

static void xf_grab_outbuf (struct xover_file *xf, int len)
/* [<][>][^][v][top][bottom][index][help] */
{
#define GRAB_SIZE 32768
        if (!xf->outbuf)
        {
                xf->outbuf = Smalloc (GRAB_SIZE);
                xf->outbuf_len = GRAB_SIZE;
        } else if (xf->outbuf_used + len > xf->outbuf_len)
                xf->outbuf = Srealloc (xf->outbuf, (xf->outbuf_len += GRAB_SIZE));
#undef GRAB_SIZE
}

static struct xover_file *xf_open (int index)
/* [<][>][^][v][top][bottom][index][help] */
{
        int fd;
        char fn[64];
        struct stat st;
        struct xover_file *xf;

        sprintf (fn, "%d_xover", index * MI);
        fd = open (fn, O_RDWR | O_CREAT, 0664);
        if (fd < 0)
        {
                loge (("unable to open/create '%s/%s'", CurrentDir, fn));
                return NULL;
        }
        xf = xf_add (index);
        xf->fd = fd;
        xf->name = Sstrdup (fn);
        xf->dir = Sstrdup (CurrentDir);
        if (lockex (xf->fd) != 0)
        {
                loge (("unable to lockex '%s/%s' (...unlinking)", xf->dir, xf->name));
                unlink (xf->name);
                xf_close (xf);
                return NULL;
        }
        if (fstat (xf->fd, &st) != 0)
        {
                loge (("unable to fstat '%s/%s' (...unlinking)", xf->dir, xf->name));
                unlink (xf->name);
                lockun (xf->fd);
                xf_close (xf);
                return NULL;
        }
        if (st.st_size < MI * 4 && ftruncate (xf->fd, MI * 4) != 0)
        {
                loge (("unable to truncate '%s/%s' (...unlinking)", xf->dir, xf->name));
                unlink (xf->name);
                lockun (xf->fd);
                xf_close (xf);
                return NULL;
        }
        lockun (xf->fd);
        return xf;
}

static bool xover_input (char *xhdr, int min, int max)
/* [<][>][^][v][top][bottom][index][help] */
{
        int len;
        int response;
        char *p;
        struct xover_file *xf;
        int t;
        n_u32 *idx;
        n_u32 i32;
        char xover[MAX_XOVER + 20];
        bool f_translated;
        if (!(len=Cget (xover, sizeof xover)))
        {
                xover_server_state = dead;
                outstanding = 0;
                CurrentScfg->share->xover_fail++;
                return FALSE;
        }
        if (EL (xover))
        {
                xover_server_state = listening;
                outstanding--;
                CurrentScfg->share->xover_good++;
                return TRUE;
        }
        response = strToi (xover);
        if (xover_server_state == listening)
        {
                if (response != NNTP_OVERVIEW_FOLLOWS_VAL)
                {
                        outstanding--;
                        if (xover_client_state == listening)
                        {
                                emit (xover);
                                return FALSE;
                        } else
                                return TRUE; /* not sure this is correct behavior */
                }
                xover_server_state = sending;
                return TRUE;
        } /* must be an xover then .. */
        if (response<min || response>max)
                return TRUE; /* not what we asked for */
        if (xover_client_state == listening)
        {
                emitrn (xhdr? NNTP_HEAD_FOLLOWS: NNTP_OVERVIEW_FOLLOWS);
                xover_client_state = receiving;
        }
        /* skip artnum */
        p = strchr (xover, '\t');
        if (!p)
        {
                logw (("xover %d in %s contains no fields", response, CurrentGroup));
                return TRUE;
        }
        ++p;
        strnStripEOL (xover, len);
        if (overviewFmt_hash != CurrentGroupScfg->share->overview_fmt_hash)
        {
                p = xover_translate (p, CurrentGroupScfg);
                f_translated = TRUE;
        } else
        {
                f_translated = FALSE;
                if (CurrentGroupScfg->share->overview_fmt_xref)
                {
                        char *t;
                        struct strList *f=CurrentGroupScfg->share->overview_fmt;
                        for (t=p; f && t; f=f->next, t=strchr(t, '\t'))
                        {
                                if (*++t != '\t' && f == CurrentGroupScfg->share->overview_fmt_xref)
                                {
                                        xrefRewrite(CurrentGroupScfg, t, CurrentGroupScfg->share->overview_fmt_xref_full);
                                        break;
                                }
                        }
                }
        }
        t = response / MI;
        if (!(xf = xf_find (t)))
        {
                if (!(xf = xf_open (t)))
                        goto auth_xover;

#if 0
                /* This is now always done in xover_process. */
                /*
                 * set failed flag in index for xover region asked.
                 * we then knock em out one by one as xovers come in
                 *
                 * the bit map per xover cache file initially looks like this:
                 *
                 *    0           min                                max             511
                 *    |            |                                  |               |
                 *    uuuuuuuuuuuuuffffffffffffffffffffffffffffffffffffuuuuuuuuuuuuuuuu
                 *        |                             |                       |
                 *      unknown                       failed                 unknown
                 *
                 *   after the xover responses come in, we transform to:
                 *
                 *                min            cached              max
                 *                 |               |                  |
                 *    uuuuuuuuuuuuuff******f******************f********uuuuuuuuuuuuuuuu
                 *        |         \______|_________________/                  |
                 *        |                     |                               |
                 *      unknown           failed/expired                     unknown
                 *
                 * keep in mind that min and max may be outside 0-511 making the
                 * current xover bitmap only a "window" into the bitmaps tickled
                 * by the request
                 *
                 * the next time an xover is requested on the region:
                 *
                 *      u - will be fetched from the server
                 *      f - will be skipped
                 *      * - will be pulled from the cache
                 *
                 * note that nocem detected spams are marked as f
                 */
                base = (min<t*MI)? 0: min%MI;
                memset (xf->buf+base*4, 0xff, (MIN(MI, max+1-t*MI)-base)*4);
#endif
        }
        if (!xf_acquire_writer (xf, FALSE))
                goto auth_xover;
        len = strlen (p) + 1;
        idx = (n_u32 *) xf->buf;
        i32 = idx[response % MI];
        if (CurrentGroupNocem)
        {
                char *blocker = xover_nocem(p, f_translated? overviewFmt: CurrentGroupScfg->share->overview_fmt);
                if (blocker)
                {
                        logd (("nocem filter %s blocked xover of %s/%d", blocker, CurrentDir, response));
                        idx[response % MI] = 0xffffffff;
                        CS->nocem_blocked++;
                        return TRUE;
                }
        }
        if (!CurrentGroupNode->lo_xover || CurrentGroupNode->lo_xover > response)
            CurrentGroupNode->lo_xover = response;
        if (CurrentGroupNode->hi_xover < response)
            CurrentGroupNode->hi_xover = response;

        if (i32 == 0 || i32 == 0xffffffff)      /* only if we don't have it already */
        {
                idx[response % MI] = htonl ((len << 20) | (((xf->size > 0) ? xf->size : MI * 4) + xf->outbuf_used));
                xf_grab_outbuf(xf, len);
                memcpy (xf->outbuf + xf->outbuf_used, p, len);
                xf->outbuf_used += len;
        }
auth_xover:
        {
        char *blocker;
        if (FILT && (blocker = xover_filter(p, f_translated? overviewFmt: CurrentGroupScfg->share->overview_fmt)))
        {
                logd (("content filter %s blocked xover of %s/%d", blocker, CurrentDir, response));
                CS->filter_blocked++;
                return TRUE;
        }
        if (f_translated)
        {
                if (xhdr)
                        xover_emit_xhdr(p, response, overviewFmt, xhdr);
                else
                        emitf ("%d\t%s\r\n", response, p);
        } else
        {
                if (xhdr)
                        xover_emit_xhdr(p, response, CurrentGroupScfg->share->overview_fmt, xhdr);
                else
                        emitrn (xover);
        }}
        return TRUE;
}

static void xover_output (int min, int max)
/* [<][>][^][v][top][bottom][index][help] */
{
        Cemitf("xover %d-%d\r\n", min, max);
        Cflush ();
        outstanding++;
}

static bool xover_io (int min, int max, char *xhdr, int orig_min, int orig_max)
/* [<][>][^][v][top][bottom][index][help] */
{
        while (min || outstanding>0)
        {
                if (xover_server_state != dead && (!min || Smore (CurrentGroupScfg->fd)))
                {
                        if (xover_input (xhdr, orig_min, orig_max) && Smore (CurrentGroupScfg->fd))
                                continue;
                }
                /*
                 * here, not above, as it is possible output will block due to input not
                 * being cleared
                 */
                if (min)
                {
                        xover_output (min, max);
                        return TRUE;
                }
                if (outstanding>0) /* TODO: work out when we have to wait on a socket */
                        usleep (5000); /* doesn't use SIGALRM */
        }
        if (xover_client_state == listening)
                emitrn (xhdr? NNTP_HEAD_FOLLOWS: NNTP_OVERVIEW_FOLLOWS);
        emitrn (".");
        return TRUE;
}

static bool xover_finish (char *xhdr, int orig_min, int orig_max)
/* [<][>][^][v][top][bottom][index][help] */
{
        bool ret;
        ret = xover_io (0, 0, xhdr, orig_min, orig_max);        /* finish all pending i/o */
        xf_release_all ();
        return ret;
}

static bool xover_process (int min, int max, char *xhdr)
/* [<][>][^][v][top][bottom][index][help] */
{
        int colmin = min, colmax, colmax_high=0;
        int xmin = min / MI, xmax = max / MI;
        int xi;
        FD_ZERO (&fdset);
        xover_client_state = xover_server_state = listening;
        outstanding = 0;
        for (xi = xmin; xi <= xmax; xi++)
        {
                char xover[20 + MAX_XOVER];
                int start, end;
                n_u32 *idx;
                char *xover_buf = NULL;         /* not null when entire xover db file in incore */
                int n;
                int xn;
                struct xover_file *xf;

                 if (!(xf = xf_find (xi)) && !(xf = xf_open (xi)))
                        continue;
                /*
                 * we have two techniques depending on how much of
                 * the xover file is required. if we require more than 16
                 * xovers then we read the whole xover file into memory
                 * else we merely lseek() to the required positions.
                 * this appears to be more efficient than mmap or readv.
                 */
                start = (xi == xmin) ? min % MI : 0;
                end = (xi == xmax) ? max % MI : MI - 1;         /* upto and including */
                if (!xf_acquire_writer (xf, end - start >= 16))
                        goto bad;
                if (end - start >= 16)
                        xover_buf = xf->buf + MI * 4;
                idx = (n_u32 *) (xf->buf);
                for (n = start, xn = start + MI * xi, colmax = 0; n <= end; n++, xn++)
                {
                        char *blocker;
                        int offset;
                        int len;
                        char *xov;
                        n_u32 i = idx[n];
                        /*
                         * if we couldn't get it before, don't try now
                         */
                        if (i == 0xffffffff)
                        {
                                if (colmax == 0)
                                        colmin = xn+1;
                                continue;
                        }
                        if (i==0)
                        {
                                idx[n] = 0xffffffff; /* set failed flag, xover_input will judge */
                                colmax = xn;
                                continue;
                        }
                        if (colmax != 0)
                        {
                                xover_io (colmin, colmax, xhdr, min, max);
                                colmax_high=colmax;
                                colmax = 0;
                        }
                        i = ntohl (i);
                        colmin = xn + 1;
                        offset = i & 0xfffff;
                        len = (i >> 20) & 0xfff;
                        if (xover_client_state == listening)
                        {
                                emitrn (xhdr? NNTP_HEAD_FOLLOWS: NNTP_OVERVIEW_FOLLOWS);
                                xover_client_state = receiving;
                        }
                        if (xover_buf)
                        {
                                if (offset >= xf->size)
                                {
                                        loge (("bad xover offset in '%s/%s' (%d >= %d) (...unlinked)", xf->dir, xf->name, offset, xf->size));
                                        goto bad;
                                }
                                xov = xover_buf + offset - 4 * MI;
                        } else
                        {
                                if (lseek (xf->fd, offset, SEEK_SET) != offset)
                                {
                                        loge (("lseek failed for '%s/%s' (...unlinked)", xf->dir, xf->name));
                                        goto bad;
                                }
                                if (read (xf->fd, xover, len) != len)
                                {
                                        loge (("only read part of '%s/%s' (...unlinked)", xf->dir, xf->name));
                                        goto bad;
                                }
                                xov = xover;
                        }
                        if (CurrentGroupNode->lo_xover > xn)
                            CurrentGroupNode->lo_xover = xn;
                        if (CurrentGroupNode->hi_xover < xn)
                            CurrentGroupNode->hi_xover = xn;
                        if (FILT && (blocker = xover_filter(xov, overviewFmt)))
                        {
                                logd (("content filter %s blocked xover of '%s/%d'", blocker, CurrentDir, xn));
                                CS->filter_blocked++;
                        }
                        else
                        {
                                if (xhdr)
                                        xover_emit_xhdr (xov, xn, overviewFmt, xhdr);
                                else
                                        emitf ("%d\t%s\r\n", xn, xov);
                        }
                }
                if (xover_buf)
                        Srealloc (xf->buf, MI * 4);     /* keep only the index in the cache */
                xf_release_reader (xf);
                continue;
              bad:
                logw (("unlinking %s dir %s cwd %s", xf->name, xf->dir, CurrentDir));
                unlink (xf->name);
                xf_close (xf);
                continue;
        }
        if (colmax_high < max && colmin <= max)
                xover_io (colmin, max, xhdr, min, max);
        xover_finish (xhdr, min, max);
        return TRUE;
}

EXPORT bool CMDxover (char *args)
/* [<][>][^][v][top][bottom][index][help] */
{
        int min=0, max = 0;

        sscanf (args, "%*s %d-%d", &min, &max);
        if (min<0 || max < 0 || (max > 0 && min > max))
        {
                emitrn (NNTP_SYNTAX_USE);
                return FALSE;
        }
        CS->by_artnum++;
        if (!*CurrentGroup || !setGD())
        {
                emitrn (NNTP_NOTINGROUP);
                return FALSE;
        }
        if (!CurrentGroupScfg->share->overview_fmt)
        {
                emitf("%d no overview.fmt for this server\r\n", NNTP_DONTHAVEIT_VAL);
                return FALSE;
        }
        if (CurrentGroupScfg->xover_timeout > 0)
        {
                if (min == 0 && max == 0)
                {
                        min = max = CurrentGroupArtNum;
                } else
                {
                        int nmax = getHi(CurrentGroupNode);
                        int nmin = getLo(CurrentGroupNode);
                        if (min<nmin)
                                min = nmin;
                        if (max<1 || max >nmax)
                                max = strchr (args, '-')? nmax: min;
                }
                xover_process (min, max, NULL);
        } else
        {
                char line[MAX_LINE];
                    
                Cemit (args);
                Cflush ();
                if (!Cget (line, sizeof line))
                {
                        emitrn (NNTP_SERVERDOWN);
                        return FALSE;
                }
                emit (line);
                if (strToi (line) == NNTP_OVERVIEW_FOLLOWS_VAL)
                {
                        int len;
                        while ((len=Cget (line, sizeof line)) && !EL(line))
                        {
                                char *blocker;
                                char *p=strchr(line, '\t');
                                if (!p)
                                {
                                        emit (line);
                                        continue;
                                }
                                if (CurrentGroupNocem)
                                {
                                        blocker = xover_nocem(p+1, CurrentGroupScfg->share->overview_fmt);
                                        if (blocker)
                                        {
                                                logd (("nocem filter %s blocked xover of %s/%d", blocker, CurrentDir, strToi(line)));
                                                CS->nocem_blocked++;
                                                continue;
                                        }
                                }
                                if (FILT && (blocker = xover_filter(p+1, CurrentGroupScfg->share->overview_fmt)))
                                {
                                        logd (("content filter %s blocked xover of %s/%d", blocker, CurrentDir, strToi(line)));
                                        CS->filter_blocked++;
                                        continue;
                                }
                                if (overviewFmt_hash != CurrentGroupScfg->share->overview_fmt_hash)
                                {
                                        *p++ = '\0';    /* kill tab */
                                        strStripEOL(p);
                                        p = xover_translate (p, CurrentGroupScfg);
                                        emitf("%s\t%s\r\n", line, p);
                                } else
                                        emit(line);
                        }
                        emitrn(".");
                }
        }
        return TRUE;
}

static bool isFieldInXover(char *field)
/* [<][>][^][v][top][bottom][index][help] */
{
        struct strList *o_fmt;
        for (o_fmt = overviewFmt->head; o_fmt; o_fmt = o_fmt->next)
                if (strCaseEq (o_fmt->data, field))
                        return TRUE;
        return FALSE;
}
        
EXPORT bool CMDxhdr (char *args)
/* [<][>][^][v][top][bottom][index][help] */
{
        int min, max = 0;

        char msgid[MAX_MSGID];
        char buf[MAX_LINE];
        char header[MAX_HEADER];
        char field[MAX_HEADER];
        int len;

        if (sscanf (args, "%*s %127s", header) != 1)
        {
                emitrn (NNTP_SYNTAX_USE);
            bad:
                return FALSE;
        }
        if (strSnip(args, 0, "<", ">\r\n", msgid, sizeof msgid) < 1)
        {
                sscanf (args, "%*s %*s %d-%d", &min, &max);
                if (min<0 || max < 0 || (max > 0 && min > max) || (max == 0 && min ==0))
                        goto bad;
        }
        if (!*msgid && (!*CurrentGroup || !setGD()))
        {
                emitrn (NNTP_NOTINGROUP);
                goto bad;
        }
        sprintf(field, "%.126s:", header);
        if (*msgid) /* TODO: cache xhdr header <msgid> */
        {
                struct server_cfg *osrvrs = ServerList;
                struct server_cfg *cf;
                int n;

                CS->by_msgid++;
                Cemit (args);
                Cflush ();
                if (con->maxMsgIDsearch<1 || !Cget (buf, sizeof buf))
                {
                        CurrentScfg->share->xhdr_fail++;
                        emitrn (NNTP_SERVERTEMPDOWN);
                        return FALSE;
                }
                if (strToi (buf) == NNTP_HEAD_FOLLOWS_VAL)
                {
                        emit (buf);
                        while ((len=Cget (buf, sizeof buf)) && !EL (buf))
                        {
                                if (FILT)
                                {
                                        char *p;
                                        char *blocker;
                                        for (p=buf; *p && *p!=' ' && *p!='\t'; p++) ;
                                        if (*p && *++p && (blocker = filter_header(header, p)))
                                        {
                                                logd (("content filter %s blocked xhdr of <%s>", blocker, msgid));
                                                CS->filter_blocked++;
                                                continue;
                                        }
                                }
                                emit (buf);
                        }
                        emitrn(".");
                        CurrentScfg->share->xhdr_good++;
                        return TRUE;
                }
                /*
                 * we are doing a get by <art-id> and current server didn't have it,
                 * so loop through all servers until
                 * we find one that does have it and make it the current server
                 */

                for (n=0, osrvrs = ServerList; osrvrs && n<con->maxMsgIDsearch; osrvrs = osrvrs->next)
                {
                        if (CurrentScfg==osrvrs)
                                continue;
                        n++;
                        cf = attachServer (osrvrs);
                        if (!cf)
                                continue;
                        Cfemit (cf, args);
                        Cfflush (cf);
                        if (!(len=Cfget (cf, buf, sizeof (buf))))
                                continue;
                        if (strToi (buf) != NNTP_HEAD_FOLLOWS_VAL)
                                continue;
                        emit(buf);
                        CurrentScfg = cf;
                        while ((len=Cget (buf, sizeof buf)) && !EL (buf))
                        {
                                if (FILT)
                                {
                                        char *p;
                                        char *blocker;
                                        for (p=buf; *p && *p!=' ' && *p!='\t'; p++) ;
                                        if (*p && *++p && (blocker = filter_header(header, p)))
                                        {
                                                logd (("content filter %s blocked xhdr of <%s>", blocker, msgid));
                                                CS->filter_blocked++;
                                                continue;
                                        }
                                }
                                emit (buf);
                        }
                        CurrentScfg->share->xhdr_good++;
                        emitrn(".");
                        break;
                }
                if (!osrvrs)    /* no servers had <msgid> */
                        emitrn (NNTP_DONTHAVEIT);
                return TRUE;
        }
        CS->by_artnum++;
        if (CurrentScfg->xover_timeout < 1 || !isFieldInXover(field) || !CurrentScfg->share->overview_fmt)
        {
                Cemit (args);
                Cflush ();
                if (!Cget (buf, sizeof buf))
                {
                        CurrentScfg->share->xhdr_fail++;
                        emitrn (NNTP_SERVERDOWN);
                        return FALSE;
                }
                emit (buf);
                if (strToi(buf) != NNTP_HEAD_FOLLOWS_VAL)
                {
                        CurrentScfg->share->xhdr_fail++;
                        return FALSE;
                }
                while ((len=Cget (buf, sizeof buf)) && !EL (buf))
                {
                        if (FILT)
                        {
                                char *p;
                                char *blocker;
                                for (p=buf; *p && *p!=' ' && *p!='\t'; p++) ;
                                if (*p && *++p && (blocker = filter_header(header, p)))
                                {
                                        logd (("content filter %s blocked xhdr of %s/%d", blocker, CurrentDir, strToi(buf)));
                                        CS->filter_blocked++;
                                        continue;
                                }
                        }
                        emit (buf);
                }
                CurrentScfg->share->xhdr_good++;
                emitrn(".");
                return TRUE;
        }
        /* cached and not <msgid>. convert to xovers */
        if (min == 0 && max == 0)
        {
                min = max = CurrentGroupArtNum;
        } else
        {
                int nmax = getHi(CurrentGroupNode);
                int nmin = getLo(CurrentGroupNode);
                if (min<nmin)
                        min = nmin;
                if (max<1 || max >nmax)
                        max = strchr (args, '-')? nmax: min;
        }
        xover_process (min, max, field);
        return TRUE;
}

/* [<][>][^][v][top][bottom][index][help] */