GCOV - Varnish Cache


/*-
 * Copyright (c) 2006 Verdens Gang AS
 * Copyright (c) 2006-2015 Varnish Software AS
 * All rights reserved.
 *
 * Author: Poul-Henning Kamp <phk@phk.freebsd.dk>
 *
 * SPDX-License-Identifier: BSD-2-Clause
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */

#include "config.h"

#include "cache_varnishd.h"

#include <stdio.h>
#include <stdlib.h>

#include "vgz.h"
#include "vsl_priv.h"
#include "vmb.h"

#include "common/heritage.h"
#include "common/vsmw.h"

/* ------------------------------------------------------------
 * strands helpers - move elsewhere?
 */

static unsigned
strands_len(const struct strands *s)
{
        unsigned r = 0;
        int i;

        CHECK_OBJ_NOTNULL(s, STRANDS_MAGIC);
        for (i = 0; i < s->n; i++) {
                if (s->p[i] == NULL || *s->p[i] == '\0')
                        continue;
                r += strlen(s->p[i]);
        }

        return (r);
}

/*
 * like VRT_Strands(), but truncating instead of failing for end of buffer
 *
 * returns number of bytes including NUL
 */
static unsigned
strands_cat(char *buf, unsigned bufl, const struct strands *s)
{
        unsigned l = 0, ll;
        int i;

        /* NUL-terminated */
        assert(bufl > 0);
        bufl--;
        CHECK_OBJ_NOTNULL(s, STRANDS_MAGIC);

        for (i = 0; i < s->n && bufl > 0; i++) {
                if (s->p[i] == NULL || *s->p[i] == '\0')
                        continue;
                ll = vmin_t(unsigned, strlen(s->p[i]), bufl);
                memcpy(buf, s->p[i], ll);
                l += ll;
                buf += ll;
                bufl -= ll;
        }
        *buf = '\0';    /* NUL-terminated */
        return (l + 1);
}

/* These cannot be struct lock, which depends on vsm/vsl working */
static pthread_mutex_t vsl_mtx;
static pthread_mutex_t vsc_mtx;
static pthread_mutex_t vsm_mtx;

static struct VSL_head          *vsl_head;
static const uint32_t           *vsl_end;
static uint32_t                 *vsl_ptr;
static unsigned                 vsl_segment_n;
static ssize_t                  vsl_segsize;

struct VSC_main *VSC_C_main;

static void
vsl_sanity(const struct vsl_log *vsl)
{
        AN(vsl);
        AN(vsl->wlp);
        AN(vsl->wlb);
        AN(vsl->wle);
        assert(vsl->wlb <= vsl->wlp);
        assert(vsl->wlp <= vsl->wle);
}

/*--------------------------------------------------------------------
 * Check if the VSL_tag is masked by parameter bitmap
 */

static inline int
vsl_tag_is_masked(enum VSL_tag_e tag)
{
        volatile uint8_t *bm = &cache_param->vsl_mask[0];
        uint8_t b;

        assert(tag > SLT__Bogus);
        assert(tag < SLT__Reserved);
        bm += ((unsigned)tag >> 3);
        b = (0x80 >> ((unsigned)tag & 7));
        return (*bm & b);
}

int
VSL_tag_is_masked(enum VSL_tag_e tag)
{
        return (vsl_tag_is_masked(tag));
}

/*--------------------------------------------------------------------
 * Lay down a header fields, and return pointer to the next record
 */

static inline uint32_t *
vsl_hdr(enum VSL_tag_e tag, uint32_t *p, unsigned len, vxid_t vxid)
{

        AZ((uintptr_t)p & 0x3);
        assert(tag > SLT__Bogus);
        assert(tag < SLT__Reserved);
        AZ(len & ~VSL_LENMASK);

        p[2] = vxid.vxid >> 32;
        p[1] = vxid.vxid;
        p[0] = (((unsigned)tag & VSL_IDMASK) << VSL_IDSHIFT) |
             (VSL_VERSION_3 << VSL_VERSHIFT) |
             len;
        return (VSL_END(p, len));
}

/*--------------------------------------------------------------------
 * Space available in a VSL buffer when accounting for overhead
 */

static unsigned
vsl_space(const struct vsl_log *vsl)
{
        ptrdiff_t mlen;

        mlen = vsl->wle - vsl->wlp;
        assert(mlen >= 0);
        if (mlen < VSL_OVERHEAD + 1)
                return (0);
        mlen -= VSL_OVERHEAD;
        mlen *= sizeof *vsl->wlp;
        if (mlen > cache_param->vsl_reclen)
                mlen = cache_param->vsl_reclen;
        return(mlen);
}

/*--------------------------------------------------------------------
 * Wrap the VSL buffer
 */

static void
vsl_wrap(void)
{

        assert(vsl_ptr >= vsl_head->log);
        assert(vsl_ptr < vsl_end);
        vsl_segment_n += VSL_SEGMENTS - (vsl_segment_n % VSL_SEGMENTS);
        assert(vsl_segment_n % VSL_SEGMENTS == 0);
        vsl_head->offset[0] = 0;
        vsl_head->log[0] = VSL_ENDMARKER;
        VWMB();
        if (vsl_ptr != vsl_head->log) {
                *vsl_ptr = VSL_WRAPMARKER;
                vsl_ptr = vsl_head->log;
        }
        vsl_head->segment_n = vsl_segment_n;
        VSC_C_main->shm_cycles++;
}

/*--------------------------------------------------------------------
 * Reserve bytes for a record, wrap if necessary
 */

static uint32_t *
vsl_get(unsigned len, unsigned records, unsigned flushes)
{
        uint32_t *p;
        int err;

        err = pthread_mutex_trylock(&vsl_mtx);
        if (err == EBUSY) {
                PTOK(pthread_mutex_lock(&vsl_mtx));
                VSC_C_main->shm_cont++;
        } else {
                AZ(err);
        }
        assert(vsl_ptr < vsl_end);
        AZ((uintptr_t)vsl_ptr & 0x3);

        VSC_C_main->shm_writes++;
        VSC_C_main->shm_flushes += flushes;
        VSC_C_main->shm_records += records;
        VSC_C_main->shm_bytes +=
            VSL_BYTES(VSL_OVERHEAD + VSL_WORDS((uint64_t)len));

        /* Wrap if necessary */
        if (VSL_END(vsl_ptr, len) >= vsl_end)
                vsl_wrap();

        p = vsl_ptr;
        vsl_ptr = VSL_END(vsl_ptr, len);
        assert(vsl_ptr < vsl_end);
        AZ((uintptr_t)vsl_ptr & 0x3);

        *vsl_ptr = VSL_ENDMARKER;

        while ((vsl_ptr - vsl_head->log) / vsl_segsize >
            vsl_segment_n % VSL_SEGMENTS) {
                vsl_segment_n++;
                vsl_head->offset[vsl_segment_n % VSL_SEGMENTS] =
                    vsl_ptr - vsl_head->log;
        }

        PTOK(pthread_mutex_unlock(&vsl_mtx));
        /* Implicit VWMB() in mutex op ensures ENDMARKER and new table
           values are seen before new segment number */
        vsl_head->segment_n = vsl_segment_n;

        return (p);
}

/*--------------------------------------------------------------------
 * Stick a finished record into VSL.
 */

static void
vslr(enum VSL_tag_e tag, vxid_t vxid, const char *b, unsigned len)
{
        uint32_t *p;
        unsigned mlen;

        mlen = cache_param->vsl_reclen;

        /* Truncate */
        if (len > mlen)
                len = mlen;

        p = vsl_get(len, 1, 0);

        memcpy(p + VSL_OVERHEAD, b, len);

        /*
         * the vxid needs to be written before the barrier to
         * ensure it is valid when vsl_hdr() marks the record
         * ready by writing p[0]
         */
        p[2] = vxid.vxid >> 32;
        p[1] = vxid.vxid;
        VWMB();
        (void)vsl_hdr(tag, p, len, vxid);
}

/*--------------------------------------------------------------------
 * Add a unbuffered record to VSL
 *
 * NB: This variant should be used sparingly and only for low volume
 * NB: since it significantly adds to the mutex load on the VSL.
 */

void
VSLv(enum VSL_tag_e tag, vxid_t vxid, const char *fmt, va_list ap)
{
        unsigned n, mlen = cache_param->vsl_reclen;
        char buf[mlen];

        AN(fmt);
        if (vsl_tag_is_masked(tag))
                return;

        if (strchr(fmt, '%') == NULL) {
                vslr(tag, vxid, fmt, strlen(fmt) + 1);
        } else {
                n = vsnprintf(buf, mlen, fmt, ap);
                n = vmin(n, mlen - 1);
                buf[n++] = '\0'; /* NUL-terminated */
                vslr(tag, vxid, buf, n);
        }

}

void
VSLs(enum VSL_tag_e tag, vxid_t vxid, const struct strands *s)
{
        unsigned n, mlen = cache_param->vsl_reclen;
        char buf[mlen];

        if (vsl_tag_is_masked(tag))
                return;

        n = strands_cat(buf, mlen, s);

        vslr(tag, vxid, buf, n);
}

void
VSL(enum VSL_tag_e tag, vxid_t vxid, const char *fmt, ...)
{
        va_list ap;

        va_start(ap, fmt);
        VSLv(tag, vxid, fmt, ap);
        va_end(ap);
}

/*--------------------------------------------------------------------*/

void
VSL_Flush(struct vsl_log *vsl, int overflow)
{
        uint32_t *p;
        unsigned l;

        vsl_sanity(vsl);
        l = pdiff(vsl->wlb, vsl->wlp);
        if (l == 0)
                return;

        assert(l >= 8);

        p = vsl_get(l, vsl->wlr, overflow);

        memcpy(p + VSL_OVERHEAD, vsl->wlb, l);
        p[1] = l;
        VWMB();
        p[0] = ((((unsigned)SLT__Batch & 0xff) << VSL_IDSHIFT));
        vsl->wlp = vsl->wlb;
        vsl->wlr = 0;
}

/*--------------------------------------------------------------------
 * Buffered VSLs
 */

static char *
vslb_get(struct vsl_log *vsl, enum VSL_tag_e tag, unsigned *length)
{
        unsigned mlen = cache_param->vsl_reclen;
        char *retval;

        vsl_sanity(vsl);
        if (*length < mlen)
                mlen = *length;

        if (VSL_END(vsl->wlp, mlen) > vsl->wle)
                VSL_Flush(vsl, 1);

        retval = VSL_DATA(vsl->wlp);

        /* If it still doesn't fit, truncate */
        if (VSL_END(vsl->wlp, mlen) > vsl->wle)
                mlen = vsl_space(vsl);

        vsl->wlp = vsl_hdr(tag, vsl->wlp, mlen, vsl->wid);
        vsl->wlr++;
        *length = mlen;
        return (retval);
}

static void
vslb_simple(struct vsl_log *vsl, enum VSL_tag_e tag,
    unsigned length, const char *str)
{
        char *p;

        if (length == 0)
                length = strlen(str);
        length += 1; // NUL
        p = vslb_get(vsl, tag, &length);
        memcpy(p, str, length - 1);
        p[length - 1] = '\0';

        if (DO_DEBUG(DBG_SYNCVSL))
                VSL_Flush(vsl, 0);
}

/*--------------------------------------------------------------------
 * VSL-buffered-txt
 */

void
VSLbt(struct vsl_log *vsl, enum VSL_tag_e tag, txt t)
{

        Tcheck(t);
        if (vsl_tag_is_masked(tag))
                return;

        vslb_simple(vsl, tag, Tlen(t), t.b);
}

/*--------------------------------------------------------------------
 * VSL-buffered-strands
 */
void
VSLbs(struct vsl_log *vsl, enum VSL_tag_e tag, const struct strands *s)
{
        unsigned l;
        char *p;

        if (vsl_tag_is_masked(tag))
                return;

        l = strands_len(s) + 1;
        p = vslb_get(vsl, tag, &l);

        (void)strands_cat(p, l, s);

        if (DO_DEBUG(DBG_SYNCVSL))
                VSL_Flush(vsl, 0);
}

/*--------------------------------------------------------------------
 * VSL-buffered
 */

void
VSLbv(struct vsl_log *vsl, enum VSL_tag_e tag, const char *fmt, va_list ap)
{
        char *p, *p1;
        unsigned n = 0, mlen;
        va_list ap2;

        AN(fmt);
        if (vsl_tag_is_masked(tag))
                return;

        /*
         * If there are no printf-expansions, don't waste time expanding them
         */
        if (strchr(fmt, '%') == NULL) {
                vslb_simple(vsl, tag, 0, fmt);
                return;
        }

        /*
         * If the format is trivial, deal with it directly
         */
        if (!strcmp(fmt, "%s")) {
                p1 = va_arg(ap, char *);
                vslb_simple(vsl, tag, 0, p1);
                return;
        }

        vsl_sanity(vsl);

        mlen = vsl_space(vsl);

        // First attempt, only if any space at all
        if (mlen > 0) {
                p = VSL_DATA(vsl->wlp);
                va_copy(ap2, ap);
                n = vsnprintf(p, mlen, fmt, ap2);
                va_end(ap2);
        }

        // Second attempt, if a flush might help
        if (mlen == 0 || (n + 1 > mlen && n + 1 <= cache_param->vsl_reclen)) {
                VSL_Flush(vsl, 1);
                mlen = vsl_space(vsl);
                p = VSL_DATA(vsl->wlp);
                n = vsnprintf(p, mlen, fmt, ap);
        }
        if (n + 1 < mlen)
                mlen = n + 1;
        (void)vslb_get(vsl, tag, &mlen);

        if (DO_DEBUG(DBG_SYNCVSL))
                VSL_Flush(vsl, 0);
}

void
VSLb(struct vsl_log *vsl, enum VSL_tag_e tag, const char *fmt, ...)
{
        va_list ap;

        vsl_sanity(vsl);
        va_start(ap, fmt);
        VSLbv(vsl, tag, fmt, ap);
        va_end(ap);
}

#define Tf6 "%ju.%06ju"
#define Ta6(t) (uintmax_t)floor((t)), (uintmax_t)floor((t) * 1e6) % 1000000U

void
VSLb_ts(struct vsl_log *vsl, const char *event, vtim_real first,
    vtim_real *pprev, vtim_real now)
{

        /*
         * XXX: Make an option to turn off some unnecessary timestamp
         * logging. This must be done carefully because some functions
         * (e.g. V1L_Open) takes the last timestamp as its initial
         * value for timeout calculation.
         */
        vsl_sanity(vsl);
        AN(event);
        AN(pprev);
        assert(!isnan(now) && now != 0.);
        VSLb(vsl, SLT_Timestamp, "%s: " Tf6 " " Tf6 " " Tf6,
            event, Ta6(now), Ta6(now - first), Ta6(now - *pprev));
        *pprev = now;
}

void
VSLb_bin(struct vsl_log *vsl, enum VSL_tag_e tag, ssize_t len, const void *ptr)
{
        unsigned mlen;
        char *p;

        vsl_sanity(vsl);
        AN(ptr);
        if (vsl_tag_is_masked(tag))
                return;
        mlen = cache_param->vsl_reclen;

        /* Truncate */
        len = vmin_t(ssize_t, len, mlen);

        assert(vsl->wlp <= vsl->wle);

        /* Flush if necessary */
        if (VSL_END(vsl->wlp, len) > vsl->wle)
                VSL_Flush(vsl, 1);
        assert(VSL_END(vsl->wlp, len) <= vsl->wle);
        p = VSL_DATA(vsl->wlp);
        memcpy(p, ptr, len);
        vsl->wlp = vsl_hdr(tag, vsl->wlp, len, vsl->wid);
        assert(vsl->wlp <= vsl->wle);
        vsl->wlr++;

        if (DO_DEBUG(DBG_SYNCVSL))
                VSL_Flush(vsl, 0);
}

/*--------------------------------------------------------------------
 * Setup a VSL buffer, allocate space if none provided.
 */

void
VSL_Setup(struct vsl_log *vsl, void *ptr, size_t len)
{

        if (ptr == NULL) {
                len = cache_param->vsl_buffer;
                ptr = malloc(len);
                AN(ptr);
        }
        vsl->wlp = ptr;
        vsl->wlb = ptr;
        vsl->wle = ptr;
        vsl->wle += len / sizeof(*vsl->wle);
        vsl->wlr = 0;
        vsl->wid = NO_VXID;
        vsl_sanity(vsl);
}

/*--------------------------------------------------------------------*/

void
VSL_ChgId(struct vsl_log *vsl, const char *typ, const char *why, vxid_t vxid)
{
        vxid_t ovxid;

        vsl_sanity(vsl);
        ovxid = vsl->wid;
        VSLb(vsl, SLT_Link, "%s %ju %s", typ, VXID(vxid), why);
        VSL_End(vsl);
        vsl->wid = vxid;
        VSLb(vsl, SLT_Begin, "%s %ju %s", typ, VXID(ovxid), why);
}

/*--------------------------------------------------------------------*/

void
VSL_End(struct vsl_log *vsl)
{
        txt t;
        char p[] = "";

        vsl_sanity(vsl);
        assert(!IS_NO_VXID(vsl->wid));
        t.b = p;
        t.e = p;
        VSLbt(vsl, SLT_End, t);
        VSL_Flush(vsl, 0);
        vsl->wid = NO_VXID;
}

static void v_matchproto_(vsm_lock_f)
vsm_vsc_lock(void)
{
        PTOK(pthread_mutex_lock(&vsc_mtx));
}

static void v_matchproto_(vsm_lock_f)
vsm_vsc_unlock(void)
{
        PTOK(pthread_mutex_unlock(&vsc_mtx));
}

static void v_matchproto_(vsm_lock_f)
vsm_vsmw_lock(void)
{
        PTOK(pthread_mutex_lock(&vsm_mtx));
}

static void v_matchproto_(vsm_lock_f)
vsm_vsmw_unlock(void)
{
        PTOK(pthread_mutex_unlock(&vsm_mtx));
}

/*--------------------------------------------------------------------*/

void
VSM_Init(void)
{
        unsigned u;

        assert(UINT_MAX % VSL_SEGMENTS == VSL_SEGMENTS - 1);

        PTOK(pthread_mutex_init(&vsl_mtx, &mtxattr_errorcheck));
        PTOK(pthread_mutex_init(&vsc_mtx, &mtxattr_errorcheck));
        PTOK(pthread_mutex_init(&vsm_mtx, &mtxattr_errorcheck));

        vsc_lock = vsm_vsc_lock;
        vsc_unlock = vsm_vsc_unlock;
        vsmw_lock = vsm_vsmw_lock;
        vsmw_unlock = vsm_vsmw_unlock;

        heritage.proc_vsmw = VSMW_New(heritage.vsm_fd, 0640, "_.index");
        AN(heritage.proc_vsmw);

        VSC_C_main = VSC_main_New(NULL, NULL, "");
        AN(VSC_C_main);

        AN(heritage.proc_vsmw);
        vsl_head = VSMW_Allocf(heritage.proc_vsmw, NULL, VSL_CLASS,
            cache_param->vsl_space, VSL_CLASS);
        AN(vsl_head);
        vsl_segsize = ((cache_param->vsl_space - sizeof *vsl_head) /
            sizeof *vsl_end) / VSL_SEGMENTS;
        vsl_end = vsl_head->log + vsl_segsize * VSL_SEGMENTS;
        /* Make segment_n always overflow on first log wrap to make any
           problems with regard to readers on that event visible */
        vsl_segment_n = UINT_MAX - (VSL_SEGMENTS - 1);
        AZ(vsl_segment_n % VSL_SEGMENTS);
        vsl_ptr = vsl_head->log;
        *vsl_ptr = VSL_ENDMARKER;

        memset(vsl_head, 0, sizeof *vsl_head);
        vsl_head->segsize = vsl_segsize;
        vsl_head->offset[0] = 0;
        vsl_head->segment_n = vsl_segment_n;
        for (u = 1; u < VSL_SEGMENTS; u++)
                vsl_head->offset[u] = -1;
        VWMB();
        memcpy(vsl_head->marker, VSL_HEAD_MARKER, sizeof vsl_head->marker);
}

		varnish-cache/bin/varnishd/cache/cache_shmlog.c
0		/*-
1		* Copyright (c) 2006 Verdens Gang AS
2		* Copyright (c) 2006-2015 Varnish Software AS
3		* All rights reserved.
4		*
5		* Author: Poul-Henning Kamp <phk@phk.freebsd.dk>
6		*
7		* SPDX-License-Identifier: BSD-2-Clause
8		*
9		* Redistribution and use in source and binary forms, with or without
10		* modification, are permitted provided that the following conditions
11		* are met:
12		* 1. Redistributions of source code must retain the above copyright
13		* notice, this list of conditions and the following disclaimer.
14		* 2. Redistributions in binary form must reproduce the above copyright
15		* notice, this list of conditions and the following disclaimer in the
16		* documentation and/or other materials provided with the distribution.
17		*
18		* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19		* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20		* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21		* ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
22		* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23		* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24		* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25		* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26		* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27		* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28		* SUCH DAMAGE.
29		*/
30
31		#include "config.h"
32
33		#include "cache_varnishd.h"
34
35		#include <stdio.h>
36		#include <stdlib.h>
37
38		#include "vgz.h"
39		#include "vsl_priv.h"
40		#include "vmb.h"
41
42		#include "common/heritage.h"
43		#include "common/vsmw.h"
44
45		/* ------------------------------------------------------------
46		* strands helpers - move elsewhere?
47		*/
48
49		static unsigned
50	1977705	strands_len(const struct strands *s)
51		{
52	1977705	unsigned r = 0;
53		int i;
54
55	1977705	CHECK_OBJ_NOTNULL(s, STRANDS_MAGIC);
56	3957872	for (i = 0; i < s->n; i++) {
57	1980167	if (s->p[i] == NULL \|\| *s->p[i] == '\0')
58	188275	continue;
59	1791892	r += strlen(s->p[i]);
60	1791892	}
61
62	1977705	return (r);
63		}
64
65		/*
66		* like VRT_Strands(), but truncating instead of failing for end of buffer
67		*
68		* returns number of bytes including NUL
69		*/
70		static unsigned
71	1985488	strands_cat(char buf, unsigned bufl, const struct strands s)
72		{
73	1985488	unsigned l = 0, ll;
74		int i;
75
76		/* NUL-terminated */
77	1985488	assert(bufl > 0);
78	1985488	bufl--;
79	1985488	CHECK_OBJ_NOTNULL(s, STRANDS_MAGIC);
80
81	3789068	for (i = 0; i < s->n && bufl > 0; i++) {
82	1803584	if (s->p[i] == NULL \|\| *s->p[i] == '\0')
83	88	continue;
84	1803500	ll = vmin_t(unsigned, strlen(s->p[i]), bufl);
85	1803500	memcpy(buf, s->p[i], ll);
86	1803500	l += ll;
87	1803500	buf += ll;
88	1803500	bufl -= ll;
89	1803500	}
90	1985484	buf = '\0'; / NUL-terminated */
91	1985484	return (l + 1);
92		}
93
94		/* These cannot be struct lock, which depends on vsm/vsl working */
95		static pthread_mutex_t vsl_mtx;
96		static pthread_mutex_t vsc_mtx;
97		static pthread_mutex_t vsm_mtx;
98
99		static struct VSL_head *vsl_head;
100		static const uint32_t *vsl_end;
101		static uint32_t *vsl_ptr;
102		static unsigned vsl_segment_n;
103		static ssize_t vsl_segsize;
104
105		struct VSC_main *VSC_C_main;
106
107		static void
108	19645050	vsl_sanity(const struct vsl_log *vsl)
109		{
110	19645050	AN(vsl);
111	19645050	AN(vsl->wlp);
112	19645050	AN(vsl->wlb);
113	19645050	AN(vsl->wle);
114	19645050	assert(vsl->wlb <= vsl->wlp);
115	19645050	assert(vsl->wlp <= vsl->wle);
116	19645050	}
117
118		/*--------------------------------------------------------------------
119		* Check if the VSL_tag is masked by parameter bitmap
120		*/
121
122		static inline int
123	14604232	vsl_tag_is_masked(enum VSL_tag_e tag)
124		{
125	14604232	volatile uint8_t *bm = &cache_param->vsl_mask[0];
126		uint8_t b;
127
128	14604232	assert(tag > SLT__Bogus);
129	14604232	assert(tag < SLT__Reserved);
130	14604232	bm += ((unsigned)tag >> 3);
131	14604232	b = (0x80 >> ((unsigned)tag & 7));
132	14604232	return (*bm & b);
133		}
134
135		int
136	51051	VSL_tag_is_masked(enum VSL_tag_e tag)
137		{
138	51051	return (vsl_tag_is_masked(tag));
139		}
140
141		/*--------------------------------------------------------------------
142		* Lay down a header fields, and return pointer to the next record
143		*/
144
145		static inline uint32_t *
146	12037166	vsl_hdr(enum VSL_tag_e tag, uint32_t *p, unsigned len, vxid_t vxid)
147		{
148
149	12037166	AZ((uintptr_t)p & 0x3);
150	12037166	assert(tag > SLT__Bogus);
151	12037166	assert(tag < SLT__Reserved);
152	12037166	AZ(len & ~VSL_LENMASK);
153
154	12037166	p[2] = vxid.vxid >> 32;
155	12037166	p[1] = vxid.vxid;
156	24074332	p[0] = (((unsigned)tag & VSL_IDMASK) << VSL_IDSHIFT) \|
157	12037166	(VSL_VERSION_3 << VSL_VERSHIFT) \|
158	12037166	len;
159	12037166	return (VSL_END(p, len));
160		}
161
162		/*--------------------------------------------------------------------
163		* Space available in a VSL buffer when accounting for overhead
164		*/
165
166		static unsigned
167	2925730	vsl_space(const struct vsl_log *vsl)
168		{
169		ptrdiff_t mlen;
170
171	2925730	mlen = vsl->wle - vsl->wlp;
172	2925730	assert(mlen >= 0);
173	2925730	if (mlen < VSL_OVERHEAD + 1)
174	80	return (0);
175	2925650	mlen -= VSL_OVERHEAD;
176	2925650	mlen = sizeof vsl->wlp;
177	2925650	if (mlen > cache_param->vsl_reclen)
178	2924930	mlen = cache_param->vsl_reclen;
179	2925650	return(mlen);
180	2925730	}
181
182		/*--------------------------------------------------------------------
183		* Wrap the VSL buffer
184		*/
185
186		static void
187	0	vsl_wrap(void)
188		{
189
190	0	assert(vsl_ptr >= vsl_head->log);
191	0	assert(vsl_ptr < vsl_end);
192	0	vsl_segment_n += VSL_SEGMENTS - (vsl_segment_n % VSL_SEGMENTS);
193	0	assert(vsl_segment_n % VSL_SEGMENTS == 0);
194	0	vsl_head->offset[0] = 0;
195	0	vsl_head->log[0] = VSL_ENDMARKER;
196	0	VWMB();
197	0	if (vsl_ptr != vsl_head->log) {
198	0	*vsl_ptr = VSL_WRAPMARKER;
199	0	vsl_ptr = vsl_head->log;
200	0	}
201	0	vsl_head->segment_n = vsl_segment_n;
202	0	VSC_C_main->shm_cycles++;
203	0	}
204
205		/*--------------------------------------------------------------------
206		* Reserve bytes for a record, wrap if necessary
207		*/
208
209		static uint32_t *
210	3573178	vsl_get(unsigned len, unsigned records, unsigned flushes)
211		{
212		uint32_t *p;
213		int err;
214
215	3573178	err = pthread_mutex_trylock(&vsl_mtx);
216	3573178	if (err == EBUSY) {
217	23195	PTOK(pthread_mutex_lock(&vsl_mtx));
218	23195	VSC_C_main->shm_cont++;
219	23195	} else {
220	3549983	AZ(err);
221		}
222	3573178	assert(vsl_ptr < vsl_end);
223	3573178	AZ((uintptr_t)vsl_ptr & 0x3);
224
225	3573178	VSC_C_main->shm_writes++;
226	3573178	VSC_C_main->shm_flushes += flushes;
227	3573178	VSC_C_main->shm_records += records;
228	3573178	VSC_C_main->shm_bytes +=
229	3573178	VSL_BYTES(VSL_OVERHEAD + VSL_WORDS((uint64_t)len));
230
231		/* Wrap if necessary */
232	3573178	if (VSL_END(vsl_ptr, len) >= vsl_end)
233	0	vsl_wrap();
234
235	3573178	p = vsl_ptr;
236	3573178	vsl_ptr = VSL_END(vsl_ptr, len);
237	3573178	assert(vsl_ptr < vsl_end);
238	3573178	AZ((uintptr_t)vsl_ptr & 0x3);
239
240	3573178	*vsl_ptr = VSL_ENDMARKER;
241
242	3573258	while ((vsl_ptr - vsl_head->log) / vsl_segsize >
243	3573258	vsl_segment_n % VSL_SEGMENTS) {
244	80	vsl_segment_n++;
245	80	vsl_head->offset[vsl_segment_n % VSL_SEGMENTS] =
246	80	vsl_ptr - vsl_head->log;
247		}
248
249	3573178	PTOK(pthread_mutex_unlock(&vsl_mtx));
250		/* Implicit VWMB() in mutex op ensures ENDMARKER and new table
251		values are seen before new segment number */
252	3573178	vsl_head->segment_n = vsl_segment_n;
253
254	3573178	return (p);
255		}
256
257		/*--------------------------------------------------------------------
258		* Stick a finished record into VSL.
259		*/
260
261		static void
262	2320625	vslr(enum VSL_tag_e tag, vxid_t vxid, const char *b, unsigned len)
263		{
264		uint32_t *p;
265		unsigned mlen;
266
267	2320625	mlen = cache_param->vsl_reclen;
268
269		/* Truncate */
270	2320625	if (len > mlen)
271	0	len = mlen;
272
273	2320625	p = vsl_get(len, 1, 0);
274
275	2320625	memcpy(p + VSL_OVERHEAD, b, len);
276
277		/*
278		* the vxid needs to be written before the barrier to
279		* ensure it is valid when vsl_hdr() marks the record
280		* ready by writing p[0]
281		*/
282	2320625	p[2] = vxid.vxid >> 32;
283	2320625	p[1] = vxid.vxid;
284	2320625	VWMB();
285	2320625	(void)vsl_hdr(tag, p, len, vxid);
286	2320625	}
287
288		/*--------------------------------------------------------------------
289		* Add a unbuffered record to VSL
290		*
291		* NB: This variant should be used sparingly and only for low volume
292		* NB: since it significantly adds to the mutex load on the VSL.
293		*/
294
295		void
296	3116979	VSLv(enum VSL_tag_e tag, vxid_t vxid, const char *fmt, va_list ap)
297		{
298	3116979	unsigned n, mlen = cache_param->vsl_reclen;
299	3116979	char buf[mlen];
300
301	3116979	AN(fmt);
302	3116979	if (vsl_tag_is_masked(tag))
303	804317	return;
304
305	2312662	if (strchr(fmt, '%') == NULL) {
306	45823	vslr(tag, vxid, fmt, strlen(fmt) + 1);
307	45823	} else {
308	2266839	n = vsnprintf(buf, mlen, fmt, ap);
309	2266839	n = vmin(n, mlen - 1);
310	2266839	buf[n++] = '\0'; /* NUL-terminated */
311	2266839	vslr(tag, vxid, buf, n);
312		}
313
314	3116979	}
315
316		void
317	7960	VSLs(enum VSL_tag_e tag, vxid_t vxid, const struct strands *s)
318		{
319	7960	unsigned n, mlen = cache_param->vsl_reclen;
320	7960	char buf[mlen];
321
322	7960	if (vsl_tag_is_masked(tag))
323	0	return;
324
325	7960	n = strands_cat(buf, mlen, s);
326
327	7960	vslr(tag, vxid, buf, n);
328	7960	}
329
330		void
331	3051179	VSL(enum VSL_tag_e tag, vxid_t vxid, const char *fmt, ...)
332		{
333		va_list ap;
334
335	3051179	va_start(ap, fmt);
336	3051179	VSLv(tag, vxid, fmt, ap);
337	3051179	va_end(ap);
338	3051179	}
339
340		/--------------------------------------------------------------------/
341
342		void
343	1575682	VSL_Flush(struct vsl_log *vsl, int overflow)
344		{
345		uint32_t *p;
346		unsigned l;
347
348	1575682	vsl_sanity(vsl);
349	1575682	l = pdiff(vsl->wlb, vsl->wlp);
350	1575682	if (l == 0)
351	327219	return;
352
353	1248463	assert(l >= 8);
354
355	1248463	p = vsl_get(l, vsl->wlr, overflow);
356
357	1248463	memcpy(p + VSL_OVERHEAD, vsl->wlb, l);
358	1248463	p[1] = l;
359	1248463	VWMB();
360	1248463	p[0] = ((((unsigned)SLT__Batch & 0xff) << VSL_IDSHIFT));
361	1248463	vsl->wlp = vsl->wlb;
362	1248463	vsl->wlr = 0;
363	1575682	}
364
365		/*--------------------------------------------------------------------
366		* Buffered VSLs
367		*/
368
369		static char *
370	9660739	vslb_get(struct vsl_log vsl, enum VSL_tag_e tag, unsigned length)
371		{
372	9660739	unsigned mlen = cache_param->vsl_reclen;
373		char *retval;
374
375	9660739	vsl_sanity(vsl);
376	9660739	if (*length < mlen)
377	9638404	mlen = *length;
378
379	9660739	if (VSL_END(vsl->wlp, mlen) > vsl->wle)
380	280	VSL_Flush(vsl, 1);
381
382	9660739	retval = VSL_DATA(vsl->wlp);
383
384		/* If it still doesn't fit, truncate */
385	9660739	if (VSL_END(vsl->wlp, mlen) > vsl->wle)
386	80	mlen = vsl_space(vsl);
387
388	9660739	vsl->wlp = vsl_hdr(tag, vsl->wlp, mlen, vsl->wid);
389	9660739	vsl->wlr++;
390	9660739	*length = mlen;
391	9660739	return (retval);
392		}
393
394		static void
395	4761380	vslb_simple(struct vsl_log *vsl, enum VSL_tag_e tag,
396		unsigned length, const char *str)
397		{
398		char *p;
399
400	4761380	if (length == 0)
401	372100	length = strlen(str);
402	4761380	length += 1; // NUL
403	4761380	p = vslb_get(vsl, tag, &length);
404	4761380	memcpy(p, str, length - 1);
405	4761380	p[length - 1] = '\0';
406
407	4761380	if (DO_DEBUG(DBG_SYNCVSL))
408	463681	VSL_Flush(vsl, 0);
409	4761380	}
410
411		/*--------------------------------------------------------------------
412		* VSL-buffered-txt
413		*/
414
415		void
416	5344132	VSLbt(struct vsl_log *vsl, enum VSL_tag_e tag, txt t)
417		{
418
419	5344132	Tcheck(t);
420	5344132	if (vsl_tag_is_masked(tag))
421	705156	return;
422
423	4638976	vslb_simple(vsl, tag, Tlen(t), t.b);
424	5344132	}
425
426		/*--------------------------------------------------------------------
427		* VSL-buffered-strands
428		*/
429		void
430	2340691	VSLbs(struct vsl_log vsl, enum VSL_tag_e tag, const struct strands s)
431		{
432		unsigned l;
433		char *p;
434
435	2340691	if (vsl_tag_is_masked(tag))
436	362873	return;
437
438	1977818	l = strands_len(s) + 1;
439	1977818	p = vslb_get(vsl, tag, &l);
440
441	1977818	(void)strands_cat(p, l, s);
442
443	1977818	if (DO_DEBUG(DBG_SYNCVSL))
444	184814	VSL_Flush(vsl, 0);
445	2340691	}
446
447		/*--------------------------------------------------------------------
448		* VSL-buffered
449		*/
450
451		void
452	3641118	VSLbv(struct vsl_log vsl, enum VSL_tag_e tag, const char fmt, va_list ap)
453		{
454		char p, p1;
455	3641118	unsigned n = 0, mlen;
456		va_list ap2;
457
458	3641118	AN(fmt);
459	3641118	if (vsl_tag_is_masked(tag))
460	592507	return;
461
462		/*
463		* If there are no printf-expansions, don't waste time expanding them
464		*/
465	3048611	if (strchr(fmt, '%') == NULL) {
466	121788	vslb_simple(vsl, tag, 0, fmt);
467	121788	return;
468		}
469
470		/*
471		* If the format is trivial, deal with it directly
472		*/
473	2926823	if (!strcmp(fmt, "%s")) {
474	720	p1 = va_arg(ap, char *);
475	720	vslb_simple(vsl, tag, 0, p1);
476	720	return;
477		}
478
479	2926103	vsl_sanity(vsl);
480
481	2926103	mlen = vsl_space(vsl);
482
483		// First attempt, only if any space at all
484	2926103	if (mlen > 0) {
485	2924622	p = VSL_DATA(vsl->wlp);
486	2924622	va_copy(ap2, ap);
487	2924622	n = vsnprintf(p, mlen, fmt, ap2);
488	2924622	va_end(ap2);
489	2924622	}
490
491		// Second attempt, if a flush might help
492	2926103	if (mlen == 0 \|\| (n + 1 > mlen && n + 1 <= cache_param->vsl_reclen)) {
493	2052	VSL_Flush(vsl, 1);
494	2052	mlen = vsl_space(vsl);
495	2052	p = VSL_DATA(vsl->wlp);
496	2052	n = vsnprintf(p, mlen, fmt, ap);
497	2052	}
498	2926583	if (n + 1 < mlen)
499	2915662	mlen = n + 1;