GCOV - Varnish Cache


/*-
 * Copyright (c) 2008-2011 Varnish Software AS
 * All rights reserved.
 *
 * Author: Poul-Henning Kamp <phk@phk.freebsd.dk>
 *
 * SPDX-License-Identifier: BSD-2-Clause
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 * Persistent storage method
 *
 * XXX: Before we start the client or maybe after it stops, we should give the
 * XXX: stevedores a chance to examine their storage for consistency.
 *
 * XXX: Do we ever free the LRU-lists ?
 */

#include "config.h"

#include "cache/cache_varnishd.h"
#include "common/heritage.h"

#include <sys/mman.h>

#include <stdio.h>
#include <stdlib.h>

#include "storage/storage.h"
#include "storage/storage_simple.h"

#include "vsha256.h"

#include "storage/storage_persistent.h"

#ifdef HAVE_SYS_PERSONALITY_H
#include <sys/personality.h>
#endif

#ifndef MAP_NOCORE
#ifdef MAP_CONCEAL
#define MAP_NOCORE MAP_CONCEAL /* XXX OpenBSD */
#else
#define MAP_NOCORE 0 /* XXX Linux */
#endif
#endif

#ifndef MAP_NOSYNC
#define MAP_NOSYNC 0 /* XXX Linux */
#endif

/*--------------------------------------------------------------------
 * Calculate cleaner metrics from silo dimensions
 */

static void
smp_metrics(struct smp_sc *sc)
{

        /*
         * We do not want to loose too big chunks of the silos
         * content when we are forced to clean a segment.
         *
         * For now insist that a segment covers no more than 1% of the silo.
         *
         * XXX: This should possibly depend on the size of the silo so
         * XXX: trivially small silos do not run into trouble along
         * XXX: the lines of "one object per segment".
         */

        sc->min_nseg = 10;
        sc->max_segl = smp_stuff_len(sc, SMP_SPC_STUFF) / sc->min_nseg;

        fprintf(stderr, "min_nseg = %u, max_segl = %ju\n",
            sc->min_nseg, (uintmax_t)sc->max_segl);

        /*
         * The number of segments are limited by the size of the segment
         * table(s) and from that follows the minimum size of a segment.
         */

        sc->max_nseg = smp_stuff_len(sc, SMP_SEG1_STUFF) / sc->min_nseg;
        sc->min_segl = smp_stuff_len(sc, SMP_SPC_STUFF) / sc->max_nseg;

        while (sc->min_segl < sizeof(struct object)) {
                sc->max_nseg /= 2;
                sc->min_segl = smp_stuff_len(sc, SMP_SPC_STUFF) / sc->max_nseg;
        }

        fprintf(stderr, "max_nseg = %u, min_segl = %ju\n",
            sc->max_nseg, (uintmax_t)sc->min_segl);

        /*
         * Set our initial aim point at the exponential average of the
         * two extremes.
         *
         * XXX: This is a pretty arbitrary choice, but having no idea
         * XXX: object count, size distribution or ttl pattern at this
         * XXX: point, we have to do something.
         */

        sc->aim_nseg =
           (unsigned) exp((log(sc->min_nseg) + log(sc->max_nseg))*.5);
        sc->aim_segl = smp_stuff_len(sc, SMP_SPC_STUFF) / sc->aim_nseg;

        fprintf(stderr, "aim_nseg = %u, aim_segl = %ju\n",
            sc->aim_nseg, (uintmax_t)sc->aim_segl);

        /*
         * How much space in the free reserve pool ?
         */
        sc->free_reserve = sc->aim_segl * 10;

        fprintf(stderr, "free_reserve = %ju\n", (uintmax_t)sc->free_reserve);
}

/*--------------------------------------------------------------------
 * Set up persistent storage silo in the master process.
 */

void v_matchproto_(storage_init_f)
smp_mgt_init(struct stevedore *parent, int ac, char * const *av)
{
        struct smp_sc           *sc;
        void *target;
        int i, mmap_flags;

        AZ(av[ac]);

#ifdef HAVE_SYS_PERSONALITY_H
        i = personality(0xffffffff); /* Fetch old personality. */
        if (!(i & ADDR_NO_RANDOMIZE)) {
                i = personality(i | ADDR_NO_RANDOMIZE);
                if (i < 0)
                        fprintf(stderr, "WARNING: Could not disable ASLR\n");
                else
                        fprintf(stderr, "NB: Disabled ASLR for Persistent\n");
        }
#endif

        /* Necessary alignment. See also smp_object::__filler__ */
        assert(sizeof(struct smp_object) % 8 == 0);

#define SIZOF(foo)       fprintf(stderr, \
    "sizeof(%s) = %zu = 0x%zx\n", #foo, sizeof(foo), sizeof(foo));
        SIZOF(struct smp_ident);
        SIZOF(struct smp_sign);
        SIZOF(struct smp_segptr);
        SIZOF(struct smp_object);
#undef SIZOF

        /* See comments in storage_persistent.h */
        assert(sizeof(struct smp_ident) == SMP_IDENT_SIZE);

        /* Allocate softc */
        ALLOC_OBJ(sc, SMP_SC_MAGIC);
        XXXAN(sc);
        sc->parent = parent;
        sc->fd = -1;
        VTAILQ_INIT(&sc->segments);

        /* Argument processing */
        if (ac != 2)
                ARGV_ERR("(-spersistent) wrong number of arguments\n");

        i = STV_GetFile(av[0], &sc->fd, &sc->filename, "-spersistent");
        if (i == 2)
                ARGV_ERR("(-spersistent) need filename (not directory)\n");

        sc->align = sizeof(void*) * 2;
        sc->granularity = getpagesize();
        sc->mediasize = STV_FileSize(sc->fd, av[1], &sc->granularity,
            "-spersistent");

        AZ(ftruncate(sc->fd, sc->mediasize));

        /* Try to determine correct mmap address */
        target = NULL;

        mmap_flags = MAP_NOCORE | MAP_NOSYNC | MAP_SHARED;

#ifdef MAP_ALIGNED_SUPER
        mmap_flags |= MAP_ALIGNED_SUPER;
#endif

        sc->base = (void*)mmap(target, sc->mediasize, PROT_READ|PROT_WRITE,
            mmap_flags, sc->fd, 0);

        if (sc->base == MAP_FAILED)
                ARGV_ERR("(-spersistent) failed to mmap (%s) @%p\n",
                    VAS_errtxt(errno), target);

        smp_def_sign(sc, &sc->idn, 0, "SILO");
        sc->ident = SIGN_DATA(&sc->idn);

        i = smp_valid_silo(sc);
        if (i) {
                printf("Warning SILO (%s) not reloaded (reason=%d)\n",
                    sc->filename, i);
                smp_newsilo(sc);
        }
        AZ(smp_valid_silo(sc));

        smp_metrics(sc);

        parent->priv = sc;

        /* XXX: only for sendfile I guess... */
        MCH_Fd_Inherit(sc->fd, "storage_persistent");
}

		varnish-cache/bin/varnishd/storage/mgt_storage_persistent.c
0		/*-
1		* Copyright (c) 2008-2011 Varnish Software AS
2		* All rights reserved.
3		*
4		* Author: Poul-Henning Kamp <phk@phk.freebsd.dk>
5		*
6		* SPDX-License-Identifier: BSD-2-Clause
7		*
8		* Redistribution and use in source and binary forms, with or without
9		* modification, are permitted provided that the following conditions
10		* are met:
11		* 1. Redistributions of source code must retain the above copyright
12		* notice, this list of conditions and the following disclaimer.
13		* 2. Redistributions in binary form must reproduce the above copyright
14		* notice, this list of conditions and the following disclaimer in the
15		* documentation and/or other materials provided with the distribution.
16		*
17		* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18		* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19		* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20		* ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
21		* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22		* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23		* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24		* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25		* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26		* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27		* SUCH DAMAGE.
28		*
29		* Persistent storage method
30		*
31		* XXX: Before we start the client or maybe after it stops, we should give the
32		* XXX: stevedores a chance to examine their storage for consistency.
33		*
34		* XXX: Do we ever free the LRU-lists ?
35		*/
36
37		#include "config.h"
38
39		#include "cache/cache_varnishd.h"
40		#include "common/heritage.h"
41
42		#include <sys/mman.h>
43
44		#include <stdio.h>
45		#include <stdlib.h>
46
47		#include "storage/storage.h"
48		#include "storage/storage_simple.h"
49
50		#include "vsha256.h"
51
52		#include "storage/storage_persistent.h"
53
54		#ifdef HAVE_SYS_PERSONALITY_H
55		#include <sys/personality.h>
56		#endif
57
58		#ifndef MAP_NOCORE
59		#ifdef MAP_CONCEAL
60		#define MAP_NOCORE MAP_CONCEAL /* XXX OpenBSD */
61		#else
62		#define MAP_NOCORE 0 /* XXX Linux */
63		#endif
64		#endif
65
66		#ifndef MAP_NOSYNC
67		#define MAP_NOSYNC 0 /* XXX Linux */
68		#endif
69
70		/*--------------------------------------------------------------------
71		* Calculate cleaner metrics from silo dimensions
72		*/
73
74		static void
75	950	smp_metrics(struct smp_sc *sc)
76		{
77
78		/*
79		* We do not want to loose too big chunks of the silos
80		* content when we are forced to clean a segment.
81		*
82		* For now insist that a segment covers no more than 1% of the silo.
83		*
84		* XXX: This should possibly depend on the size of the silo so
85		* XXX: trivially small silos do not run into trouble along
86		* XXX: the lines of "one object per segment".
87		*/
88
89	950	sc->min_nseg = 10;
90	950	sc->max_segl = smp_stuff_len(sc, SMP_SPC_STUFF) / sc->min_nseg;
91
92	1900	fprintf(stderr, "min_nseg = %u, max_segl = %ju\n",
93	950	sc->min_nseg, (uintmax_t)sc->max_segl);
94
95		/*
96		* The number of segments are limited by the size of the segment
97		* table(s) and from that follows the minimum size of a segment.
98		*/
99
100	950	sc->max_nseg = smp_stuff_len(sc, SMP_SEG1_STUFF) / sc->min_nseg;
101	950	sc->min_segl = smp_stuff_len(sc, SMP_SPC_STUFF) / sc->max_nseg;
102
103	4750	while (sc->min_segl < sizeof(struct object)) {
104	3800	sc->max_nseg /= 2;
105	3800	sc->min_segl = smp_stuff_len(sc, SMP_SPC_STUFF) / sc->max_nseg;
106		}
107
108	1900	fprintf(stderr, "max_nseg = %u, min_segl = %ju\n",
109	950	sc->max_nseg, (uintmax_t)sc->min_segl);
110
111		/*
112		* Set our initial aim point at the exponential average of the
113		* two extremes.
114		*
115		* XXX: This is a pretty arbitrary choice, but having no idea
116		* XXX: object count, size distribution or ttl pattern at this
117		* XXX: point, we have to do something.
118		*/
119
120	950	sc->aim_nseg =
121	950	(unsigned) exp((log(sc->min_nseg) + log(sc->max_nseg))*.5);
122	950	sc->aim_segl = smp_stuff_len(sc, SMP_SPC_STUFF) / sc->aim_nseg;
123
124	1900	fprintf(stderr, "aim_nseg = %u, aim_segl = %ju\n",
125	950	sc->aim_nseg, (uintmax_t)sc->aim_segl);
126
127		/*
128		* How much space in the free reserve pool ?
129		*/
130	950	sc->free_reserve = sc->aim_segl * 10;
131
132	950	fprintf(stderr, "free_reserve = %ju\n", (uintmax_t)sc->free_reserve);
133	950	}
134
135		/*--------------------------------------------------------------------
136		* Set up persistent storage silo in the master process.
137		*/
138
139		void v_matchproto_(storage_init_f)
140	950	smp_mgt_init(struct stevedore parent, int ac, char const *av)
141		{
142		struct smp_sc *sc;
143		void *target;
144		int i, mmap_flags;
145
146	950	AZ(av[ac]);
147
148		#ifdef HAVE_SYS_PERSONALITY_H
149		i = personality(0xffffffff); /* Fetch old personality. */
150		if (!(i & ADDR_NO_RANDOMIZE)) {
151		i = personality(i \| ADDR_NO_RANDOMIZE);
152		if (i < 0)
153		fprintf(stderr, "WARNING: Could not disable ASLR\n");
154		else
155		fprintf(stderr, "NB: Disabled ASLR for Persistent\n");
156		}
157		#endif
158
159		/* Necessary alignment. See also smp_object::__filler__ */
160	950	assert(sizeof(struct smp_object) % 8 == 0);
161
162		#define SIZOF(foo) fprintf(stderr, \
163		"sizeof(%s) = %zu = 0x%zx\n", #foo, sizeof(foo), sizeof(foo));
164	950	SIZOF(struct smp_ident);
165	950	SIZOF(struct smp_sign);
166	950	SIZOF(struct smp_segptr);
167	950	SIZOF(struct smp_object);
168		#undef SIZOF
169
170		/* See comments in storage_persistent.h */
171	950	assert(sizeof(struct smp_ident) == SMP_IDENT_SIZE);
172
173		/* Allocate softc */
174	950	ALLOC_OBJ(sc, SMP_SC_MAGIC);
175	950	XXXAN(sc);
176	950	sc->parent = parent;
177	950	sc->fd = -1;
178	950	VTAILQ_INIT(&sc->segments);
179
180		/* Argument processing */
181	950	if (ac != 2)
182	0	ARGV_ERR("(-spersistent) wrong number of arguments\n");
183
184	950	i = STV_GetFile(av[0], &sc->fd, &sc->filename, "-spersistent");
185	950	if (i == 2)
186	0	ARGV_ERR("(-spersistent) need filename (not directory)\n");
187
188	950	sc->align = sizeof(void) 2;
189	950	sc->granularity = getpagesize();
190	950	sc->mediasize = STV_FileSize(sc->fd, av[1], &sc->granularity,
191		"-spersistent");
192
193	950	AZ(ftruncate(sc->fd, sc->mediasize));
194
195		/* Try to determine correct mmap address */
196	950	target = NULL;
197
198	950	mmap_flags = MAP_NOCORE \| MAP_NOSYNC \| MAP_SHARED;
199
200		#ifdef MAP_ALIGNED_SUPER
201	950	mmap_flags \|= MAP_ALIGNED_SUPER;
202		#endif
203
204	1900	sc->base = (void*)mmap(target, sc->mediasize, PROT_READ\|PROT_WRITE,
205	950	mmap_flags, sc->fd, 0);
206
207	950	if (sc->base == MAP_FAILED)
208	0	ARGV_ERR("(-spersistent) failed to mmap (%s) @%p\n",
209		VAS_errtxt(errno), target);
210
211	950	smp_def_sign(sc, &sc->idn, 0, "SILO");
212	950	sc->ident = SIGN_DATA(&sc->idn);
213
214	950	i = smp_valid_silo(sc);
215	950	if (i) {
216	425	printf("Warning SILO (%s) not reloaded (reason=%d)\n",
217	425	sc->filename, i);
218	425	smp_newsilo(sc);
219	425	}
220	950	AZ(smp_valid_silo(sc));
221
222	950	smp_metrics(sc);
223
224	950	parent->priv = sc;
225
226		/* XXX: only for sendfile I guess... */
227	950	MCH_Fd_Inherit(sc->fd, "storage_persistent");
228	950	}