GCOV - Varnish Cache


/*-
 * Copyright (c) 2011 Varnish Software AS
 * All rights reserved.
 *
 * Author: Poul-Henning Kamp <phk@phk.freebsd.dk>
 *
 * SPDX-License-Identifier: BSD-2-Clause
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 * Generic memory pool
 */

#include "config.h"

#include "cache_varnishd.h"

#include <stdio.h>
#include <stdlib.h>

#include "vtim.h"

#include "VSC_mempool.h"

struct memitem {
        unsigned                        magic;
#define MEMITEM_MAGIC                   0x42e55401
        unsigned                        size;
        VTAILQ_ENTRY(memitem)           list;
        vtim_real                       touched;        // XXX -> mono?
};

VTAILQ_HEAD(memhead_s, memitem);

struct mempool {
        unsigned                        magic;
#define MEMPOOL_MAGIC                   0x37a75a8d
        char                            name[12];
        struct memhead_s                list;
        struct memhead_s                surplus;
        struct lock                     mtx;
        volatile struct poolparam       *param;
        volatile unsigned               *cur_size;
        uint64_t                        live;
        struct vsc_seg                  *vsc_seg;
        struct VSC_mempool              *vsc;
        unsigned                        n_pool;
        pthread_t                       thread;
        vtim_real                       t_now;  // XXX -> mono?
        int                             self_destruct;
};

/*---------------------------------------------------------------------
 */

static struct memitem *
mpl_alloc(const struct mempool *mpl)
{
        unsigned tsz;
        struct memitem *mi;

        CHECK_OBJ_NOTNULL(mpl, MEMPOOL_MAGIC);
        tsz = *mpl->cur_size;
        mi = calloc(1, tsz);
        AN(mi);
        mi->magic = MEMITEM_MAGIC;
        mi->size = tsz;
        mpl->vsc->sz_wanted = tsz;
        mpl->vsc->sz_actual = tsz - sizeof *mi;
        return (mi);
}

/*---------------------------------------------------------------------
 * Pool-guard
 *   Attempt to keep number of free items in pool inside bounds with
 *   minimum locking activity, and keep an eye on items at the tail
 *   of the list not getting too old.
 */

static void *
mpl_guard(void *priv)
{
        struct mempool *mpl;
        struct memitem *mi = NULL;
        vtim_dur v_statevariable_(mpl_slp);
        vtim_real last = 0;

        CAST_OBJ_NOTNULL(mpl, priv, MEMPOOL_MAGIC);
        THR_SetName(mpl->name);
        THR_Init();
        mpl_slp = 0.15; // random
        while (1) {
                VTIM_sleep(mpl_slp);
                mpl_slp = 0.814;        // random
                mpl->t_now = VTIM_real();

                if (mi != NULL && (mpl->n_pool > mpl->param->max_pool ||
                    mi->size < *mpl->cur_size)) {
                        CHECK_OBJ(mi, MEMITEM_MAGIC);
                        FREE_OBJ(mi);
                }

                if (mi == NULL && mpl->n_pool < mpl->param->min_pool)
                        mi = mpl_alloc(mpl);

                if (mpl->n_pool < mpl->param->min_pool && mi != NULL) {
                        /* can do */
                } else if (mpl->n_pool > mpl->param->max_pool && mi == NULL) {
                        /* can do */
                } else if (!VTAILQ_EMPTY(&mpl->surplus)) {
                        /* can do */
                } else if (last + .1 * mpl->param->max_age < mpl->t_now) {
                        /* should do */
                } else if (mpl->self_destruct) {
                        /* can do */
                } else {
                        continue;       /* nothing to do */
                }

                mpl_slp = 0.314;        // random

                if (Lck_Trylock(&mpl->mtx))
                        continue;

                if (mpl->self_destruct) {
                        AZ(mpl->live);
                        while (1) {
                                if (mi == NULL) {
                                        mi = VTAILQ_FIRST(&mpl->list);
                                        if (mi != NULL) {
                                                mpl->vsc->pool = --mpl->n_pool;
                                                VTAILQ_REMOVE(&mpl->list,
                                                    mi, list);
                                        }
                                }
                                if (mi == NULL) {
                                        mi = VTAILQ_FIRST(&mpl->surplus);
                                        if (mi != NULL)
                                                VTAILQ_REMOVE(&mpl->surplus,
                                                    mi, list);
                                }
                                if (mi == NULL)
                                        break;
                                CHECK_OBJ(mi, MEMITEM_MAGIC);
                                FREE_OBJ(mi);
                        }
                        VSC_mempool_Destroy(&mpl->vsc_seg);
                        Lck_Unlock(&mpl->mtx);
                        Lck_Delete(&mpl->mtx);
                        FREE_OBJ(mpl);
                        break;
                }

                if (mpl->n_pool < mpl->param->min_pool &&
                    mi != NULL && mi->size >= *mpl->cur_size) {
                        CHECK_OBJ(mi, MEMITEM_MAGIC);
                        mpl->vsc->pool = ++mpl->n_pool;
                        mi->touched = mpl->t_now;
                        VTAILQ_INSERT_HEAD(&mpl->list, mi, list);
                        mi = NULL;
                        mpl_slp = .01;  // random

                }
                if (mpl->n_pool > mpl->param->max_pool && mi == NULL) {
                        mi = VTAILQ_FIRST(&mpl->list);
                        CHECK_OBJ_NOTNULL(mi, MEMITEM_MAGIC);
                        mpl->vsc->pool = --mpl->n_pool;
                        mpl->vsc->surplus++;
                        VTAILQ_REMOVE(&mpl->list, mi, list);
                        mpl_slp = .01;  // random
                }
                if (mi == NULL) {
                        mi = VTAILQ_FIRST(&mpl->surplus);
                        if (mi != NULL) {
                                CHECK_OBJ(mi, MEMITEM_MAGIC);
                                VTAILQ_REMOVE(&mpl->surplus, mi, list);
                                mpl_slp = .01;  // random
                        }
                }
                if (mi == NULL && mpl->n_pool > mpl->param->min_pool) {
                        mi = VTAILQ_LAST(&mpl->list, memhead_s);
                        CHECK_OBJ_NOTNULL(mi, MEMITEM_MAGIC);
                        if (mi->touched + mpl->param->max_age < mpl->t_now) {
                                mpl->vsc->pool = --mpl->n_pool;
                                mpl->vsc->timeout++;
                                VTAILQ_REMOVE(&mpl->list, mi, list);
                                mpl_slp = .01;  // random
                        } else {
                                mi = NULL;
                                last = mpl->t_now;
                        }
                } else if (mpl->n_pool <= mpl->param->min_pool) {
                        last = mpl->t_now;
                }

                Lck_Unlock(&mpl->mtx);

                if (mi != NULL) {
                        CHECK_OBJ(mi, MEMITEM_MAGIC);
                        FREE_OBJ(mi);
                }
        }
        return (NULL);
}

/*---------------------------------------------------------------------
 * Create a new memory pool, and start the guard thread for it.
 */

struct mempool *
MPL_New(const char *name,
    volatile struct poolparam *pp, volatile unsigned *cur_size)
{
        struct mempool *mpl;

        ALLOC_OBJ(mpl, MEMPOOL_MAGIC);
        AN(mpl);
        bprintf(mpl->name, "MPL_%s", name);
        mpl->param = pp;
        mpl->cur_size = cur_size;
        VTAILQ_INIT(&mpl->list);
        VTAILQ_INIT(&mpl->surplus);
        Lck_New(&mpl->mtx, lck_mempool);
        /* XXX: prealloc min_pool */
        mpl->vsc = VSC_mempool_New(NULL, &mpl->vsc_seg, mpl->name + 4);
        AN(mpl->vsc);
        PTOK(pthread_create(&mpl->thread, NULL, mpl_guard, mpl));
        PTOK(pthread_detach(mpl->thread));
        return (mpl);
}

/*---------------------------------------------------------------------
 * Destroy a memory pool.  There must be no live items, and we cheat
 * and leave all the hard work to the guard thread.
 */

void
MPL_Destroy(struct mempool **mpp)
{
        struct mempool *mpl;

        TAKE_OBJ_NOTNULL(mpl, mpp, MEMPOOL_MAGIC);
        Lck_Lock(&mpl->mtx);
        AZ(mpl->live);
        mpl->self_destruct = 1;
        Lck_Unlock(&mpl->mtx);
}

/*---------------------------------------------------------------------
 */

void *
MPL_Get(struct mempool *mpl, unsigned *size)
{
        struct memitem *mi;

        CHECK_OBJ_NOTNULL(mpl, MEMPOOL_MAGIC);
        AN(size);

        Lck_Lock(&mpl->mtx);

        mpl->vsc->allocs++;
        mpl->vsc->live = ++mpl->live;

        do {
                mi = VTAILQ_FIRST(&mpl->list);
                if (mi == NULL) {
                        mpl->vsc->randry++;
                        break;
                }
                mpl->vsc->pool = --mpl->n_pool;
                CHECK_OBJ(mi, MEMITEM_MAGIC);
                VTAILQ_REMOVE(&mpl->list, mi, list);
                if (mi->size < *mpl->cur_size) {
                        mpl->vsc->toosmall++;
                        VTAILQ_INSERT_HEAD(&mpl->surplus, mi, list);
                        mi = NULL;
                } else {
                        mpl->vsc->recycle++;
                }
        } while (mi == NULL);

        Lck_Unlock(&mpl->mtx);

        if (mi == NULL)
                mi = mpl_alloc(mpl);
        *size = mi->size - sizeof *mi;

        CHECK_OBJ(mi, MEMITEM_MAGIC);
        /* Throw away sizeof info for FlexeLint: */
        return ((void *)(uintptr_t)(mi + 1));
}

void
MPL_Free(struct mempool *mpl, void *item)
{
        struct memitem *mi;

        CHECK_OBJ_NOTNULL(mpl, MEMPOOL_MAGIC);
        AN(item);

        mi = (void*)((uintptr_t)item - sizeof(*mi));
        CHECK_OBJ_NOTNULL(mi, MEMITEM_MAGIC);
        memset(item, 0, mi->size - sizeof *mi);

        Lck_Lock(&mpl->mtx);

        mpl->vsc->frees++;
        mpl->vsc->live = --mpl->live;

        if (mi->size < *mpl->cur_size) {
                mpl->vsc->toosmall++;
                VTAILQ_INSERT_HEAD(&mpl->surplus, mi, list);
        } else {
                mpl->vsc->pool = ++mpl->n_pool;
                mi->touched = mpl->t_now;
                VTAILQ_INSERT_HEAD(&mpl->list, mi, list);
        }

        Lck_Unlock(&mpl->mtx);
}

void
MPL_AssertSane(const void *item)
{
        struct memitem *mi;
        mi = (void*)((uintptr_t)item - sizeof(*mi));
        CHECK_OBJ_NOTNULL(mi, MEMITEM_MAGIC);
}

		varnish-cache/bin/varnishd/cache/cache_mempool.c
0		/*-
1		* Copyright (c) 2011 Varnish Software AS
2		* All rights reserved.
3		*
4		* Author: Poul-Henning Kamp <phk@phk.freebsd.dk>
5		*
6		* SPDX-License-Identifier: BSD-2-Clause
7		*
8		* Redistribution and use in source and binary forms, with or without
9		* modification, are permitted provided that the following conditions
10		* are met:
11		* 1. Redistributions of source code must retain the above copyright
12		* notice, this list of conditions and the following disclaimer.
13		* 2. Redistributions in binary form must reproduce the above copyright
14		* notice, this list of conditions and the following disclaimer in the
15		* documentation and/or other materials provided with the distribution.
16		*
17		* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18		* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19		* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20		* ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
21		* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22		* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23		* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24		* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25		* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26		* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27		* SUCH DAMAGE.
28		*
29		* Generic memory pool
30		*/
31
32		#include "config.h"
33
34		#include "cache_varnishd.h"
35
36		#include <stdio.h>
37		#include <stdlib.h>
38
39		#include "vtim.h"
40
41		#include "VSC_mempool.h"
42
43		struct memitem {
44		unsigned magic;
45		#define MEMITEM_MAGIC 0x42e55401
46		unsigned size;
47		VTAILQ_ENTRY(memitem) list;
48		vtim_real touched; // XXX -> mono?
49		};
50
51		VTAILQ_HEAD(memhead_s, memitem);
52
53		struct mempool {
54		unsigned magic;
55		#define MEMPOOL_MAGIC 0x37a75a8d
56		char name[12];
57		struct memhead_s list;
58		struct memhead_s surplus;
59		struct lock mtx;
60		volatile struct poolparam *param;
61		volatile unsigned *cur_size;
62		uint64_t live;
63		struct vsc_seg *vsc_seg;
64		struct VSC_mempool *vsc;
65		unsigned n_pool;
66		pthread_t thread;
67		vtim_real t_now; // XXX -> mono?
68		int self_destruct;
69		};
70
71		/*---------------------------------------------------------------------
72		*/
73
74		static struct memitem *
75	1835553	mpl_alloc(const struct mempool *mpl)
76		{
77		unsigned tsz;
78		struct memitem *mi;
79
80	1835553	CHECK_OBJ_NOTNULL(mpl, MEMPOOL_MAGIC);
81	1835553	tsz = *mpl->cur_size;
82	1835553	mi = calloc(1, tsz);
83	1835553	AN(mi);
84	1835553	mi->magic = MEMITEM_MAGIC;
85	1835553	mi->size = tsz;
86	1835553	mpl->vsc->sz_wanted = tsz;
87	1835553	mpl->vsc->sz_actual = tsz - sizeof *mi;
88	1835553	return (mi);
89		}
90
91		/*---------------------------------------------------------------------
92		* Pool-guard
93		* Attempt to keep number of free items in pool inside bounds with
94		* minimum locking activity, and keep an eye on items at the tail
95		* of the list not getting too old.
96		*/
97
98		static void *
99	156	mpl_guard(void *priv)
100		{
101		struct mempool *mpl;
102	156	struct memitem *mi = NULL;
103		vtim_dur v_statevariable_(mpl_slp);
104	156	vtim_real last = 0;
105
106	156	CAST_OBJ_NOTNULL(mpl, priv, MEMPOOL_MAGIC);
107	156	THR_SetName(mpl->name);
108	156	THR_Init();
109	156	mpl_slp = 0.15; // random
110	1934973	while (1) {
111	2260537	VTIM_sleep(mpl_slp);
112	2260537	mpl_slp = 0.814; // random
113	2260537	mpl->t_now = VTIM_real();
114
115	2260537	if (mi != NULL && (mpl->n_pool > mpl->param->max_pool \|\|
116	4	mi->size < *mpl->cur_size)) {
117	0	CHECK_OBJ(mi, MEMITEM_MAGIC);
118	0	FREE_OBJ(mi);
119	0	}
120
121	2260537	if (mi == NULL && mpl->n_pool < mpl->param->min_pool)
122	1809389	mi = mpl_alloc(mpl);
123
124	2260537	if (mpl->n_pool < mpl->param->min_pool && mi != NULL) {
125		/* can do */
126	2260539	} else if (mpl->n_pool > mpl->param->max_pool && mi == NULL) {
127		/* can do */
128	423965	} else if (!VTAILQ_EMPTY(&mpl->surplus)) {
129		/* can do */
130	423043	} else if (last + .1 * mpl->param->max_age < mpl->t_now) {
131		/* should do */
132	423003	} else if (mpl->self_destruct) {
133		/* can do */
134	4	} else {
135	323990	continue; /* nothing to do */
136		}
137
138	1936549	mpl_slp = 0.314; // random
139
140	1936549	if (Lck_Trylock(&mpl->mtx))
141	1574	continue;
142
143	1934975	if (mpl->self_destruct) {
144	160	AZ(mpl->live);
145	1765	while (1) {
146	1765	if (mi == NULL) {
147	1758	mi = VTAILQ_FIRST(&mpl->list);
148	1758	if (mi != NULL) {
149	1604	mpl->vsc->pool = --mpl->n_pool;
150	1604	VTAILQ_REMOVE(&mpl->list,
151		mi, list);
152	1604	}
153	1758	}
154	1765	if (mi == NULL) {
155	160	mi = VTAILQ_FIRST(&mpl->surplus);
156	160	if (mi != NULL)
157	0	VTAILQ_REMOVE(&mpl->surplus,
158		mi, list);
159	160	}
160	1765	if (mi == NULL)
161	160	break;
162	1605	CHECK_OBJ(mi, MEMITEM_MAGIC);
163	1605	FREE_OBJ(mi);
164		}
165	160	VSC_mempool_Destroy(&mpl->vsc_seg);
166	160	Lck_Unlock(&mpl->mtx);
167	160	Lck_Delete(&mpl->mtx);
168	160	FREE_OBJ(mpl);
169	160	break;
170		}
171
172	3770279	if (mpl->n_pool < mpl->param->min_pool &&
173	1835798	mi != NULL && mi->size >= *mpl->cur_size) {
174	1835208	CHECK_OBJ(mi, MEMITEM_MAGIC);
175	1835208	mpl->vsc->pool = ++mpl->n_pool;
176	1835208	mi->touched = mpl->t_now;
177	1835208	VTAILQ_INSERT_HEAD(&mpl->list, mi, list);
178	1835208	mi = NULL;
179	1835208	mpl_slp = .01; // random
180
181	1835208	}
182	1934815	if (mpl->n_pool > mpl->param->max_pool && mi == NULL) {
183	922	mi = VTAILQ_FIRST(&mpl->list);
184	922	CHECK_OBJ_NOTNULL(mi, MEMITEM_MAGIC);
185	922	mpl->vsc->pool = --mpl->n_pool;
186	922	mpl->vsc->surplus++;
187	922	VTAILQ_REMOVE(&mpl->list, mi, list);
188	922	mpl_slp = .01; // random
189	922	}
190	1934817	if (mi == NULL) {
191	1934131	mi = VTAILQ_FIRST(&mpl->surplus);
192	1934131	if (mi != NULL) {
193	40	CHECK_OBJ(mi, MEMITEM_MAGIC);
194	40	VTAILQ_REMOVE(&mpl->surplus, mi, list);
195	40	mpl_slp = .01; // random
196	40	}
197	1934131	}
198	1934817	if (mi == NULL && mpl->n_pool > mpl->param->min_pool) {
199	15114	mi = VTAILQ_LAST(&mpl->list, memhead_s);
200	15114	CHECK_OBJ_NOTNULL(mi, MEMITEM_MAGIC);
201	15114	if (mi->touched + mpl->param->max_age < mpl->t_now) {
202	6119	mpl->vsc->pool = --mpl->n_pool;
203	6119	mpl->vsc->timeout++;
204	6119	VTAILQ_REMOVE(&mpl->list, mi, list);
205	6119	mpl_slp = .01; // random
206	6119	} else {
207	8995	mi = NULL;
208	8995	last = mpl->t_now;
209		}
210	1934817	} else if (mpl->n_pool <= mpl->param->min_pool) {
211	1919062	last = mpl->t_now;
212	1919062	}
213
214	1934817	Lck_Unlock(&mpl->mtx);
215
216	1934817	if (mi != NULL) {
217	7038	CHECK_OBJ(mi, MEMITEM_MAGIC);
218	7038	FREE_OBJ(mi);
219	7038	}
220		}
221	160	return (NULL);
222		}
223
224		/*---------------------------------------------------------------------
225		* Create a new memory pool, and start the guard thread for it.
226		*/
227
228		struct mempool *
229	181778	MPL_New(const char *name,
230		volatile struct poolparam pp, volatile unsigned cur_size)
231		{
232		struct mempool *mpl;
233
234	181778	ALLOC_OBJ(mpl, MEMPOOL_MAGIC);
235	181778	AN(mpl);
236	181778	bprintf(mpl->name, "MPL_%s", name);
237	181778	mpl->param = pp;
238	181778	mpl->cur_size = cur_size;
239	181778	VTAILQ_INIT(&mpl->list);
240	181778	VTAILQ_INIT(&mpl->surplus);
241	181778	Lck_New(&mpl->mtx, lck_mempool);
242		/* XXX: prealloc min_pool */
243	181778	mpl->vsc = VSC_mempool_New(NULL, &mpl->vsc_seg, mpl->name + 4);
244	181778	AN(mpl->vsc);
245	181778	PTOK(pthread_create(&mpl->thread, NULL, mpl_guard, mpl));
246	181778	PTOK(pthread_detach(mpl->thread));
247	181778	return (mpl);
248		}
249
250		/*---------------------------------------------------------------------
251		* Destroy a memory pool. There must be no live items, and we cheat
252		* and leave all the hard work to the guard thread.
253		*/
254
255		void
256	160	MPL_Destroy(struct mempool **mpp)
257		{
258		struct mempool *mpl;
259
260	160	TAKE_OBJ_NOTNULL(mpl, mpp, MEMPOOL_MAGIC);
261	160	Lck_Lock(&mpl->mtx);
262	160	AZ(mpl->live);
263	160	mpl->self_destruct = 1;
264	160	Lck_Unlock(&mpl->mtx);
265	160	}
266
267		/*---------------------------------------------------------------------
268		*/
269
270		void *
271	290937	MPL_Get(struct mempool mpl, unsigned size)
272		{
273		struct memitem *mi;
274
275	290937	CHECK_OBJ_NOTNULL(mpl, MEMPOOL_MAGIC);
276	290937	AN(size);
277
278	290937	Lck_Lock(&mpl->mtx);
279
280	290937	mpl->vsc->allocs++;
281	290937	mpl->vsc->live = ++mpl->live;
282
283	290937	do {
284	290979	mi = VTAILQ_FIRST(&mpl->list);
285	290979	if (mi == NULL) {
286	805	mpl->vsc->randry++;
287	805	break;
288		}
289	290174	mpl->vsc->pool = --mpl->n_pool;
290	290174	CHECK_OBJ(mi, MEMITEM_MAGIC);
291	290174	VTAILQ_REMOVE(&mpl->list, mi, list);
292	290174	if (mi->size < *mpl->cur_size) {
293	40	mpl->vsc->toosmall++;
294	40	VTAILQ_INSERT_HEAD(&mpl->surplus, mi, list);
295	40	mi = NULL;
296	40	} else {
297	290134	mpl->vsc->recycle++;
298		}
299	290174	} while (mi == NULL);
300
301	290955	Lck_Unlock(&mpl->mtx);
302
303	290955	if (mi == NULL)
304	805	mi = mpl_alloc(mpl);
305	290955	size = mi->size - sizeof mi;
306
307	290955	CHECK_OBJ(mi, MEMITEM_MAGIC);
308		/* Throw away sizeof info for FlexeLint: */
309	290955	return ((void *)(uintptr_t)(mi + 1));
310		}
311
312		void
313	289231	MPL_Free(struct mempool mpl, void item)
314		{
315		struct memitem *mi;
316
317	289231	CHECK_OBJ_NOTNULL(mpl, MEMPOOL_MAGIC);
318	289231	AN(item);
319
320	289231	mi = (void)((uintptr_t)item - sizeof(mi));
321	289231	CHECK_OBJ_NOTNULL(mi, MEMITEM_MAGIC);
322	289231	memset(item, 0, mi->size - sizeof *mi);
323
324	289231	Lck_Lock(&mpl->mtx);
325
326	289231	mpl->vsc->frees++;
327	289231	mpl->vsc->live = --mpl->live;
328
329	289231	if (mi->size < *mpl->cur_size) {
330	0	mpl->vsc->toosmall++;
331	0	VTAILQ_INSERT_HEAD(&mpl->surplus, mi, list);
332	0	} else {
333	289231	mpl->vsc->pool = ++mpl->n_pool;
334	289231	mi->touched = mpl->t_now;
335	289231	VTAILQ_INSERT_HEAD(&mpl->list, mi, list);
336		}
337
338	289231	Lck_Unlock(&mpl->mtx);
339	289231	}
340
341		void
342	116320	MPL_AssertSane(const void *item)
343		{
344		struct memitem *mi;
345	116320	mi = (void)((uintptr_t)item - sizeof(mi));
346	116320	CHECK_OBJ_NOTNULL(mi, MEMITEM_MAGIC);
347	116320	}