varnish-cache/bin/varnishd/storage/storage_simple.c
0
/*-
1
 * Copyright (c) 2007-2015 Varnish Software AS
2
 * All rights reserved.
3
 *
4
 * Author: Poul-Henning Kamp <phk@phk.freebsd.dk>
5
 *
6
 * SPDX-License-Identifier: BSD-2-Clause
7
 *
8
 * Redistribution and use in source and binary forms, with or without
9
 * modification, are permitted provided that the following conditions
10
 * are met:
11
 * 1. Redistributions of source code must retain the above copyright
12
 *    notice, this list of conditions and the following disclaimer.
13
 * 2. Redistributions in binary form must reproduce the above copyright
14
 *    notice, this list of conditions and the following disclaimer in the
15
 *    documentation and/or other materials provided with the distribution.
16
 *
17
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20
 * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
21
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27
 * SUCH DAMAGE.
28
 *
29
 */
30
31
#include "config.h"
32
33
#include <stdlib.h>
34
35 5500
#include "cache/cache_varnishd.h"
36 4488
37 1048
#include "cache/cache_obj.h"
38 725
#include "cache/cache_objhead.h"
39 4488
40
#include "storage/storage.h"
41
#include "storage/storage_simple.h"
42
43
#include "vtim.h"
44
45 6100
/* Flags for allocating memory in sml_stv_alloc */
46 11810
#define LESS_MEM_ALLOCED_IS_OK  1
47
48
// marker pointer for sml_trimstore
49
static void *trim_once = &trim_once;
50
// for delayed return of hdl->last resume pointer
51
static void *null_iov = &null_iov;
52 7987
53
/*-------------------------------------------------------------------*/
54
55
static struct storage *
56
objallocwithnuke(struct worker *, const struct stevedore *, ssize_t size,
57
    int flags);
58
59
static struct storage *
60 6930
sml_stv_alloc(const struct stevedore *stv, ssize_t size, int flags)
61
{
62
        struct storage *st;
63
64 6930
        CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
65 6930
        AN(stv->sml_alloc);
66
67 6930
        if (!(flags & LESS_MEM_ALLOCED_IS_OK)) {
68 1252
                if (size > cache_param->fetch_maxchunksize)
69 0
                        return (NULL);
70
                else
71 1252
                        return (stv->sml_alloc(stv, size));
72
        }
73
74 5678
        if (size > cache_param->fetch_maxchunksize)
75 0
                size = cache_param->fetch_maxchunksize;
76
77 5678
        assert(size <= UINT_MAX);       /* field limit in struct storage */
78
79 5948
        for (;;) {
80
                /* try to allocate from it */
81 5948
                assert(size > 0);
82 5948
                st = stv->sml_alloc(stv, size);
83 5948
                if (st != NULL)
84 5656
                        break;
85
86 292
                if (size <= cache_param->fetch_chunksize)
87 22
                        break;
88
89 270
                size /= 2;
90
        }
91 5678
        CHECK_OBJ_ORNULL(st, STORAGE_MAGIC);
92 5678
        return (st);
93 6930
}
94
95
static void
96 8918
sml_stv_free(const struct stevedore *stv, struct storage *st)
97
{
98
99 8918
        CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
100 8918
        CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC);
101 8918
        if (stv->sml_free != NULL)
102 8918
                stv->sml_free(st);
103 8918
}
104
105
/*--------------------------------------------------------------------
106
 * This function is called by stevedores ->allocobj() method, which
107
 * very often will be SML_allocobj() below, to convert a slab
108
 * of storage into object which the stevedore can then register in its
109
 * internal state, before returning it to STV_NewObject().
110
 * As you probably guessed: All this for persistence.
111
 */
112
113
struct object *
114 5746
SML_MkObject(const struct stevedore *stv, struct objcore *oc, void *ptr)
115
{
116
        struct object *o;
117
118 5746
        CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
119 5746
        AN(stv->methods);
120 5746
        CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
121
122 5746
        assert(PAOK(ptr));
123
124 5746
        o = ptr;
125 5746
        INIT_OBJ(o, OBJECT_MAGIC);
126
127 5746
        VTAILQ_INIT(&o->list);
128
129 5746
        oc->stobj->stevedore = stv;
130 5746
        oc->stobj->priv = o;
131 5746
        oc->stobj->priv2 = 0;
132 5746
        return (o);
133
}
134
135
/*--------------------------------------------------------------------
136
 * This is the default ->allocobj() which all stevedores who do not
137
 * implement persistent storage can rely on.
138
 */
139
140
int v_matchproto_(storage_allocobj_f)
141 5724
SML_allocobj(struct worker *wrk, const struct stevedore *stv,
142
    struct objcore *oc, unsigned wsl)
143
{
144
        struct object *o;
145 5724
        struct storage *st = NULL;
146
        unsigned ltot;
147
148 5724
        CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
149 5724
        CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
150 5724
        CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
151
152 5724
        AN(stv->sml_alloc);
153
154 5724
        ltot = sizeof(*o) + PRNDUP(wsl);
155
156 5724
        do {
157 5730
                st = stv->sml_alloc(stv, ltot);
158 5730
                if (st != NULL && st->space < ltot) {
159 0
                        stv->sml_free(st);
160 0
                        st = NULL;
161 0
                }
162 5730
        } while (st == NULL && LRU_NukeOne(wrk, stv->lru));
163 5724
        if (st == NULL)
164 22
                return (0);
165
166 5702
        CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC);
167 5702
        o = SML_MkObject(stv, oc, st->ptr);
168 5702
        CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
169 5702
        st->len = sizeof(*o);
170 5702
        o->objstore = st;
171 5702
        return (1);
172 5724
}
173
174
void * v_matchproto_(storage_allocbuf_t)
175 588
SML_AllocBuf(struct worker *wrk, const struct stevedore *stv, size_t size,
176
    uintptr_t *ppriv)
177
{
178
        struct storage *st;
179
180 588
        CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
181 588
        CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
182 588
        AN(ppriv);
183
184 588
        if (size > UINT_MAX)
185 0
                return (NULL);
186 588
        st = objallocwithnuke(wrk, stv, size, 0);
187 588
        if (st == NULL)
188 0
                return (NULL);
189 588
        assert(st->space >= size);
190 588
        st->flags = STORAGE_F_BUFFER;
191 588
        st->len = size;
192 588
        *ppriv = (uintptr_t)st;
193 588
        return (st->ptr);
194 588
}
195
196
void v_matchproto_(storage_freebuf_t)
197 588
SML_FreeBuf(struct worker *wrk, const struct stevedore *stv, uintptr_t priv)
198
{
199
        struct storage *st;
200
201 588
        CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
202 588
        CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
203
204 588
        CAST_OBJ_NOTNULL(st, (void *)priv, STORAGE_MAGIC);
205 588
        assert(st->flags == STORAGE_F_BUFFER);
206 588
        sml_stv_free(stv, st);
207 588
}
208
209
/*---------------------------------------------------------------------
210
 */
211
212
static struct object *
213 311380
sml_getobj(struct worker *wrk, struct objcore *oc)
214
{
215
        const struct stevedore *stv;
216
        struct object *o;
217
218 311380
        CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
219 311380
        CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
220 311380
        stv = oc->stobj->stevedore;
221 311380
        CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
222 311380
        if (stv->sml_getobj != NULL)
223 586
                return (stv->sml_getobj(wrk, oc));
224 310794
        if (oc->stobj->priv == NULL)
225 0
                return (NULL);
226 310794
        CAST_OBJ_NOTNULL(o, oc->stobj->priv, OBJECT_MAGIC);
227 310794
        return (o);
228 311380
}
229
230
static void v_matchproto_(objslim_f)
231 6433
sml_slim(struct worker *wrk, struct objcore *oc)
232
{
233
        const struct stevedore *stv;
234
        struct object *o;
235
        struct storage *st, *stn;
236
237 6433
        CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
238
239 6433
        stv = oc->stobj->stevedore;
240 6433
        CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
241 6433
        o = sml_getobj(wrk, oc);
242 6433
        CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
243
244
#define OBJ_AUXATTR(U, l)                                       \
245
        do {                                                    \
246
                if (o->aa_##l != NULL) {                        \
247
                        sml_stv_free(stv, o->aa_##l);           \
248
                        o->aa_##l = NULL;                       \
249
                }                                               \
250
        } while (0);
251
#include "tbl/obj_attr.h"
252
253 9639
        VTAILQ_FOREACH_SAFE(st, &o->list, list, stn) {
254 3206
                CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC);
255 3206
                VTAILQ_REMOVE(&o->list, st, list);
256 3206
                sml_stv_free(stv, st);
257 3206
        }
258
}
259
260
static void
261 5778
sml_bocfini(const struct stevedore *stv, struct boc *boc)
262
{
263
        struct storage *st;
264
265 5778
        CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
266 5778
        CHECK_OBJ_NOTNULL(boc, BOC_MAGIC);
267
268 5778
        if (boc->stevedore_priv == NULL ||
269 3806
            boc->stevedore_priv == trim_once)
270 5368
                return;
271
272
        /* Free any leftovers from Trim */
273 410
        TAKE_OBJ_NOTNULL(st, &boc->stevedore_priv, STORAGE_MAGIC);
274 410
        sml_stv_free(stv, st);
275 5778
}
276
277
/*
278
 * called in two cases:
279
 * - oc->boc == NULL: cache object on LRU freed
280
 * - oc->boc != NULL: cache object replaced for backend error
281
 */
282
static void v_matchproto_(objfree_f)
283 3613
sml_objfree(struct worker *wrk, struct objcore *oc)
284
{
285
        const struct stevedore *stv;
286
        struct storage *st;
287
        struct object *o;
288
289 3613
        CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
290 3613
        CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
291 3613
        stv = oc->stobj->stevedore;
292 3613
        CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
293 3613
        CAST_OBJ_NOTNULL(o, oc->stobj->priv, OBJECT_MAGIC);
294
295 3613
        sml_slim(wrk, oc);
296 3613
        st = o->objstore;
297 3613
        CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC);
298 3613
        FINI_OBJ(o);
299
300 3613
        if (oc->boc != NULL)
301 42
                sml_bocfini(stv, oc->boc);
302 3571
        else if (stv->lru != NULL)
303 3571
                LRU_Remove(oc);
304
305 3613
        sml_stv_free(stv, st);
306
307 3613
        memset(oc->stobj, 0, sizeof oc->stobj);
308
309 3613
        wrk->stats->n_object--;
310 3613
}
311
312
// kept for reviewers - XXX remove later
313
#undef VAI_DBG
314
315
struct sml_hdl {
316
        struct vai_hdl_preamble preamble;
317
#define SML_HDL_MAGIC           0x37dfd996
318
        struct vai_qe           qe;
319
        struct pool_task        task;   // unfortunate
320
        struct ws               *ws;    // NULL is malloc()
321
        struct objcore          *oc;
322
        struct object           *obj;
323
        const struct stevedore  *stv;
324
        struct boc              *boc;
325
326
        struct storage          *st;    // updated by _lease()
327
328
        // only for _lease_boc()
329
        uint64_t                st_off; // already returned fragment of current st
330
        uint64_t                avail, returned;
331
        struct storage          *last;  // to resume, held back by _return()
332
};
333
334
static inline void
335 3797
sml_ai_viov_fill(struct viov *viov, struct storage *st)
336
{
337 3797
        viov->iov.iov_base = TRUST_ME(st->ptr);
338 3797
        viov->iov.iov_len = st->len;
339 3797
        viov->lease = ptr2lease(st);
340 3797
        VAI_ASSERT_LEASE(viov->lease);
341 3797
}
342
343
// sml has no mechanism to notify "I got free space again now"
344
// (we could add that, but because storage.h is used in mgt, a first attempt
345
//  looks at least like this would cause some include spill for vai_q_head or
346
//  something similar)
347
//
348
// So anyway, to get ahead we just implement a pretty stupid "call the notify
349
// some time later" on a thread
350
static void
351 0
sml_ai_later_task(struct worker *wrk, void *priv)
352
{
353
        struct sml_hdl *hdl;
354 0
        const vtim_dur dur = 0.0042;
355
356 0
        (void)wrk;
357 0
        VTIM_sleep(dur);
358 0
        CAST_VAI_HDL_NOTNULL(hdl, priv, SML_HDL_MAGIC);
359 0
        memset(&hdl->task, 0, sizeof hdl->task);
360 0
        hdl->qe.cb(hdl, hdl->qe.priv);
361 0
}
362
static void
363 0
sml_ai_later(struct worker *wrk, struct sml_hdl *hdl)
364
{
365 0
        AZ(hdl->task.func);
366 0
        AZ(hdl->task.priv);
367 0
        hdl->task.func = sml_ai_later_task;
368 0
        hdl->task.priv = hdl;
369 0
        AZ(Pool_Task(wrk->pool, &hdl->task, TASK_QUEUE_BG));
370 0
}
371
372
373
static int
374 31
sml_ai_buffer(struct worker *wrk, vai_hdl vhdl, struct vscarab *scarab)
375
{
376
        const struct stevedore *stv;
377
        struct sml_hdl *hdl;
378
        struct storage *st;
379
        struct viov *vio;
380 31
        int r = 0;
381
382 31
        (void) wrk;
383 31
        CAST_VAI_HDL_NOTNULL(hdl, vhdl, SML_HDL_MAGIC);
384 31
        stv = hdl->stv;
385 31
        CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
386
387 64
        VSCARAB_FOREACH(vio, scarab)
388 33
                if (vio->iov.iov_len > UINT_MAX)
389 0
                        return (-EINVAL);
390
391 63
        VSCARAB_FOREACH(vio, scarab) {
392 32
                st = objallocwithnuke(wrk, stv, vio->iov.iov_len, 0);
393 32
                if (st == NULL)
394 0
                        break;
395 32
                assert(st->space >= vio->iov.iov_len);
396 32
                st->flags = STORAGE_F_BUFFER;
397 32
                st->len = st->space;
398
399 32
                sml_ai_viov_fill(vio, st);
400 32
                r++;
401 32
        }
402 31
        if (r == 0) {
403 0
                sml_ai_later(wrk, hdl);
404 0
                r = -EAGAIN;
405 0
        }
406 31
        return (r);
407 31
}
408
409
static int
410 6728
sml_ai_lease_simple(struct worker *wrk, vai_hdl vhdl, struct vscarab *scarab)
411
{
412
        struct storage *st;
413
        struct sml_hdl *hdl;
414
        struct viov *viov;
415 6728
        int r = 0;
416
417 6728
        (void) wrk;
418 6728
        CAST_VAI_HDL_NOTNULL(hdl, vhdl, SML_HDL_MAGIC);
419 6728
        VSCARAB_CHECK_NOTNULL(scarab);
420
421 6728
        AZ(hdl->st_off);
422 6728
        st = hdl->st;
423 10494
        while (st != NULL && (viov = VSCARAB_GET(scarab)) != NULL) {
424 3766
                CHECK_OBJ(st, STORAGE_MAGIC);
425 3766
                sml_ai_viov_fill(viov, st);
426 3766
                r++;
427 3766
                st = VTAILQ_PREV(st, storagehead, list);
428
        }
429 6728
        hdl->st = st;
430 6728
        if (st == NULL)
431 6695
                scarab->flags |= VSCARAB_F_END;
432 6728
        return (r);
433
}
434
435
/*
436
 * on leases while streaming (with a boc):
437
 *
438
 * SML uses the lease return facility to implement the "free behind" for
439
 * OC_F_TRANSIENT objects. When streaming, we also return leases on
440
 * fragments of sts, but we must only "free behind" when we are done with the
441
 * last fragment.
442
 *
443
 * So we use a magic lease to signal "this is only a fragment", which we ignore
444
 * on returns
445
 */
446
447
static int
448 6729
sml_ai_lease_boc(struct worker *wrk, vai_hdl vhdl, struct vscarab *scarab)
449
{
450 6729
        enum boc_state_e state = BOS_INVALID;
451
        struct storage *next;
452
        struct sml_hdl *hdl;
453
        struct viov *viov;
454 6729
        int r = 0;
455
456 6729
        CAST_VAI_HDL_NOTNULL(hdl, vhdl, SML_HDL_MAGIC);
457 6729
        VSCARAB_CHECK_NOTNULL(scarab);
458 6729
        assert(hdl->boc == hdl->oc->boc);
459
460 6729
        if (hdl->avail == hdl->returned) {
461 13376
                hdl->avail = ObjVAIGetExtend(wrk, hdl->oc, hdl->returned,
462 6688
                    &state, &hdl->qe);
463 6688
                assert(state >= BOS_STREAM);
464 6688
                if (state == BOS_FAILED) {
465 32
                        hdl->last = NULL;
466 32
                        return (-EPIPE);
467
                }
468 6656
                else if (state == BOS_FINISHED)
469 1494
                        (void)0;
470 5162
                else if (hdl->avail == hdl->returned) {
471
                        // ObjVAIGetExtend() has scheduled a notification
472 2647
                        if (hdl->boc->transit_buffer > 0)
473 595
                                return (-ENOBUFS);
474
                        else
475 2052
                                return (-EAGAIN);
476
                }
477
                else
478 2515
                        assert(state < BOS_FINISHED);
479 4009
        }
480 4050
        Lck_Lock(&hdl->boc->mtx);
481 4050
        if (hdl->st == NULL && hdl->last != NULL)
482 1748
                hdl->st = VTAILQ_PREV(hdl->last, storagehead, list);
483 4050
        if (hdl->last != NULL && state < BOS_FINISHED) {
484 330
                viov = VSCARAB_GET(scarab);
485 330
                AN(viov);
486 330
                viov->iov.iov_base = null_iov;
487 330
                viov->iov.iov_len = 0;
488 330
                viov->lease = ptr2lease(hdl->last);
489 330
                r++;
490 330
        }
491 4050
        if (hdl->last != NULL)
492 1748
                hdl->last = NULL;
493 4050
        if (hdl->st == NULL) {
494 2943
                assert(hdl->returned == 0 || hdl->avail == hdl->returned);
495 2943
                hdl->st = VTAILQ_LAST(&hdl->obj->list, storagehead);
496 2943
        }
497 4050
        if (hdl->st == NULL)
498 2
                assert(hdl->avail == hdl->returned);
499
500 7000
        while (hdl->avail > hdl->returned && (viov = VSCARAB_GET(scarab)) != NULL) {
501 2950
                CHECK_OBJ_NOTNULL(hdl->st, STORAGE_MAGIC); // ObjVAIGetExtend ensures
502 2950
                assert(hdl->boc == hdl->oc->boc);
503 2950
                assert(hdl->st_off <= hdl->st->space);
504 2950
                size_t av = hdl->avail - hdl->returned;
505 2950
                size_t l = hdl->st->space - hdl->st_off;
506 2950
                AN(l);
507 2950
                if (l > av)
508 384
                        l = av;
509 2950
                viov->iov.iov_base = TRUST_ME(hdl->st->ptr + hdl->st_off);
510 2950
                viov->iov.iov_len = l;
511 2950
                if (hdl->st_off + l == hdl->st->space) {
512 2566
                        next = VTAILQ_PREV(hdl->st, storagehead, list);
513 2566
                        AZ(hdl->last);
514 2566
                        if (next == NULL) {
515 1768
                                hdl->last = hdl->st;
516 1768
                                viov->lease = VAI_LEASE_NORET;
517 1768
                        }
518
                        else {
519 798
                                CHECK_OBJ(next, STORAGE_MAGIC);
520 798
                                viov->lease = ptr2lease(hdl->st);
521
                        }
522
#ifdef VAI_DBG
523
                        if (wrk->vsl)
524
                                VSLb(wrk->vsl, SLT_Debug, "off %zu + l %zu == space st %p next st %p stvprv %p",
525
                                    hdl->st_off, l, hdl->st, next, hdl->boc->stevedore_priv);
526
#endif
527 2566
                        hdl->st_off = 0;
528 2566
                        hdl->st = next;
529 2566
                }
530
                else {
531 384
                        viov->lease = VAI_LEASE_NORET;
532 384
                        hdl->st_off += l;
533
                }
534 2950
                hdl->returned += l;
535 2950
                VAI_ASSERT_LEASE(viov->lease);
536 2950
                r++;
537
        }
538
539 4050
        Lck_Unlock(&hdl->boc->mtx);
540 4050
        if (state != BOS_FINISHED && hdl->avail == hdl->returned) {
541 5114
                hdl->avail = ObjVAIGetExtend(wrk, hdl->oc, hdl->returned,
542 2557
                    &state, &hdl->qe);
543 2557
        }
544 4050
        if (state == BOS_FINISHED && hdl->avail == hdl->returned)
545 1494
                scarab->flags |= VSCARAB_F_END;
546 4050
        return (r);
547 6729
}
548
549
// return only buffers, used if object is not streaming
550
static void v_matchproto_(vai_return_f)
551 4661
sml_ai_return_buffers(struct worker *wrk, vai_hdl vhdl, struct vscaret *scaret)
552
{
553
        struct storage *st;
554
        struct sml_hdl *hdl;
555
        uint64_t *p;
556
557 4661
        (void) wrk;
558 4661
        CAST_VAI_HDL_NOTNULL(hdl, vhdl, SML_HDL_MAGIC);
559
560 9746
        VSCARET_FOREACH(p, scaret) {
561 5085
                if (*p == VAI_LEASE_NORET)
562 1207
                        continue;
563 3878
                CAST_OBJ_NOTNULL(st, lease2ptr(*p), STORAGE_MAGIC);
564 3878
                if ((st->flags & STORAGE_F_BUFFER) == 0)
565 3846
                        continue;
566 32
                sml_stv_free(hdl->stv, st);
567 32
        }
568 4661
        VSCARET_INIT(scaret, scaret->capacity);
569 4661
}
570
571
// generic return for buffers and object leases, used when streaming
572
static void v_matchproto_(vai_return_f)
573 1353
sml_ai_return(struct worker *wrk, vai_hdl vhdl, struct vscaret *scaret)
574
{
575
        struct storage *st;
576
        struct sml_hdl *hdl;
577
        uint64_t *p;
578
579 1353
        (void) wrk;
580 1353
        CAST_VAI_HDL_NOTNULL(hdl, vhdl, SML_HDL_MAGIC);
581 1353
        VSCARET_CHECK_NOTNULL(scaret);
582 1353
        if (scaret->used == 0)
583 0
                return;
584
585
        // callback is only registered if needed
586 1353
        assert(hdl->boc != NULL && (hdl->oc->flags & OC_F_TRANSIENT) != 0);
587
588
        // filter noret and last
589 1353
        VSCARET_LOCAL(todo, scaret->used);
590 3408
        VSCARET_FOREACH(p, scaret) {
591 2055
                if (*p == VAI_LEASE_NORET)
592 1009
                        continue;
593 1046
                CAST_OBJ_NOTNULL(st, lease2ptr(*p), STORAGE_MAGIC);
594 1046
                VSCARET_ADD(todo, *p);
595 1046
        }
596 1353
        VSCARET_INIT(scaret, scaret->capacity);
597
598 1353
        Lck_Lock(&hdl->boc->mtx);
599 2399
        VSCARET_FOREACH(p, todo) {
600 1046
                CAST_OBJ_NOTNULL(st, lease2ptr(*p), STORAGE_MAGIC);
601 1046
                if ((st->flags & STORAGE_F_BUFFER) != 0)
602 0
                        continue;
603 1046
                VTAILQ_REMOVE(&hdl->obj->list, st, list);
604 1046
                if (st == hdl->boc->stevedore_priv)
605 0
                        hdl->boc->stevedore_priv = trim_once;
606 1046
        }
607 1353
        Lck_Unlock(&hdl->boc->mtx);
608
609 2399
        VSCARET_FOREACH(p, todo) {
610 1046
                CAST_OBJ_NOTNULL(st, lease2ptr(*p), STORAGE_MAGIC);
611
#ifdef VAI_DBG
612
                if (wrk->vsl != NULL)
613
                        VSLb(wrk->vsl, SLT_Debug, "ret %p", st);
614
#endif
615 1046
                sml_stv_free(hdl->stv, st);
616 1046
        }
617 1353
}
618
619
static void v_matchproto_(vai_fini_f)
620 4954
sml_ai_fini(struct worker *wrk, vai_hdl *vai_hdlp)
621
{
622
        struct sml_hdl *hdl;
623
624 4954
        AN(vai_hdlp);
625 4954
        CAST_VAI_HDL_NOTNULL(hdl, *vai_hdlp, SML_HDL_MAGIC);
626 4954
        *vai_hdlp = NULL;
627
628 4954
        if (hdl->boc != NULL) {
629 1541
                ObjVAICancel(wrk, hdl->boc, &hdl->qe);
630 1541
                HSH_DerefBoc(wrk, hdl->oc);
631 1541
                hdl->boc = NULL;
632 1541
        }
633
634 4954
        if (hdl->ws != NULL)
635 64
                WS_Release(hdl->ws, 0);
636
        else
637 4890
                free(hdl);
638 4954
}
639
640
static vai_hdl v_matchproto_(vai_init_f)
641 4955
sml_ai_init(struct worker *wrk, struct objcore *oc, struct ws *ws,
642
    vai_notify_cb *notify, void *notify_priv)
643
{
644
        struct sml_hdl *hdl;
645 4955
        const size_t sz = sizeof *hdl;
646
647 4955
        if (ws != NULL && WS_ReserveSize(ws, (unsigned)sz))
648 64
                hdl = WS_Reservation(ws);
649
        else {
650 4891
                hdl = malloc(sz);
651 4891
                ws = NULL;
652
        }
653
654 4955
        AN(hdl);
655 4955
        INIT_VAI_HDL(hdl, SML_HDL_MAGIC);
656 4955
        hdl->preamble.vai_lease = sml_ai_lease_simple;
657 4955
        hdl->preamble.vai_buffer = sml_ai_buffer;
658 4955
        hdl->preamble.vai_return = sml_ai_return_buffers;
659 4955
        hdl->preamble.vai_fini = sml_ai_fini;
660 4955
        hdl->ws = ws;
661
662 4955
        hdl->oc = oc;
663 4955
        hdl->obj = sml_getobj(wrk, oc);
664 4955
        CHECK_OBJ_NOTNULL(hdl->obj, OBJECT_MAGIC);
665 4955
        hdl->stv = oc->stobj->stevedore;
666 4955
        CHECK_OBJ_NOTNULL(hdl->stv, STEVEDORE_MAGIC);
667
668
669 4955
        hdl->qe.magic = VAI_Q_MAGIC;
670 4955
        hdl->qe.cb = notify;
671 4955
        hdl->qe.hdl = hdl;
672 4955
        hdl->qe.priv = notify_priv;
673
674 4955
        hdl->boc = HSH_RefBoc(oc);
675 4955
        if (hdl->boc == NULL) {
676 3414
                hdl->st = VTAILQ_LAST(&hdl->obj->list, storagehead);
677 3414
                CHECK_OBJ_ORNULL(hdl->st, STORAGE_MAGIC);
678 3414
                return (hdl);
679
        }
680
        /* we only initialize notifications if we have a boc, so
681
         * any wrong attempt triggers magic checks.
682
         */
683 1541
        hdl->preamble.vai_lease = sml_ai_lease_boc;
684 1541
        if ((hdl->oc->flags & OC_F_TRANSIENT) != 0)
685 685
                hdl->preamble.vai_return = sml_ai_return;
686 1541
        return (hdl);
687 4955
}
688
689
/*
690
 * trivial notification to allow the iterator to simply block
691
 */
692
struct sml_notify {
693
        unsigned                magic;
694
#define SML_NOTIFY_MAGIC        0x4589af31
695
        unsigned                hasmore;
696
        pthread_mutex_t         mtx;
697
        pthread_cond_t          cond;
698
};
699
700
static void
701 4892
sml_notify_init(struct sml_notify *sn)
702
{
703
704 4892
        INIT_OBJ(sn, SML_NOTIFY_MAGIC);
705 4892
        AZ(pthread_mutex_init(&sn->mtx, NULL));
706 4892
        AZ(pthread_cond_init(&sn->cond, NULL));
707 4892
}
708
709
static void
710 4891
sml_notify_fini(struct sml_notify *sn)
711
{
712
713 4891
        CHECK_OBJ_NOTNULL(sn, SML_NOTIFY_MAGIC);
714 4891
        AZ(pthread_mutex_destroy(&sn->mtx));
715 4891
        AZ(pthread_cond_destroy(&sn->cond));
716 4891
}
717
718
static void v_matchproto_(vai_notify_cb)
719 2654
sml_notify(vai_hdl hdl, void *priv)
720
{
721
        struct sml_notify *sn;
722
723 2654
        (void) hdl;
724 2654
        CAST_OBJ_NOTNULL(sn, priv, SML_NOTIFY_MAGIC);
725 2654
        AZ(pthread_mutex_lock(&sn->mtx));
726 2654
        sn->hasmore = 1;
727 2654
        AZ(pthread_cond_signal(&sn->cond));
728 2654
        AZ(pthread_mutex_unlock(&sn->mtx));
729
730 2654
}
731
732
static void
733 2608
sml_notify_wait(struct sml_notify *sn)
734
{
735
736 2608
        CHECK_OBJ_NOTNULL(sn, SML_NOTIFY_MAGIC);
737 2608
        AZ(pthread_mutex_lock(&sn->mtx));
738 4461
        while (sn->hasmore == 0)
739 1853
                AZ(pthread_cond_wait(&sn->cond, &sn->mtx));
740 2608
        AN(sn->hasmore);
741 2608
        sn->hasmore = 0;
742 2608
        AZ(pthread_mutex_unlock(&sn->mtx));
743 2608
}
744
745
static int v_matchproto_(objiterator_f)
746 4892
sml_iterator(struct worker *wrk, struct objcore *oc,
747
    void *priv, objiterate_f *func, int final)
748
{
749
        struct sml_notify sn;
750
        struct viov *vio, *last;
751
        unsigned u, uu;
752
        vai_hdl hdl;
753
        int nn, r, r2, islast;
754
755 4892
        VSCARAB_LOCAL(scarab, 16);
756 4892
        VSCARET_LOCAL(scaret, 16);
757
758 4892
        (void) final; // phase out?
759 4892
        sml_notify_init(&sn);
760 4892
        hdl = ObjVAIinit(wrk, oc, NULL, sml_notify, &sn);
761 4892
        AN(hdl);
762
763 4892
        r = u = 0;
764
765 4892
        do {
766 10801
                do {
767 13351
                        nn = ObjVAIlease(wrk, hdl, scarab);
768 13351
                        if (nn <= 0 || scarab->flags & VSCARAB_F_END)
769 10798
                                break;
770 2553
                } while (scarab->used < scarab->capacity);
771
772
                /*
773
                 * nn is the wait/return action or 0
774
                 * nn tells us if to flush
775
                 */
776 10801
                uu = u;
777 10801
                last = VSCARAB_LAST(scarab);
778 17673
                VSCARAB_FOREACH(vio, scarab) {
779 6980
                        islast = vio == last;
780 6980
                        AZ(u & OBJ_ITER_END);
781 6980
                        if (islast && scarab->flags & VSCARAB_F_END)
782 3373
                                u |= OBJ_ITER_END;
783
784
                        // flush if it is the scarab's last IOV and we will block next
785
                        // or if we need space in the return leases array
786 6980
                        uu = u;
787 6980
                        if ((islast && nn < 0) || scaret->used == scaret->capacity - 1)
788 2475
                                uu |= OBJ_ITER_FLUSH;
789
790
                        // null iov with the only purpose to return the resume ptr lease
791
                        // exception needed because assert(len > 0) in VDP_bytes()
792 6980
                        if (vio->iov.iov_base == null_iov)
793 330
                                r = 0;
794
                        else
795 6650
                                r = func(priv, uu, vio->iov.iov_base, vio->iov.iov_len);
796 6980
                        if (r != 0)
797 108
                                break;
798
799
                        // sufficient space ensured by capacity check above
800 6872
                        VSCARET_ADD(scaret, vio->lease);
801
802
#ifdef VAI_DBG
803
                        if (wrk->vsl)
804
                                VSLb(wrk->vsl, SLT_Debug, "len %zu scaret %u uu %u",
805
                                    vio->iov.iov_len, scaret->used, uu);
806
#endif
807
808
                        // whenever we have flushed, return leases
809 6872
                        if ((uu & OBJ_ITER_FLUSH) && scaret->used > 0)
810 2441
                                ObjVAIreturn(wrk, hdl, scaret);
811 6872
                }
812
813
                // return leases which we did not use if error (break)
814 10911
                VSCARAB_FOREACH_RESUME(vio, scarab) {
815 110
                        if (scaret->used == scaret->capacity)
816 0
                                ObjVAIreturn(wrk, hdl, scaret);
817 110
                        VSCARET_ADD(scaret, vio->lease);
818 110
                }
819
820
                // we have now completed the scarab
821 10801
                VSCARAB_INIT(scarab, scarab->capacity);
822
823
#ifdef VAI_DBG
824
                if (wrk->vsl)
825
                        VSLb(wrk->vsl, SLT_Debug, "r %d nn %d uu %u",
826
                            r, nn, uu);
827
#endif
828
829
                // flush before blocking if we did not already
830 10801
                if (r == 0 && (nn == -ENOBUFS || nn == -EAGAIN) &&
831 10694
                    (uu & OBJ_ITER_FLUSH) == 0) {
832 169
                        r = func(priv, OBJ_ITER_FLUSH, NULL, 0);
833 169
                        if (scaret->used > 0)
834 0
                                ObjVAIreturn(wrk, hdl, scaret);
835 169
                }
836
837 10801
                if (r == 0 && (nn == -ENOBUFS || nn == -EAGAIN)) {
838 2607
                        assert(scaret->used <= 1);
839 2607
                        sml_notify_wait(&sn);
840 2607
                }
841 8194
                else if (r == 0 && nn < 0)
842 32
                        r = -1;
843 10801
        } while (nn != 0 && r == 0);
844
845 4892
        if ((u & OBJ_ITER_END) == 0) {
846 1518
                r2 = func(priv, OBJ_ITER_END, NULL, 0);
847 1518
                if (r == 0)
848 1451
                        r = r2;
849 1518
        }
850
851 4892
        if (scaret->used > 0)
852 3409
                ObjVAIreturn(wrk, hdl, scaret);
853
854 4892
        ObjVAIfini(wrk, &hdl);
855 4892
        sml_notify_fini(&sn);
856
857 4892
        return (r);
858
}
859
860
/*--------------------------------------------------------------------
861
 */
862
863
static struct storage *
864 6520
objallocwithnuke(struct worker *wrk, const struct stevedore *stv, ssize_t size,
865
    int flags)
866
{
867 6520
        struct storage *st = NULL;
868
869 6520
        CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
870 6520
        CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
871
872 6520
        if (size > cache_param->fetch_maxchunksize) {
873 4
                if (!(flags & LESS_MEM_ALLOCED_IS_OK))
874 0
                        return (NULL);
875 4
                size = cache_param->fetch_maxchunksize;
876 4
        }
877
878 6520
        assert(size <= UINT_MAX);       /* field limit in struct storage */
879
880 6520
        do {
881
                /* try to allocate from it */
882 6536
                st = sml_stv_alloc(stv, size, flags);
883 6536
                if (st != NULL)
884 6510
                        break;
885
886
                /* no luck; try to free some space and keep trying */
887 26
                if (stv->lru == NULL)
888 0
                        break;
889 26
        } while (LRU_NukeOne(wrk, stv->lru));
890
891 6520
        CHECK_OBJ_ORNULL(st, STORAGE_MAGIC);
892 6520
        return (st);
893 6520
}
894
895
static int v_matchproto_(objgetspace_f)
896 116250
sml_getspace(struct worker *wrk, struct objcore *oc, ssize_t *sz,
897
    uint8_t **ptr)
898
{
899
        struct object *o;
900
        struct storage *st;
901
902 116250
        CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
903 116250
        CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
904 116250
        CHECK_OBJ_NOTNULL(oc->boc, BOC_MAGIC);
905 116250
        AN(sz);
906 116250
        AN(ptr);
907 116250
        if (*sz == 0)
908 108974
                *sz = cache_param->fetch_chunksize;
909 116250
        assert(*sz > 0);
910 116250
        if (oc->boc->transit_buffer > 0)
911 984
                *sz = vmin_t(ssize_t, *sz, oc->boc->transit_buffer);
912
913 116250
        o = sml_getobj(wrk, oc);
914 116250
        CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
915 116250
        CHECK_OBJ_NOTNULL(oc->boc, BOC_MAGIC);
916
917 116250
        st = VTAILQ_FIRST(&o->list);
918 116250
        if (st != NULL && st->len < st->space) {
919 110584
                *sz = st->space - st->len;
920 110584
                *ptr = st->ptr + st->len;
921 110584
                assert (*sz > 0);
922 110584
                return (1);
923
        }
924
925 5666
        st = objallocwithnuke(wrk, oc->stobj->stevedore, *sz,
926
            LESS_MEM_ALLOCED_IS_OK);
927 5666
        if (st == NULL)
928 10
                return (0);
929
930 5656
        CHECK_OBJ_NOTNULL(oc->boc, BOC_MAGIC);
931 5656
        Lck_Lock(&oc->boc->mtx);
932 5656
        VTAILQ_INSERT_HEAD(&o->list, st, list);
933 5656
        Lck_Unlock(&oc->boc->mtx);
934
935 5656
        *sz = st->space - st->len;
936 5656
        assert (*sz > 0);
937 5656
        *ptr = st->ptr + st->len;
938 5656
        return (1);
939 116250
}
940
941
static void v_matchproto_(objextend_f)
942 113334
sml_extend(struct worker *wrk, struct objcore *oc, ssize_t l)
943
{
944
        struct object *o;
945
        struct storage *st;
946
947 113334
        CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
948 113334
        assert(l > 0);
949
950 113334
        o = sml_getobj(wrk, oc);
951 113334
        CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
952 113334
        st = VTAILQ_FIRST(&o->list);
953 113334
        CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC);
954 113334
        assert(st->len + l <= st->space);
955 113334
        st->len += l;
956 113334
}
957
958
static void v_matchproto_(objtrimstore_f)
959 3808
sml_trimstore(struct worker *wrk, struct objcore *oc)
960
{
961
        const struct stevedore *stv;
962
        struct storage *st, *st1;
963
        struct object *o;
964
965 3808
        CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
966 3808
        CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
967 3808
        CHECK_OBJ_NOTNULL(oc->boc, BOC_MAGIC);
968
969 3808
        stv = oc->stobj->stevedore;
970 3808
        CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
971
972 3808
        if (oc->boc->stevedore_priv != NULL)
973 0
                WRONG("sml_trimstore already called");
974 3808
        oc->boc->stevedore_priv = trim_once;
975
976 3808
        if (stv->sml_free == NULL)
977 0
                return;
978
979 3808
        o = sml_getobj(wrk, oc);
980 3808
        CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
981 3808
        st = VTAILQ_FIRST(&o->list);
982
983 3808
        if (st == NULL)
984 0
                return;
985
986 3808
        if (st->len == 0) {
987 16
                Lck_Lock(&oc->boc->mtx);
988 16
                VTAILQ_REMOVE(&o->list, st, list);
989 16
                Lck_Unlock(&oc->boc->mtx);
990
                /* sml_bocdone frees this */
991 16
                oc->boc->stevedore_priv = st;
992 16
                return;
993
        }
994
995 3792
        if (st->space - st->len < 512)
996 3398
                return;
997
998 394
        st1 = sml_stv_alloc(stv, st->len, 0);
999 394
        if (st1 == NULL)
1000 0
                return;
1001 394
        assert(st1->space >= st->len);
1002
1003 394
        memcpy(st1->ptr, st->ptr, st->len);
1004 394
        st1->len = st->len;
1005 394
        Lck_Lock(&oc->boc->mtx);
1006 394
        VTAILQ_REMOVE(&o->list, st, list);
1007 394
        VTAILQ_INSERT_HEAD(&o->list, st1, list);
1008 394
        Lck_Unlock(&oc->boc->mtx);
1009
        /* sml_bocdone frees this */
1010 394
        oc->boc->stevedore_priv = st;
1011 3808
}
1012
1013
static void v_matchproto_(objbocdone_f)
1014 5736
sml_bocdone(struct worker *wrk, struct objcore *oc, struct boc *boc)
1015
{
1016
        const struct stevedore *stv;
1017
1018 5736
        CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
1019 5736
        CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
1020 5736
        CHECK_OBJ_NOTNULL(boc, BOC_MAGIC);
1021 5736
        stv = oc->stobj->stevedore;
1022 5736
        CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
1023
1024 5736
        sml_bocfini(stv, boc);
1025
1026 5736
        if (stv->lru != NULL) {
1027 5658
                if (isnan(wrk->lastused))
1028 0
                        wrk->lastused = VTIM_real();
1029 5658
                LRU_Add(oc, wrk->lastused);     // approx timestamp is OK
1030 5658
        }
1031 5736
}
1032
1033
static const void * v_matchproto_(objgetattr_f)
1034 45230
sml_getattr(struct worker *wrk, struct objcore *oc, enum obj_attr attr,
1035
   ssize_t *len)
1036
{
1037
        struct object *o;
1038
        ssize_t dummy;
1039
1040 45230
        CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
1041
1042 45230
        if (len == NULL)
1043 27691
                len = &dummy;
1044 45230
        o = sml_getobj(wrk, oc);
1045 45230
        CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
1046
1047 45230
        switch (attr) {
1048
                /* Fixed size attributes */
1049
#define OBJ_FIXATTR(U, l, s)                                            \
1050
        case OA_##U:                                                    \
1051
                *len = sizeof o->fa_##l;                                \
1052
                return (o->fa_##l);
1053
#include "tbl/obj_attr.h"
1054
1055
                /* Variable size attributes */
1056
#define OBJ_VARATTR(U, l)                                               \
1057
        case OA_##U:                                                    \
1058
                if (o->va_##l == NULL)                                  \
1059
                        return (NULL);                                  \
1060
                *len = o->va_##l##_len;                                 \
1061
                return (o->va_##l);
1062
#include "tbl/obj_attr.h"
1063
1064
                /* Auxiliary attributes */
1065
#define OBJ_AUXATTR(U, l)                                               \
1066
        case OA_##U:                                                    \
1067
                if (o->aa_##l == NULL)                                  \
1068
                        return (NULL);                                  \
1069
                CHECK_OBJ_NOTNULL(o->aa_##l, STORAGE_MAGIC);            \
1070
                *len = o->aa_##l->len;                                  \
1071
                return (o->aa_##l->ptr);
1072
#include "tbl/obj_attr.h"
1073
1074
        default:
1075
                break;
1076
        }
1077 0
        WRONG("Unsupported OBJ_ATTR");
1078 45230
}
1079
1080
static void * v_matchproto_(objsetattr_f)
1081 21401
sml_setattr(struct worker *wrk, struct objcore *oc, enum obj_attr attr,
1082
    ssize_t len, const void *ptr)
1083
{
1084
        struct object *o;
1085 21401
        void *retval = NULL;
1086
        struct storage *st;
1087
1088 21401
        CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
1089
1090 21401
        o = sml_getobj(wrk, oc);
1091 21401
        CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
1092 21401
        st = o->objstore;
1093
1094 21401
        switch (attr) {
1095
                /* Fixed size attributes */
1096
#define OBJ_FIXATTR(U, l, s)                                            \
1097
        case OA_##U:                                                    \
1098
                assert(len == sizeof o->fa_##l);                        \
1099
                retval = o->fa_##l;                                     \
1100
                break;
1101
#include "tbl/obj_attr.h"
1102
1103
                /* Variable size attributes */
1104
#define OBJ_VARATTR(U, l)                                               \
1105
        case OA_##U:                                                    \
1106
                if (o->va_##l##_len > 0) {                              \
1107
                        AN(o->va_##l);                                  \
1108
                        assert(len == o->va_##l##_len);                 \
1109
                        retval = o->va_##l;                             \
1110
                } else if (len > 0) {                                   \
1111
                        assert(len <= UINT_MAX);                        \
1112
                        assert(st->len + len <= st->space);             \
1113
                        o->va_##l = st->ptr + st->len;                  \
1114
                        st->len += len;                                 \
1115
                        o->va_##l##_len = len;                          \
1116
                        retval = o->va_##l;                             \
1117
                }                                                       \
1118
                break;
1119
#include "tbl/obj_attr.h"
1120
1121
                /* Auxiliary attributes */
1122
#define OBJ_AUXATTR(U, l)                                               \
1123
        case OA_##U:                                                    \
1124
                if (o->aa_##l != NULL) {                                \
1125
                        CHECK_OBJ_NOTNULL(o->aa_##l, STORAGE_MAGIC);    \
1126
                        assert(len == o->aa_##l->len);                  \
1127
                        retval = o->aa_##l->ptr;                        \
1128
                        break;                                          \
1129
                }                                                       \
1130
                if (len == 0)                                           \
1131
                        break;                                          \
1132
                o->aa_##l = objallocwithnuke(wrk, oc->stobj->stevedore, \
1133
                    len, 0);                                            \
1134
                if (o->aa_##l == NULL)                                  \
1135
                        break;                                          \
1136
                CHECK_OBJ_NOTNULL(o->aa_##l, STORAGE_MAGIC);            \
1137
                assert(len <= o->aa_##l->space);                        \
1138
                o->aa_##l->len = len;                                   \
1139
                retval = o->aa_##l->ptr;                                \
1140
                break;
1141
#include "tbl/obj_attr.h"
1142
1143
        default:
1144 0
                WRONG("Unsupported OBJ_ATTR");
1145
                break;
1146
        }
1147
1148 21401
        if (retval != NULL && ptr != NULL)
1149 780
                memcpy(retval, ptr, len);
1150 21401
        return (retval);
1151
}
1152
1153
const struct obj_methods SML_methods = {
1154
        .objfree        = sml_objfree,
1155
        .objiterator    = sml_iterator,
1156
        .objgetspace    = sml_getspace,
1157
        .objextend      = sml_extend,
1158
        .objtrimstore   = sml_trimstore,
1159
        .objbocdone     = sml_bocdone,
1160
        .objslim        = sml_slim,
1161
        .objgetattr     = sml_getattr,
1162
        .objsetattr     = sml_setattr,
1163
        .objtouch       = LRU_Touch,
1164
        .vai_init       = sml_ai_init
1165
};
1166
1167
static void
1168 8
sml_panic_st(struct vsb *vsb, const char *hd, const struct storage *st)
1169
{
1170 16
        VSB_printf(vsb, "%s = %p {priv=%p, ptr=%p, len=%u, space=%u},\n",
1171 8
            hd, st, st->priv, st->ptr, st->len, st->space);
1172 8
}
1173
1174
void
1175 4
SML_panic(struct vsb *vsb, const struct objcore *oc)
1176
{
1177
        struct object *o;
1178
        struct storage *st;
1179
1180 4
        VSB_printf(vsb, "Simple = %p,\n", oc->stobj->priv);
1181 4
        if (oc->stobj->priv == NULL)
1182 0
                return;
1183 4
        o = oc->stobj->priv;
1184 4
        PAN_CheckMagic(vsb, o, OBJECT_MAGIC);
1185 4
        sml_panic_st(vsb, "Obj", o->objstore);
1186
1187
#define OBJ_FIXATTR(U, l, sz) \
1188
        VSB_printf(vsb, "%s = ", #U); \
1189
        VSB_quote(vsb, (const void*)o->fa_##l, sz, VSB_QUOTE_HEX); \
1190
        VSB_printf(vsb, ",\n");
1191
1192
#define OBJ_VARATTR(U, l) \
1193
        VSB_printf(vsb, "%s = {len=%u, ptr=%p},\n", \
1194
            #U, o->va_##l##_len, o->va_##l);
1195
1196
#define OBJ_AUXATTR(U, l)                                               \
1197
        do {                                                            \
1198
                if (o->aa_##l != NULL) sml_panic_st(vsb, #U, o->aa_##l);\
1199
        } while(0);
1200
1201
#include "tbl/obj_attr.h"
1202
1203 8
        VTAILQ_FOREACH(st, &o->list, list) {
1204 4
                sml_panic_st(vsb, "Body", st);
1205 4
        }
1206
}