varnish-cache/bin/varnishd/storage/storage_simple.c
0
/*-
1
 * Copyright (c) 2007-2015 Varnish Software AS
2
 * All rights reserved.
3
 *
4
 * Author: Poul-Henning Kamp <phk@phk.freebsd.dk>
5
 *
6
 * SPDX-License-Identifier: BSD-2-Clause
7
 *
8
 * Redistribution and use in source and binary forms, with or without
9
 * modification, are permitted provided that the following conditions
10
 * are met:
11
 * 1. Redistributions of source code must retain the above copyright
12
 *    notice, this list of conditions and the following disclaimer.
13
 * 2. Redistributions in binary form must reproduce the above copyright
14
 *    notice, this list of conditions and the following disclaimer in the
15
 *    documentation and/or other materials provided with the distribution.
16
 *
17
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20
 * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
21
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27
 * SUCH DAMAGE.
28
 *
29
 */
30
31
#include "config.h"
32
33
#include <stdlib.h>
34
35 110080
#include "cache/cache_varnishd.h"
36 89798
37 20959
#include "cache/cache_obj.h"
38 14482
#include "cache/cache_objhead.h"
39 89797
40
#include "storage/storage.h"
41
#include "storage/storage_simple.h"
42
43
#include "vtim.h"
44
45 122000
/* Flags for allocating memory in sml_stv_alloc */
46 236422
#define LESS_MEM_ALLOCED_IS_OK  1
47
48
// marker pointer for sml_trimstore
49
static void *trim_once = &trim_once;
50
// for delayed return of hdl->last resume pointer
51
static void *null_iov = &null_iov;
52 159860
53
/*-------------------------------------------------------------------*/
54
55
static struct storage *
56
objallocwithnuke(struct worker *, const struct stevedore *, ssize_t size,
57
    int flags);
58
59
static struct storage *
60 138757
sml_stv_alloc(const struct stevedore *stv, ssize_t size, int flags)
61
{
62
        struct storage *st;
63
64 138757
        CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
65 138757
        AN(stv->sml_alloc);
66
67 138757
        if (!(flags & LESS_MEM_ALLOCED_IS_OK)) {
68 25044
                if (size > cache_param->fetch_maxchunksize)
69 0
                        return (NULL);
70
                else
71 25044
                        return (stv->sml_alloc(stv, size));
72
        }
73
74 113713
        if (size > cache_param->fetch_maxchunksize)
75 0
                size = cache_param->fetch_maxchunksize;
76
77 113713
        assert(size <= UINT_MAX);       /* field limit in struct storage */
78
79 119107
        for (;;) {
80
                /* try to allocate from it */
81 119107
                assert(size > 0);
82 119107
                st = stv->sml_alloc(stv, size);
83 119107
                if (st != NULL)
84 113274
                        break;
85
86 5833
                if (size <= cache_param->fetch_chunksize)
87 439
                        break;
88
89 5394
                size /= 2;
90
        }
91 113713
        CHECK_OBJ_ORNULL(st, STORAGE_MAGIC);
92 113713
        return (st);
93 138757
}
94
95
static void
96 178613
sml_stv_free(const struct stevedore *stv, struct storage *st)
97
{
98
99 178613
        CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
100 178613
        CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC);
101 178613
        if (stv->sml_free != NULL)
102 178614
                stv->sml_free(st);
103 178615
}
104
105
/*--------------------------------------------------------------------
106
 * This function is called by stevedores ->allocobj() method, which
107
 * very often will be SML_allocobj() below, to convert a slab
108
 * of storage into object which the stevedore can then register in its
109
 * internal state, before returning it to STV_NewObject().
110
 * As you probably guessed: All this for persistence.
111
 */
112
113
struct object *
114 114998
SML_MkObject(const struct stevedore *stv, struct objcore *oc, void *ptr)
115
{
116
        struct object *o;
117
118 114998
        CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
119 114998
        AN(stv->methods);
120 114998
        CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
121
122 114998
        assert(PAOK(ptr));
123
124 114998
        o = ptr;
125 114998
        INIT_OBJ(o, OBJECT_MAGIC);
126
127 114998
        VTAILQ_INIT(&o->list);
128
129 114998
        oc->stobj->stevedore = stv;
130 114998
        oc->stobj->priv = o;
131 114998
        oc->stobj->priv2 = 0;
132 114998
        return (o);
133
}
134
135
/*--------------------------------------------------------------------
136
 * This is the default ->allocobj() which all stevedores who do not
137
 * implement persistent storage can rely on.
138
 */
139
140
int v_matchproto_(storage_allocobj_f)
141 114557
SML_allocobj(struct worker *wrk, const struct stevedore *stv,
142
    struct objcore *oc, unsigned wsl)
143
{
144
        struct object *o;
145 114557
        struct storage *st = NULL;
146
        unsigned ltot;
147
148 114557
        CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
149 114557
        CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
150 114557
        CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
151
152 114557
        AN(stv->sml_alloc);
153
154 114557
        ltot = sizeof(*o) + PRNDUP(wsl);
155
156 114557
        do {
157 114678
                st = stv->sml_alloc(stv, ltot);
158 114678
                if (st != NULL && st->space < ltot) {
159 0
                        stv->sml_free(st);
160 0
                        st = NULL;
161 0
                }
162 114678
        } while (st == NULL && LRU_NukeOne(wrk, stv->lru));
163 114557
        if (st == NULL)
164 440
                return (0);
165
166 114117
        CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC);
167 114117
        o = SML_MkObject(stv, oc, st->ptr);
168 114117
        CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
169 114117
        st->len = sizeof(*o);
170 114117
        o->objstore = st;
171 114117
        return (1);
172 114557
}
173
174
void * v_matchproto_(storage_allocbuf_t)
175 11760
SML_AllocBuf(struct worker *wrk, const struct stevedore *stv, size_t size,
176
    uintptr_t *ppriv)
177
{
178
        struct storage *st;
179
180 11760
        CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
181 11760
        CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
182 11760
        AN(ppriv);
183
184 11760
        if (size > UINT_MAX)
185 0
                return (NULL);
186 11760
        st = objallocwithnuke(wrk, stv, size, 0);
187 11760
        if (st == NULL)
188 0
                return (NULL);
189 11760
        assert(st->space >= size);
190 11760
        st->flags = STORAGE_F_BUFFER;
191 11760
        st->len = size;
192 11760
        *ppriv = (uintptr_t)st;
193 11760
        return (st->ptr);
194 11760
}
195
196
void v_matchproto_(storage_freebuf_t)
197 11760
SML_FreeBuf(struct worker *wrk, const struct stevedore *stv, uintptr_t priv)
198
{
199
        struct storage *st;
200
201 11760
        CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
202 11760
        CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
203
204 11760
        CAST_OBJ_NOTNULL(st, (void *)priv, STORAGE_MAGIC);
205 11760
        assert(st->flags == STORAGE_F_BUFFER);
206 11760
        sml_stv_free(stv, st);
207 11760
}
208
209
/*---------------------------------------------------------------------
210
 */
211
212
static struct object *
213 6224731
sml_getobj(struct worker *wrk, struct objcore *oc)
214
{
215
        const struct stevedore *stv;
216
        struct object *o;
217
218 6224731
        CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
219 6224731
        CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
220 6224731
        stv = oc->stobj->stevedore;
221 6224731
        CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
222 6224731
        if (stv->sml_getobj != NULL)
223 11720
                return (stv->sml_getobj(wrk, oc));
224 6213011
        if (oc->stobj->priv == NULL)
225 0
                return (NULL);
226 6213011
        CAST_OBJ_NOTNULL(o, oc->stobj->priv, OBJECT_MAGIC);
227 6213011
        return (o);
228 6224731
}
229
230
static void v_matchproto_(objslim_f)
231 128780
sml_slim(struct worker *wrk, struct objcore *oc)
232
{
233
        const struct stevedore *stv;
234
        struct object *o;
235
        struct storage *st, *stn;
236
237 128780
        CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
238
239 128780
        stv = oc->stobj->stevedore;
240 128780
        CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
241 128780
        o = sml_getobj(wrk, oc);
242 128780
        CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
243
244
#define OBJ_AUXATTR(U, l)                                       \
245
        do {                                                    \
246
                if (o->aa_##l != NULL) {                        \
247
                        sml_stv_free(stv, o->aa_##l);           \
248
                        o->aa_##l = NULL;                       \
249
                }                                               \
250
        } while (0);
251
#include "tbl/obj_attr.h"
252
253 193243
        VTAILQ_FOREACH_SAFE(st, &o->list, list, stn) {
254 64463
                CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC);
255 64463
                VTAILQ_REMOVE(&o->list, st, list);
256 64463
                sml_stv_free(stv, st);
257 64463
        }
258
}
259
260
static void
261 115635
sml_bocfini(const struct stevedore *stv, struct boc *boc)
262
{
263
        struct storage *st;
264
265 115635
        CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
266 115635
        CHECK_OBJ_NOTNULL(boc, BOC_MAGIC);
267
268 115635
        if (boc->stevedore_priv == NULL ||
269 76122
            boc->stevedore_priv == trim_once)
270 107431
                return;
271
272
        /* Free any leftovers from Trim */
273 8204
        TAKE_OBJ_NOTNULL(st, &boc->stevedore_priv, STORAGE_MAGIC);
274 8204
        sml_stv_free(stv, st);
275 115635
}
276
277
/*
278
 * called in two cases:
279
 * - oc->boc == NULL: cache object on LRU freed
280
 * - oc->boc != NULL: cache object replaced for backend error
281
 */
282
static void v_matchproto_(objfree_f)
283 72346
sml_objfree(struct worker *wrk, struct objcore *oc)
284
{
285
        const struct stevedore *stv;
286
        struct storage *st;
287
        struct object *o;
288
289 72346
        CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
290 72346
        CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
291 72346
        stv = oc->stobj->stevedore;
292 72346
        CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
293 72346
        CAST_OBJ_NOTNULL(o, oc->stobj->priv, OBJECT_MAGIC);
294
295 72346
        sml_slim(wrk, oc);
296 72346
        st = o->objstore;
297 72346
        CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC);
298 72346
        FINI_OBJ(o);
299
300 72346
        if (oc->boc != NULL)
301 840
                sml_bocfini(stv, oc->boc);
302 71506
        else if (stv->lru != NULL)
303 71499
                LRU_Remove(oc);
304
305 72342
        sml_stv_free(stv, st);
306
307 72342
        memset(oc->stobj, 0, sizeof oc->stobj);
308
309 72342
        wrk->stats->n_object--;
310 72342
}
311
312
// kept for reviewers - XXX remove later
313
#undef VAI_DBG
314
315
struct sml_hdl {
316
        struct vai_hdl_preamble preamble;
317
#define SML_HDL_MAGIC           0x37dfd996
318
        struct vai_qe           qe;
319
        struct pool_task        task;   // unfortunate
320
        struct ws               *ws;    // NULL is malloc()
321
        struct objcore          *oc;
322
        struct object           *obj;
323
        const struct stevedore  *stv;
324
        struct boc              *boc;
325
326
        struct storage          *st;    // updated by _lease()
327
328
        // only for _lease_boc()
329
        uint64_t                st_off; // already returned fragment of current st
330
        uint64_t                avail, returned;
331
        struct storage          *last;  // to resume, held back by _return()
332
};
333
334
static inline void
335 75953
sml_ai_viov_fill(struct viov *viov, struct storage *st)
336
{
337 75953
        viov->iov.iov_base = TRUST_ME(st->ptr);
338 75953
        viov->iov.iov_len = st->len;
339 75953
        viov->lease = ptr2lease(st);
340 75953
        VAI_ASSERT_LEASE(viov->lease);
341 75953
}
342
343
// sml has no mechanism to notify "I got free space again now"
344
// (we could add that, but because storage.h is used in mgt, a first attempt
345
//  looks at least like this would cause some include spill for vai_q_head or
346
//  something similar)
347
//
348
// So anyway, to get ahead we just implement a pretty stupid "call the notify
349
// some time later" on a thread
350
static void
351 0
sml_ai_later_task(struct worker *wrk, void *priv)
352
{
353
        struct sml_hdl *hdl;
354 0
        const vtim_dur dur = 0.0042;
355
356 0
        (void)wrk;
357 0
        VTIM_sleep(dur);
358 0
        CAST_VAI_HDL_NOTNULL(hdl, priv, SML_HDL_MAGIC);
359 0
        memset(&hdl->task, 0, sizeof hdl->task);
360 0
        hdl->qe.cb(hdl, hdl->qe.priv);
361 0
}
362
static void
363 0
sml_ai_later(struct worker *wrk, struct sml_hdl *hdl)
364
{
365 0
        AZ(hdl->task.func);
366 0
        AZ(hdl->task.priv);
367 0
        hdl->task.func = sml_ai_later_task;
368 0
        hdl->task.priv = hdl;
369 0
        AZ(Pool_Task(wrk->pool, &hdl->task, TASK_QUEUE_BG));
370 0
}
371
372
373
static int
374 640
sml_ai_buffer(struct worker *wrk, vai_hdl vhdl, struct vscarab *scarab)
375
{
376
        const struct stevedore *stv;
377
        struct sml_hdl *hdl;
378
        struct storage *st;
379
        struct viov *vio;
380 640
        int r = 0;
381
382 640
        (void) wrk;
383 640
        CAST_VAI_HDL_NOTNULL(hdl, vhdl, SML_HDL_MAGIC);
384 640
        stv = hdl->stv;
385 640
        CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
386
387 1280
        VSCARAB_FOREACH(vio, scarab)
388 640
                if (vio->iov.iov_len > UINT_MAX)
389 0
                        return (-EINVAL);
390
391 1280
        VSCARAB_FOREACH(vio, scarab) {
392 640
                st = objallocwithnuke(wrk, stv, vio->iov.iov_len, 0);
393 640
                if (st == NULL)
394 0
                        break;
395 640
                assert(st->space >= vio->iov.iov_len);
396 640
                st->flags = STORAGE_F_BUFFER;
397 640
                st->len = st->space;
398
399 640
                sml_ai_viov_fill(vio, st);
400 640
                r++;
401 640
        }
402 640
        if (r == 0) {
403 0
                sml_ai_later(wrk, hdl);
404 0
                r = -EAGAIN;
405 0
        }
406 640
        return (r);
407 640
}
408
409
static int
410 134608
sml_ai_lease_simple(struct worker *wrk, vai_hdl vhdl, struct vscarab *scarab)
411
{
412
        struct storage *st;
413
        struct sml_hdl *hdl;
414
        struct viov *viov;
415 134608
        int r = 0;
416
417 134608
        (void) wrk;
418 134608
        CAST_VAI_HDL_NOTNULL(hdl, vhdl, SML_HDL_MAGIC);
419 134608
        VSCARAB_CHECK_NOTNULL(scarab);
420
421 134608
        AZ(hdl->st_off);
422 134608
        st = hdl->st;
423 209922
        while (st != NULL && (viov = VSCARAB_GET(scarab)) != NULL) {
424 75314
                CHECK_OBJ(st, STORAGE_MAGIC);
425 75314
                sml_ai_viov_fill(viov, st);
426 75314
                r++;
427 75314
                st = VTAILQ_PREV(st, storagehead, list);
428
        }
429 134608
        hdl->st = st;
430 134608
        if (st == NULL)
431 133968
                scarab->flags |= VSCARAB_F_END;
432 134608
        return (r);
433
}
434
435
/*
436
 * on leases while streaming (with a boc):
437
 *
438
 * SML uses the lease return facility to implement the "free behind" for
439
 * OC_F_TRANSIENT objects. When streaming, we also return leases on
440
 * fragments of sts, but we must only "free behind" when we are done with the
441
 * last fragment.
442
 *
443
 * So we use a magic lease to signal "this is only a fragment", which we ignore
444
 * on returns
445
 */
446
447
static int
448 134444
sml_ai_lease_boc(struct worker *wrk, vai_hdl vhdl, struct vscarab *scarab)
449
{
450 134444
        enum boc_state_e state = BOS_INVALID;
451
        struct storage *next;
452
        struct sml_hdl *hdl;
453
        struct viov *viov;
454 134444
        int r = 0;
455
456 134444
        CAST_VAI_HDL_NOTNULL(hdl, vhdl, SML_HDL_MAGIC);
457 134444
        VSCARAB_CHECK_NOTNULL(scarab);
458 134444
        assert(hdl->boc == hdl->oc->boc);
459
460 134444
        if (hdl->avail == hdl->returned) {
461 267160
                hdl->avail = ObjVAIGetExtend(wrk, hdl->oc, hdl->returned,
462 133580
                    &state, &hdl->qe);
463 133580
                assert(state >= BOS_STREAM);
464 133580
                if (state == BOS_FAILED) {
465 639
                        hdl->last = NULL;
466 639
                        return (-EPIPE);
467
                }
468 132941
                else if (state == BOS_FINISHED)
469 29799
                        (void)0;
470 103142
                else if (hdl->avail == hdl->returned) {
471
                        // ObjVAIGetExtend() has scheduled a notification
472 52982
                        if (hdl->boc->transit_buffer > 0)
473 12043
                                return (-ENOBUFS);
474
                        else
475 40939
                                return (-EAGAIN);
476
                }
477
                else
478 50160
                        assert(state < BOS_FINISHED);
479 79959
        }
480 80823
        Lck_Lock(&hdl->boc->mtx);
481 80823
        if (hdl->st == NULL && hdl->last != NULL)
482 34813
                hdl->st = VTAILQ_PREV(hdl->last, storagehead, list);
483 80823
        if (hdl->last != NULL && state < BOS_FINISHED) {
484 6616
                viov = VSCARAB_GET(scarab);
485 6616
                AN(viov);
486 6616
                viov->iov.iov_base = null_iov;
487 6616
                viov->iov.iov_len = 0;
488 6616
                viov->lease = ptr2lease(hdl->last);
489 6616
                r++;
490 6616
        }
491 80823
        if (hdl->last != NULL)
492 34813
                hdl->last = NULL;
493 80823
        if (hdl->st == NULL && hdl->returned == 0)
494 30658
                hdl->st = VTAILQ_LAST(&hdl->obj->list, storagehead);
495 80823
        if (hdl->st == NULL)
496 28141
                assert(hdl->avail == hdl->returned);
497
498 139561
        while (hdl->avail > hdl->returned && (viov = VSCARAB_GET(scarab)) != NULL) {
499 58738
                CHECK_OBJ_NOTNULL(hdl->st, STORAGE_MAGIC); // ObjVAIGetExtend ensures
500 58738
                assert(hdl->boc == hdl->oc->boc);
501 58738
                assert(hdl->st_off <= hdl->st->space);
502 58738
                size_t av = hdl->avail - hdl->returned;
503 58738
                size_t l = hdl->st->space - hdl->st_off;
504 58738
                AN(l);
505 58738
                if (l > av)
506 7742
                        l = av;
507 58738
                viov->iov.iov_base = TRUST_ME(hdl->st->ptr + hdl->st_off);
508 58738
                viov->iov.iov_len = l;
509 58738
                if (hdl->st_off + l == hdl->st->space) {
510 50996
                        next = VTAILQ_PREV(hdl->st, storagehead, list);
511 50996
                        AZ(hdl->last);
512 50996
                        if (next == NULL) {
513 35223
                                hdl->last = hdl->st;
514 35223
                                viov->lease = VAI_LEASE_NORET;
515 35223
                        }
516
                        else {
517 15773
                                CHECK_OBJ(next, STORAGE_MAGIC);
518 15773
                                viov->lease = ptr2lease(hdl->st);
519
                        }
520
#ifdef VAI_DBG
521
                        if (wrk->vsl)
522
                                VSLb(wrk->vsl, SLT_Debug, "off %zu + l %zu == space st %p next st %p stvprv %p",
523
                                    hdl->st_off, l, hdl->st, next, hdl->boc->stevedore_priv);
524
#endif
525 50996
                        hdl->st_off = 0;
526 50996
                        hdl->st = next;
527 50996
                }
528
                else {
529 7742
                        viov->lease = VAI_LEASE_NORET;
530 7742
                        hdl->st_off += l;
531
                }
532 58738
                hdl->returned += l;
533 58738
                VAI_ASSERT_LEASE(viov->lease);
534 58738
                r++;
535
        }
536
537 80823
        Lck_Unlock(&hdl->boc->mtx);
538 80823
        if (state != BOS_FINISHED && hdl->avail == hdl->returned) {
539 102054
                hdl->avail = ObjVAIGetExtend(wrk, hdl->oc, hdl->returned,
540 51027
                    &state, &hdl->qe);
541 51027
        }
542 80823
        if (state == BOS_FINISHED && hdl->avail == hdl->returned)
543 29804
                scarab->flags |= VSCARAB_F_END;
544 80823
        return (r);
545 134444
}
546
547
// return only buffers, used if object is not streaming
548
static void v_matchproto_(vai_return_f)
549 93025
sml_ai_return_buffers(struct worker *wrk, vai_hdl vhdl, struct vscaret *scaret)
550
{
551
        struct storage *st;
552
        struct sml_hdl *hdl;
553
        uint64_t *p;
554
555 93025
        (void) wrk;
556 93025
        CAST_VAI_HDL_NOTNULL(hdl, vhdl, SML_HDL_MAGIC);
557
558 194613
        VSCARET_FOREACH(p, scaret) {
559 101588
                if (*p == VAI_LEASE_NORET)
560 23977
                        continue;
561 77611
                CAST_OBJ_NOTNULL(st, lease2ptr(*p), STORAGE_MAGIC);
562 77611
                if ((st->flags & STORAGE_F_BUFFER) == 0)
563 76971
                        continue;
564 640
                sml_stv_free(hdl->stv, st);
565 640
        }
566 93025
        VSCARET_INIT(scaret, scaret->capacity);
567 93025
}
568
569
// generic return for buffers and object leases, used when streaming
570
static void v_matchproto_(vai_return_f)
571 27195
sml_ai_return(struct worker *wrk, vai_hdl vhdl, struct vscaret *scaret)
572
{
573
        struct storage *st;
574
        struct sml_hdl *hdl;
575
        uint64_t *p;
576
577 27195
        (void) wrk;
578 27195
        CAST_VAI_HDL_NOTNULL(hdl, vhdl, SML_HDL_MAGIC);
579 27195
        VSCARET_CHECK_NOTNULL(scaret);
580 27195
        if (scaret->used == 0)
581 0
                return;
582
583
        // callback is only registered if needed
584 27195
        assert(hdl->boc != NULL && (hdl->oc->flags & OC_F_TRANSIENT) != 0);
585
586
        // filter noret and last
587 27195
        VSCARET_LOCAL(todo, scaret->used);
588 68198
        VSCARET_FOREACH(p, scaret) {
589 41003
                if (*p == VAI_LEASE_NORET)
590 20275
                        continue;
591 20728
                CAST_OBJ_NOTNULL(st, lease2ptr(*p), STORAGE_MAGIC);
592 20728
                VSCARET_ADD(todo, *p);
593 20728
        }
594 27195
        VSCARET_INIT(scaret, scaret->capacity);
595
596 27195
        Lck_Lock(&hdl->boc->mtx);
597 47923
        VSCARET_FOREACH(p, todo) {
598 20728
                CAST_OBJ_NOTNULL(st, lease2ptr(*p), STORAGE_MAGIC);
599 20728
                if ((st->flags & STORAGE_F_BUFFER) != 0)
600 0
                        continue;
601 20728
                VTAILQ_REMOVE(&hdl->obj->list, st, list);
602 20728
                if (st == hdl->boc->stevedore_priv)
603 0
                        hdl->boc->stevedore_priv = trim_once;
604 20728
        }
605 27195
        Lck_Unlock(&hdl->boc->mtx);
606
607 47926
        VSCARET_FOREACH(p, todo) {
608 20731
                CAST_OBJ_NOTNULL(st, lease2ptr(*p), STORAGE_MAGIC);
609
#ifdef VAI_DBG
610
                if (wrk->vsl != NULL)
611
                        VSLb(wrk->vsl, SLT_Debug, "ret %p", st);
612
#endif
613 20731
                sml_stv_free(hdl->stv, st);
614 20731
        }
615 27195
}
616
617
static void v_matchproto_(vai_fini_f)
618 99107
sml_ai_fini(struct worker *wrk, vai_hdl *vai_hdlp)
619
{
620
        struct sml_hdl *hdl;
621
622 99107
        AN(vai_hdlp);
623 99107
        CAST_VAI_HDL_NOTNULL(hdl, *vai_hdlp, SML_HDL_MAGIC);
624 99107
        *vai_hdlp = NULL;
625
626 99107
        if (hdl->boc != NULL) {
627 30839
                ObjVAICancel(wrk, hdl->boc, &hdl->qe);
628 30839
                HSH_DerefBoc(wrk, hdl->oc);
629 30839
                hdl->boc = NULL;
630 30839
        }
631
632 99107
        if (hdl->ws != NULL)
633 1280
                WS_Release(hdl->ws, 0);
634
        else
635 97827
                free(hdl);
636 99107
}
637
638
static vai_hdl v_matchproto_(vai_init_f)
639 99111
sml_ai_init(struct worker *wrk, struct objcore *oc, struct ws *ws,
640
    vai_notify_cb *notify, void *notify_priv)
641
{
642
        struct sml_hdl *hdl;
643 99111
        const size_t sz = sizeof *hdl;
644
645 99111
        if (ws != NULL && WS_ReserveSize(ws, (unsigned)sz))
646 1279
                hdl = WS_Reservation(ws);
647
        else {
648 97832
                hdl = malloc(sz);
649 97832
                ws = NULL;
650
        }
651
652 99111
        AN(hdl);
653 99111
        INIT_VAI_HDL(hdl, SML_HDL_MAGIC);
654 99111
        hdl->preamble.vai_lease = sml_ai_lease_simple;
655 99111
        hdl->preamble.vai_buffer = sml_ai_buffer;
656 99111
        hdl->preamble.vai_return = sml_ai_return_buffers;
657 99111
        hdl->preamble.vai_fini = sml_ai_fini;
658 99111
        hdl->ws = ws;
659
660 99111
        hdl->oc = oc;
661 99111
        hdl->obj = sml_getobj(wrk, oc);
662 99111
        CHECK_OBJ_NOTNULL(hdl->obj, OBJECT_MAGIC);
663 99111
        hdl->stv = oc->stobj->stevedore;
664 99111
        CHECK_OBJ_NOTNULL(hdl->stv, STEVEDORE_MAGIC);
665
666
667 99111
        hdl->qe.magic = VAI_Q_MAGIC;
668 99111
        hdl->qe.cb = notify;
669 99111
        hdl->qe.hdl = hdl;
670 99111
        hdl->qe.priv = notify_priv;
671
672 99111
        hdl->boc = HSH_RefBoc(oc);
673 99111
        if (hdl->boc == NULL) {
674 68275
                hdl->st = VTAILQ_LAST(&hdl->obj->list, storagehead);
675 68275
                CHECK_OBJ_ORNULL(hdl->st, STORAGE_MAGIC);
676 68275
                return (hdl);
677
        }
678
        /* we only initialize notifications if we have a boc, so
679
         * any wrong attempt triggers magic checks.
680
         */
681 30836
        hdl->preamble.vai_lease = sml_ai_lease_boc;
682 30836
        if ((hdl->oc->flags & OC_F_TRANSIENT) != 0)
683 13718
                hdl->preamble.vai_return = sml_ai_return;
684 30836
        return (hdl);
685 99111
}
686
687
/*
688
 * trivial notification to allow the iterator to simply block
689
 */
690
struct sml_notify {
691
        unsigned                magic;
692
#define SML_NOTIFY_MAGIC        0x4589af31
693
        unsigned                hasmore;
694
        pthread_mutex_t         mtx;
695
        pthread_cond_t          cond;
696
};
697
698
static void
699 97831
sml_notify_init(struct sml_notify *sn)
700
{
701
702 97831
        INIT_OBJ(sn, SML_NOTIFY_MAGIC);
703 97831
        AZ(pthread_mutex_init(&sn->mtx, NULL));
704 97831
        AZ(pthread_cond_init(&sn->cond, NULL));
705 97831
}
706
707
static void
708 97825
sml_notify_fini(struct sml_notify *sn)
709
{
710
711 97825
        CHECK_OBJ_NOTNULL(sn, SML_NOTIFY_MAGIC);
712 97825
        AZ(pthread_mutex_destroy(&sn->mtx));
713 97825
        AZ(pthread_cond_destroy(&sn->cond));
714 97825
}
715
716
static void v_matchproto_(vai_notify_cb)
717 53025
sml_notify(vai_hdl hdl, void *priv)
718
{
719
        struct sml_notify *sn;
720
721 53025
        (void) hdl;
722 53025
        CAST_OBJ_NOTNULL(sn, priv, SML_NOTIFY_MAGIC);
723 53025
        AZ(pthread_mutex_lock(&sn->mtx));
724 53025
        sn->hasmore = 1;
725 53025
        AZ(pthread_cond_signal(&sn->cond));
726 53025
        AZ(pthread_mutex_unlock(&sn->mtx));
727
728 53025
}
729
730
static void
731 52152
sml_notify_wait(struct sml_notify *sn)
732
{
733
734 52152
        CHECK_OBJ_NOTNULL(sn, SML_NOTIFY_MAGIC);
735 52152
        AZ(pthread_mutex_lock(&sn->mtx));
736 89259
        while (sn->hasmore == 0)
737 37107
                AZ(pthread_cond_wait(&sn->cond, &sn->mtx));
738 52152
        AN(sn->hasmore);
739 52152
        sn->hasmore = 0;
740 52152
        AZ(pthread_mutex_unlock(&sn->mtx));
741 52152
}
742
743
static int v_matchproto_(objiterator_f)
744 97837
sml_iterator(struct worker *wrk, struct objcore *oc,
745
    void *priv, objiterate_f *func, int final)
746
{
747
        struct sml_notify sn;
748
        struct viov *vio, *last;
749
        unsigned u, uu;
750
        vai_hdl hdl;
751
        int nn, r, r2, islast;
752
753 97837
        VSCARAB_LOCAL(scarab, 16);
754 97837
        VSCARET_LOCAL(scaret, 16);
755
756 97837
        (void) final; // phase out?
757 97837
        sml_notify_init(&sn);
758 97837
        hdl = ObjVAIinit(wrk, oc, NULL, sml_notify, &sn);
759 97837
        AN(hdl);
760
761 97837
        r = u = 0;
762
763 97837
        do {
764 216026
                do {
765 266932
                        nn = ObjVAIlease(wrk, hdl, scarab);
766 266932
                        if (nn <= 0 || scarab->flags & VSCARAB_F_END)
767 215987
                                break;
768 50945
                } while (scarab->used < scarab->capacity);
769
770
                /*
771
                 * nn is the wait/return action or 0
772
                 * nn tells us if to flush
773
                 */
774 216026
                uu = u;
775 216026
                last = VSCARAB_LAST(scarab);
776 353225
                VSCARAB_FOREACH(vio, scarab) {
777 139332
                        islast = vio == last;
778 139332
                        AZ(u & OBJ_ITER_END);
779 139332
                        if (islast && scarab->flags & VSCARAB_F_END)
780 67397
                                u |= OBJ_ITER_END;
781
782
                        // flush if it is the scarab's last IOV and we will block next
783
                        // or if we need space in the return leases array
784 139332
                        uu = u;
785 139332
                        if ((islast && nn < 0) || scaret->used == scaret->capacity - 1)
786 49469
                                uu |= OBJ_ITER_FLUSH;
787
788
                        // null iov with the only purpose to return the resume ptr lease
789
                        // exception needed because assert(len > 0) in VDP_bytes()
790 139332
                        if (vio->iov.iov_base == null_iov)
791 6616
                                r = 0;
792
                        else
793 132716
                                r = func(priv, uu, vio->iov.iov_base, vio->iov.iov_len);
794 139332
                        if (r != 0)
795 2133
                                break;
796
797
                        // sufficient space ensured by capacity check above
798 137199
                        VSCARET_ADD(scaret, vio->lease);
799
800
#ifdef VAI_DBG
801
                        if (wrk->vsl)
802
                                VSLb(wrk->vsl, SLT_Debug, "len %zu scaret %u uu %u",
803
                                    vio->iov.iov_len, scaret->used, uu);
804
#endif
805
806
                        // whenever we have flushed, return leases
807 137199
                        if ((uu & OBJ_ITER_FLUSH) && scaret->used > 0)
808 48770
                                ObjVAIreturn(wrk, hdl, scaret);
809 137199
                }
810
811
                // return leases which we did not use if error (break)
812 218210
                VSCARAB_FOREACH_RESUME(vio, scarab) {
813 2184
                        if (scaret->used == scaret->capacity)
814 0
                                ObjVAIreturn(wrk, hdl, scaret);
815 2184
                        VSCARET_ADD(scaret, vio->lease);
816 2184
                }
817
818
                // we have now completed the scarab
819 216026
                VSCARAB_INIT(scarab, scarab->capacity);
820
821
#ifdef VAI_DBG
822
                if (wrk->vsl)
823
                        VSLb(wrk->vsl, SLT_Debug, "r %d nn %d uu %u",
824
                            r, nn, uu);
825
#endif
826
827
                // flush before blocking if we did not already
828 216026
                if (r == 0 && (nn == -ENOBUFS || nn == -EAGAIN) &&
829 213900
                    (uu & OBJ_ITER_FLUSH) == 0) {
830 3419
                        r = func(priv, OBJ_ITER_FLUSH, NULL, 0);
831 3419
                        if (scaret->used > 0)
832 0
                                ObjVAIreturn(wrk, hdl, scaret);
833 3419
                }
834
835 216026
                if (r == 0 && (nn == -ENOBUFS || nn == -EAGAIN)) {
836 52152
                        assert(scaret->used <= 1);
837 52152
                        sml_notify_wait(&sn);
838 52152
                }
839 163874
                else if (r == 0 && nn < 0)
840 639
                        r = -1;
841 216026
        } while (nn != 0 && r == 0);
842
843 97837
        if ((u & OBJ_ITER_END) == 0) {
844 30436
                r2 = func(priv, OBJ_ITER_END, NULL, 0);
845 30436
                if (r == 0)
846 29047
                        r = r2;
847 30436
        }
848
849 97837
        if (scaret->used > 0)
850 68148
                ObjVAIreturn(wrk, hdl, scaret);
851
852 97837
        ObjVAIfini(wrk, &hdl);
853 97837
        sml_notify_fini(&sn);
854
855 97837
        return (r);
856
}
857
858
/*--------------------------------------------------------------------
859
 */
860
861
static struct storage *
862 130553
objallocwithnuke(struct worker *wrk, const struct stevedore *stv, ssize_t size,
863
    int flags)
864
{
865 130553
        struct storage *st = NULL;
866
867 130553
        CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
868 130553
        CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
869
870 130553
        if (size > cache_param->fetch_maxchunksize) {
871 80
                if (!(flags & LESS_MEM_ALLOCED_IS_OK))
872 0
                        return (NULL);
873 80
                size = cache_param->fetch_maxchunksize;
874 80
        }
875
876 130553
        assert(size <= UINT_MAX);       /* field limit in struct storage */
877
878 130553
        do {
879
                /* try to allocate from it */
880 130873
                st = sml_stv_alloc(stv, size, flags);
881 130873
                if (st != NULL)
882 130354
                        break;
883
884
                /* no luck; try to free some space and keep trying */
885 519
                if (stv->lru == NULL)
886 0
                        break;
887 519
        } while (LRU_NukeOne(wrk, stv->lru));
888
889 130553
        CHECK_OBJ_ORNULL(st, STORAGE_MAGIC);
890 130553
        return (st);
891 130553
}
892
893
static int v_matchproto_(objgetspace_f)
894 2323448
sml_getspace(struct worker *wrk, struct objcore *oc, ssize_t *sz,
895
    uint8_t **ptr)
896
{
897
        struct object *o;
898
        struct storage *st;
899
900 2323448
        CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
901 2323448
        CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
902 2323448
        CHECK_OBJ_NOTNULL(oc->boc, BOC_MAGIC);
903 2323448
        AN(sz);
904 2323448
        AN(ptr);
905 2323448
        if (*sz == 0)
906 2178513
                *sz = cache_param->fetch_chunksize;
907 2323448
        assert(*sz > 0);
908 2323448
        if (oc->boc->transit_buffer > 0)
909 19794
                *sz = vmin_t(ssize_t, *sz, oc->boc->transit_buffer);
910
911 2323448
        o = sml_getobj(wrk, oc);
912 2323448
        CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
913 2323448
        CHECK_OBJ_NOTNULL(oc->boc, BOC_MAGIC);
914
915 2323448
        st = VTAILQ_FIRST(&o->list);
916 2323448
        if (st != NULL && st->len < st->space) {
917 2209975
                *sz = st->space - st->len;
918 2209975
                *ptr = st->ptr + st->len;
919 2209975
                assert (*sz > 0);
920 2209975
                return (1);
921
        }
922
923 113473
        st = objallocwithnuke(wrk, oc->stobj->stevedore, *sz,
924
            LESS_MEM_ALLOCED_IS_OK);
925 113473
        if (st == NULL)
926 199
                return (0);
927
928 113274
        CHECK_OBJ_NOTNULL(oc->boc, BOC_MAGIC);
929 113274
        Lck_Lock(&oc->boc->mtx);
930 113274
        VTAILQ_INSERT_HEAD(&o->list, st, list);
931 113274
        Lck_Unlock(&oc->boc->mtx);
932
933 113274
        *sz = st->space - st->len;
934 113274
        assert (*sz > 0);
935 113274
        *ptr = st->ptr + st->len;
936 113274
        return (1);
937 2323448
}
938
939
static void v_matchproto_(objextend_f)
940 2265047
sml_extend(struct worker *wrk, struct objcore *oc, ssize_t l)
941
{
942
        struct object *o;
943
        struct storage *st;
944
945 2265047
        CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
946 2265047
        assert(l > 0);
947
948 2265047
        o = sml_getobj(wrk, oc);
949 2265047
        CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
950 2265047
        st = VTAILQ_FIRST(&o->list);
951 2265047
        CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC);
952 2265047
        assert(st->len + l <= st->space);
953 2265047
        st->len += l;
954 2265047
}
955
956
static void v_matchproto_(objtrimstore_f)
957 76161
sml_trimstore(struct worker *wrk, struct objcore *oc)
958
{
959
        const struct stevedore *stv;
960
        struct storage *st, *st1;
961
        struct object *o;
962
963 76161
        CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
964 76161
        CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
965 76161
        CHECK_OBJ_NOTNULL(oc->boc, BOC_MAGIC);
966
967 76161
        stv = oc->stobj->stevedore;
968 76161
        CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
969
970 76161
        if (oc->boc->stevedore_priv != NULL)
971 0
                WRONG("sml_trimstore already called");
972 76161
        oc->boc->stevedore_priv = trim_once;
973
974 76161
        if (stv->sml_free == NULL)
975 0
                return;
976
977 76161
        o = sml_getobj(wrk, oc);
978 76161
        CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
979 76161
        st = VTAILQ_FIRST(&o->list);
980
981 76161
        if (st == NULL)
982 0
                return;
983
984 76161
        if (st->len == 0) {
985 320
                Lck_Lock(&oc->boc->mtx);
986 320
                VTAILQ_REMOVE(&o->list, st, list);
987 320
                Lck_Unlock(&oc->boc->mtx);
988
                /* sml_bocdone frees this */
989 320
                oc->boc->stevedore_priv = st;
990 320
                return;
991
        }
992
993 75841
        if (st->space - st->len < 512)
994 67959
                return;
995
996 7882
        st1 = sml_stv_alloc(stv, st->len, 0);
997 7882
        if (st1 == NULL)
998 0
                return;
999 7882
        assert(st1->space >= st->len);
1000
1001 7882
        memcpy(st1->ptr, st->ptr, st->len);
1002 7882
        st1->len = st->len;
1003 7882
        Lck_Lock(&oc->boc->mtx);
1004 7882
        VTAILQ_REMOVE(&o->list, st, list);
1005 7882
        VTAILQ_INSERT_HEAD(&o->list, st1, list);
1006 7882
        Lck_Unlock(&oc->boc->mtx);
1007
        /* sml_bocdone frees this */
1008 7882
        oc->boc->stevedore_priv = st;
1009 76161
}
1010
1011
static void v_matchproto_(objbocdone_f)
1012 114797
sml_bocdone(struct worker *wrk, struct objcore *oc, struct boc *boc)
1013
{
1014
        const struct stevedore *stv;
1015
1016 114797
        CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
1017 114797
        CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
1018 114797
        CHECK_OBJ_NOTNULL(boc, BOC_MAGIC);
1019 114797
        stv = oc->stobj->stevedore;
1020 114797
        CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
1021
1022 114797
        sml_bocfini(stv, boc);
1023
1024 114797
        if (stv->lru != NULL) {
1025 113230
                if (isnan(wrk->lastused))
1026 0
                        wrk->lastused = VTIM_real();
1027 113230
                LRU_Add(oc, wrk->lastused);     // approx timestamp is OK
1028 113230
        }
1029 114797
}
1030
1031
static const void * v_matchproto_(objgetattr_f)
1032 904903
sml_getattr(struct worker *wrk, struct objcore *oc, enum obj_attr attr,
1033
   ssize_t *len)
1034
{
1035
        struct object *o;
1036
        ssize_t dummy;
1037
1038 904903
        CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
1039
1040 904903
        if (len == NULL)
1041 554136
                len = &dummy;
1042 904903
        o = sml_getobj(wrk, oc);
1043 904903
        CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
1044
1045 904903
        switch (attr) {
1046
                /* Fixed size attributes */
1047
#define OBJ_FIXATTR(U, l, s)                                            \
1048
        case OA_##U:                                                    \
1049
                *len = sizeof o->fa_##l;                                \
1050
                return (o->fa_##l);
1051
#include "tbl/obj_attr.h"
1052
1053
                /* Variable size attributes */
1054
#define OBJ_VARATTR(U, l)                                               \
1055
        case OA_##U:                                                    \
1056
                if (o->va_##l == NULL)                                  \
1057
                        return (NULL);                                  \
1058
                *len = o->va_##l##_len;                                 \
1059
                return (o->va_##l);
1060
#include "tbl/obj_attr.h"
1061
1062
                /* Auxiliary attributes */
1063
#define OBJ_AUXATTR(U, l)                                               \
1064
        case OA_##U:                                                    \
1065
                if (o->aa_##l == NULL)                                  \
1066
                        return (NULL);                                  \
1067
                CHECK_OBJ_NOTNULL(o->aa_##l, STORAGE_MAGIC);            \
1068
                *len = o->aa_##l->len;                                  \
1069
                return (o->aa_##l->ptr);
1070
#include "tbl/obj_attr.h"
1071
1072
        default:
1073
                break;
1074
        }
1075 0
        WRONG("Unsupported OBJ_ATTR");
1076 904903
}
1077
1078
static void * v_matchproto_(objsetattr_f)
1079 428194
sml_setattr(struct worker *wrk, struct objcore *oc, enum obj_attr attr,
1080
    ssize_t len, const void *ptr)
1081
{
1082
        struct object *o;
1083 428194
        void *retval = NULL;
1084
        struct storage *st;
1085
1086 428194
        CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
1087
1088 428194
        o = sml_getobj(wrk, oc);
1089 428194
        CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
1090 428194
        st = o->objstore;
1091
1092 428194
        switch (attr) {
1093
                /* Fixed size attributes */
1094
#define OBJ_FIXATTR(U, l, s)                                            \
1095
        case OA_##U:                                                    \
1096
                assert(len == sizeof o->fa_##l);                        \
1097
                retval = o->fa_##l;                                     \
1098
                break;
1099
#include "tbl/obj_attr.h"
1100
1101
                /* Variable size attributes */
1102
#define OBJ_VARATTR(U, l)                                               \
1103
        case OA_##U:                                                    \
1104
                if (o->va_##l##_len > 0) {                              \
1105
                        AN(o->va_##l);                                  \
1106
                        assert(len == o->va_##l##_len);                 \
1107
                        retval = o->va_##l;                             \
1108
                } else if (len > 0) {                                   \
1109
                        assert(len <= UINT_MAX);                        \
1110
                        assert(st->len + len <= st->space);             \
1111
                        o->va_##l = st->ptr + st->len;                  \
1112
                        st->len += len;                                 \
1113
                        o->va_##l##_len = len;                          \
1114
                        retval = o->va_##l;                             \
1115
                }                                                       \
1116
                break;
1117
#include "tbl/obj_attr.h"
1118
1119
                /* Auxiliary attributes */
1120
#define OBJ_AUXATTR(U, l)                                               \
1121
        case OA_##U:                                                    \
1122
                if (o->aa_##l != NULL) {                                \
1123
                        CHECK_OBJ_NOTNULL(o->aa_##l, STORAGE_MAGIC);    \
1124
                        assert(len == o->aa_##l->len);                  \
1125
                        retval = o->aa_##l->ptr;                        \
1126
                        break;                                          \
1127
                }                                                       \
1128
                if (len == 0)                                           \
1129
                        break;                                          \
1130
                o->aa_##l = objallocwithnuke(wrk, oc->stobj->stevedore, \
1131
                    len, 0);                                            \
1132
                if (o->aa_##l == NULL)                                  \
1133
                        break;                                          \
1134
                CHECK_OBJ_NOTNULL(o->aa_##l, STORAGE_MAGIC);            \
1135
                assert(len <= o->aa_##l->space);                        \
1136
                o->aa_##l->len = len;                                   \
1137
                retval = o->aa_##l->ptr;                                \
1138
                break;
1139
#include "tbl/obj_attr.h"
1140
1141
        default:
1142 0
                WRONG("Unsupported OBJ_ATTR");
1143
                break;
1144
        }
1145
1146 428194
        if (retval != NULL && ptr != NULL)
1147 15600
                memcpy(retval, ptr, len);
1148 428194
        return (retval);
1149
}
1150
1151
const struct obj_methods SML_methods = {
1152
        .objfree        = sml_objfree,
1153
        .objiterator    = sml_iterator,
1154
        .objgetspace    = sml_getspace,
1155
        .objextend      = sml_extend,
1156
        .objtrimstore   = sml_trimstore,
1157
        .objbocdone     = sml_bocdone,
1158
        .objslim        = sml_slim,
1159
        .objgetattr     = sml_getattr,
1160
        .objsetattr     = sml_setattr,
1161
        .objtouch       = LRU_Touch,
1162
        .vai_init       = sml_ai_init
1163
};
1164
1165
static void
1166 160
sml_panic_st(struct vsb *vsb, const char *hd, const struct storage *st)
1167
{
1168 320
        VSB_printf(vsb, "%s = %p {priv=%p, ptr=%p, len=%u, space=%u},\n",
1169 160
            hd, st, st->priv, st->ptr, st->len, st->space);
1170 160
}
1171
1172
void
1173 80
SML_panic(struct vsb *vsb, const struct objcore *oc)
1174
{
1175
        struct object *o;
1176
        struct storage *st;
1177
1178 80
        VSB_printf(vsb, "Simple = %p,\n", oc->stobj->priv);
1179 80
        if (oc->stobj->priv == NULL)
1180 0
                return;
1181 80
        o = oc->stobj->priv;
1182 80
        PAN_CheckMagic(vsb, o, OBJECT_MAGIC);
1183 80
        sml_panic_st(vsb, "Obj", o->objstore);
1184
1185
#define OBJ_FIXATTR(U, l, sz) \
1186
        VSB_printf(vsb, "%s = ", #U); \
1187
        VSB_quote(vsb, (const void*)o->fa_##l, sz, VSB_QUOTE_HEX); \
1188
        VSB_printf(vsb, ",\n");
1189
1190
#define OBJ_VARATTR(U, l) \
1191
        VSB_printf(vsb, "%s = {len=%u, ptr=%p},\n", \
1192
            #U, o->va_##l##_len, o->va_##l);
1193
1194
#define OBJ_AUXATTR(U, l)                                               \
1195
        do {                                                            \
1196
                if (o->aa_##l != NULL) sml_panic_st(vsb, #U, o->aa_##l);\
1197
        } while(0);
1198
1199
#include "tbl/obj_attr.h"
1200
1201 160
        VTAILQ_FOREACH(st, &o->list, list) {
1202 80
                sml_panic_st(vsb, "Body", st);
1203 80
        }
1204
}