varnish-cache/bin/varnishd/cache/cache_gzip.c
0
/*-
1
 * Copyright (c) 2013-2015 Varnish Software AS
2
 * All rights reserved.
3
 *
4
 * Author: Poul-Henning Kamp <phk@phk.freebsd.dk>
5
 *
6
 * SPDX-License-Identifier: BSD-2-Clause
7
 *
8
 * Redistribution and use in source and binary forms, with or without
9
 * modification, are permitted provided that the following conditions
10
 * are met:
11
 * 1. Redistributions of source code must retain the above copyright
12
 *    notice, this list of conditions and the following disclaimer.
13
 * 2. Redistributions in binary form must reproduce the above copyright
14
 *    notice, this list of conditions and the following disclaimer in the
15
 *    documentation and/or other materials provided with the distribution.
16
 *
17
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20
 * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
21
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27
 * SUCH DAMAGE.
28
 *
29
 * Interaction with the libvgz (zlib) library.
30
 *
31
 * The zlib library pollutes namespace a LOT when you include the "vgz.h"
32
 * (aka "zlib.h") file so we contain the damage by vectoring all access
33
 * to libz through this source file.
34
 *
35
 * The API defined by this file, will also insulate the rest of the code,
36
 * should we find a better gzip library at a later date.
37
 *
38
 */
39
40
#include "config.h"
41
42
#include <stdlib.h>
43
44
#include "cache_varnishd.h"
45
#include "cache_filter.h"
46
#include "cache_objhead.h"
47
#include "cache_vgz.h"
48
49
#include "storage/storage.h"
50
51
#include "vend.h"
52
53
#include "vgz.h"
54
55
struct vgz {
56
        unsigned                magic;
57
#define VGZ_MAGIC               0x162df0cb
58
        enum {VGZ_GZ,VGZ_UN}    dir;
59
        struct vsl_log          *vsl;
60
        const char              *id;
61
        int                     last_i;
62
        enum vgz_flag           flag;
63
64
        struct stv_buffer       *stvbuf;
65
        char                    *m_buf;
66
        ssize_t                 m_sz;
67
        ssize_t                 m_len;
68
69
        intmax_t                bits;
70
71
        z_stream                vz;
72
};
73
74
static const char *
75 6
vgz_msg(const struct vgz *vg)
76
{
77 6
        CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
78 6
        return (vg->vz.msg ? vg->vz.msg : "(null)");
79
}
80
81
/*--------------------------------------------------------------------
82
 * Set up a gunzip instance
83
 */
84
85
static struct vgz *
86 248
vgz_gunzip(struct vsl_log *vsl, const char *id)
87
{
88
        struct vgz *vg;
89
90 248
        ALLOC_OBJ(vg, VGZ_MAGIC);
91 248
        AN(vg);
92 248
        vg->vsl = vsl;
93 248
        vg->id = id;
94 248
        vg->dir = VGZ_UN;
95
96
        /*
97
         * Max memory usage according to zonf.h:
98
         *      mem_needed = "a few kb" + (1 << (windowBits))
99
         * Since we don't control windowBits, we have to assume
100
         * it is 15, so 34-35KB or so.
101
         */
102 248
        assert(Z_OK == inflateInit2(&vg->vz, 31));
103 248
        return (vg);
104
}
105
106
static struct vgz *
107 217
VGZ_NewGunzip(struct vsl_log *vsl, const char *id)
108
{
109 217
        VSC_C_main->n_gunzip++;
110 217
        return (vgz_gunzip(vsl, id));
111
}
112
113
static struct vgz *
114 31
VGZ_NewTestGunzip(struct vsl_log *vsl, const char *id)
115
{
116 31
        VSC_C_main->n_test_gunzip++;
117 31
        return (vgz_gunzip(vsl, id));
118
}
119
120
struct vgz *
121 92
VGZ_NewGzip(struct vsl_log *vsl, const char *id)
122
{
123
        struct vgz *vg;
124
        int i;
125
126 92
        VSC_C_main->n_gzip++;
127 92
        ALLOC_OBJ(vg, VGZ_MAGIC);
128 92
        AN(vg);
129 92
        vg->vsl = vsl;
130 92
        vg->id = id;
131 92
        vg->dir = VGZ_GZ;
132
133
        /*
134
         * From zconf.h:
135
         *
136
         *      mem_needed = "a few kb"
137
         *              + (1 << (windowBits+2))
138
         *              +  (1 << (memLevel+9))
139
         *
140
         * windowBits [8..15] (-> 1K..128K)
141
         * memLevel [1..9] (-> 1K->256K)
142
         */
143 92
        i = deflateInit2(&vg->vz,
144
            cache_param->gzip_level,            /* Level */
145
            Z_DEFLATED,                         /* Method */
146
            16 + 15,                            /* Window bits (16=gzip) */
147
            cache_param->gzip_memlevel,         /* memLevel */
148
            Z_DEFAULT_STRATEGY);
149 92
        assert(Z_OK == i);
150 92
        return (vg);
151
}
152
153
/*--------------------------------------------------------------------
154
 */
155
156
static int
157 264
vgz_getmbuf(struct worker *wrk, struct vgz *vg)
158
{
159
        size_t sz;
160
161 264
        CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
162 264
        CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
163 264
        AZ(vg->m_sz);
164 264
        AZ(vg->m_len);
165 264
        AZ(vg->m_buf);
166 264
        AZ(vg->stvbuf);
167
168 264
        vg->stvbuf = STV_AllocBuf(wrk, stv_transient, cache_param->gzip_buffer);
169 264
        if (vg->stvbuf == NULL)
170 0
                return (-1);
171 264
        vg->m_buf = STV_GetBufPtr(vg->stvbuf, &sz);
172 264
        vg->m_sz = sz;
173 264
        AN(vg->m_buf);
174 264
        assert(vg->m_sz > 0);
175 264
        return (0);
176 264
}
177
178
/*--------------------------------------------------------------------*/
179
180
void
181 1690
VGZ_Ibuf(struct vgz *vg, const void *ptr, ssize_t len)
182
{
183
184 1690
        CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
185 1690
        AZ(vg->vz.avail_in);
186 1690
        assert(len >= 0);
187 1690
        if (len > 0)
188 1138
                AN(ptr);
189
190 1690
        vg->vz.next_in = TRUST_ME(ptr);
191 1690
        vg->vz.avail_in = len;
192 1690
}
193
194
int
195 3813
VGZ_IbufEmpty(const struct vgz *vg)
196
{
197
198 3813
        CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
199 3813
        return (vg->vz.avail_in == 0);
200
}
201
202
/*--------------------------------------------------------------------*/
203
204
void
205 1813
VGZ_Obuf(struct vgz *vg, void *ptr, ssize_t len)
206
{
207
208 1813
        CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
209 1813
        AN(ptr);
210 1813
        assert(len > 0);
211
212 1813
        vg->vz.next_out = ptr;
213 1813
        vg->vz.avail_out = len;
214 1813
}
215
216
int
217 1083
VGZ_ObufFull(const struct vgz *vg)
218
{
219
220 1083
        CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
221 1083
        return (vg->vz.avail_out == 0);
222
}
223
224
/*--------------------------------------------------------------------*/
225
226
static enum vgzret_e
227 826
VGZ_Gunzip(struct vgz *vg, const void **pptr, ssize_t *plen)
228
{
229
        int i;
230
        ssize_t l;
231
        const uint8_t *before;
232
233 826
        CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
234
235 826
        *pptr = NULL;
236 826
        *plen = 0;
237 826
        AN(vg->vz.next_out);
238 826
        AN(vg->vz.avail_out);
239 826
        before = vg->vz.next_out;
240 826
        i = inflate(&vg->vz, 0);
241 826
        if (i == Z_OK || i == Z_STREAM_END) {
242 822
                *pptr = before;
243 822
                l = (const uint8_t *)vg->vz.next_out - before;
244 822
                *plen = l;
245 822
        }
246 826
        vg->last_i = i;
247 826
        if (i == Z_OK)
248 573
                return (VGZ_OK);
249 253
        if (i == Z_STREAM_END)
250 249
                return (VGZ_END);
251 4
        if (i == Z_BUF_ERROR)
252 2
                return (VGZ_STUCK);
253 2
        VSLb(vg->vsl, SLT_Gzip, "Gunzip error: %d (%s)", i, vgz_msg(vg));
254 2
        return (VGZ_ERROR);
255 826
}
256
257
/* set vz pointers for in and out iovecs */
258
static inline void
259 16
vgz_iovec_update(struct vgz *vg, const struct iovec *in, const struct iovec *buf)
260
{
261
        /* in: either fully consumed or the same */
262 16
        assert(vg->vz.avail_in == 0 || vg->vz.next_in == TRUST_ME(in->iov_base));
263 16
        vg->vz.next_in = TRUST_ME(in->iov_base);
264 16
        vg->vz.avail_in = in->iov_len;
265 16
        vg->vz.next_out = TRUST_ME(buf->iov_base);
266 16
        vg->vz.avail_out = buf->iov_len;
267 16
}
268
269
static enum vgzret_e
270 16
vgz_gunzip_iovec(struct vgz *vg, struct iovec *in, struct iovec *buf, struct iovec *out)
271
{
272
        int i;
273
274 16
        CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
275 16
        AN(in && buf && out);
276 16
        vgz_iovec_update(vg, in, buf);
277
278 16
        i = inflate(&vg->vz, 0);
279 16
        if (i == Z_OK || i == Z_STREAM_END) {
280 16
                iovec_collect(buf, out, pdiff(buf->iov_base, vg->vz.next_out));
281 16
                in->iov_base = TRUST_ME(vg->vz.next_in);
282 16
                in->iov_len = vg->vz.avail_in;
283 16
        }
284 16
        vg->last_i = i;
285 16
        if (i == Z_OK)
286 0
                return (VGZ_OK);
287 16
        if (i == Z_STREAM_END)
288 16
                return (VGZ_END);
289 0
        if (i == Z_BUF_ERROR)
290 0
                return (VGZ_STUCK);
291 0
        VSLb(vg->vsl, SLT_Gzip, "Gunzip error: %d (%s)", i, vgz_msg(vg));
292 0
        return (VGZ_ERROR);
293 16
}
294
295
/*--------------------------------------------------------------------*/
296
297
enum vgzret_e
298 1150
VGZ_Gzip(struct vgz *vg, const void **pptr, ssize_t *plen, enum vgz_flag flags)
299
{
300
        int i;
301
        int zflg;
302
        ssize_t l;
303
        const uint8_t *before;
304
305 1150
        CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
306
307 1150
        *pptr = NULL;
308 1150
        *plen = 0;
309 1150
        AN(vg->vz.next_out);
310 1150
        AN(vg->vz.avail_out);
311 1150
        before = vg->vz.next_out;
312 1150
        switch (flags) {
313 762
        case VGZ_NORMAL:        zflg = Z_NO_FLUSH; break;
314 125
        case VGZ_ALIGN:         zflg = Z_SYNC_FLUSH; break;
315 126
        case VGZ_RESET:         zflg = Z_FULL_FLUSH; break;
316 137
        case VGZ_FINISH:        zflg = Z_FINISH; break;
317 0
        default:                WRONG("Invalid VGZ flag");
318 0
        }
319 1150
        i = deflate(&vg->vz, zflg);
320 1150
        if (i == Z_OK || i == Z_STREAM_END) {
321 1150
                *pptr = before;
322 1150
                l = (const uint8_t *)vg->vz.next_out - before;
323 1150
                *plen = l;
324 1150
        }
325 1150
        vg->last_i = i;
326 1150
        if (i == Z_OK)
327 1063
                return (VGZ_OK);
328 87
        if (i == Z_STREAM_END)
329 87
                return (VGZ_END);
330 0
        if (i == Z_BUF_ERROR)
331 0
                return (VGZ_STUCK);
332 0
        VSLb(vg->vsl, SLT_Gzip, "Gzip error: %d (%s)", i, vgz_msg(vg));
333 0
        return (VGZ_ERROR);
334 1150
}
335
336
/*--------------------------------------------------------------------
337
 * VDP for gunzip'ing
338
 */
339
340
// common for traditional interface and vdpio
341
static int vdp_gunzip_init_common(struct vdp_ctx *vdc);
342
343
static int v_matchproto_(vdp_init_f)
344 92
vdp_gunzip_init(VRT_CTX, struct vdp_ctx *vdc, void **priv)
345
{
346
        struct vgz *vg;
347
348 92
        CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
349 92
        CHECK_OBJ_NOTNULL(vdc, VDP_CTX_MAGIC);
350 92
        CHECK_OBJ_ORNULL(vdc->oc, OBJCORE_MAGIC);
351 92
        CHECK_OBJ_NOTNULL(vdc->hp, HTTP_MAGIC);
352 92
        AN(vdc->clen);
353 92
        AN(priv);
354
355 92
        vg = VGZ_NewGunzip(vdc->vsl, "U D -");
356 92
        AN(vg);
357 92
        if (vgz_getmbuf(vdc->wrk, vg)) {
358 0
                (void)VGZ_Destroy(vdc->wrk, &vg);
359 0
                return (-1);
360
        }
361
362 92
        VGZ_Obuf(vg, vg->m_buf, vg->m_sz);
363 92
        *priv = vg;
364 92
        return (vdp_gunzip_init_common(vdc));
365 92
}
366
367
static int
368 92
vdp_gunzip_init_common(struct vdp_ctx *vdc)
369
{
370
        struct boc *boc;
371
        enum boc_state_e bos;
372
        const char *p;
373
        ssize_t dl;
374
        uint64_t u;
375 92
        http_Unset(vdc->hp, H_Content_Encoding);
376
377 92
        *vdc->clen = -1;
378
379 92
        if (vdc->oc == NULL)
380 17
                return (0);
381
382 75
        boc = HSH_RefBoc(vdc->oc);
383 75
        if (boc != NULL) {
384 12
                CHECK_OBJ(boc, BOC_MAGIC);
385 12
                bos = boc->state;
386 12
                HSH_DerefBoc(vdc->wrk, vdc->oc);
387 12
                if (bos < BOS_FINISHED)
388 12
                        return (0); /* OA_GZIPBITS is not stable yet */
389 0
        }
390
391 63
        p = ObjGetAttr(vdc->wrk, vdc->oc, OA_GZIPBITS, &dl);
392 63
        if (p != NULL && dl == 32) {
393 63
                u = vbe64dec(p + 24);
394 63
                if (u != 0)
395 62
                        *vdc->clen = u;
396 63
        }
397 63
        return (0);
398 92
}
399
400
static int v_matchproto_(vdp_fini_f)
401 76
vdp_gunzip_fini(struct vdp_ctx *vdc, void **priv)
402
{
403
        struct vgz *vg;
404
405 76
        (void)vdc;
406 76
        TAKE_OBJ_NOTNULL(vg, priv, VGZ_MAGIC);
407 76
        AN(vg->m_buf);
408 76
        (void)VGZ_Destroy(vdc->wrk, &vg);
409 76
        return (0);
410
}
411
412
static int v_matchproto_(vdp_bytes_f)
413 435
vdp_gunzip_bytes(struct vdp_ctx *vdc, enum vdp_action act, void **priv,
414
    const void *ptr, ssize_t len)
415
{
416
        enum vgzret_e vr;
417
        ssize_t dl;
418
        const void *dp;
419
        struct worker *wrk;
420
        struct vgz *vg;
421
422 435
        CHECK_OBJ_NOTNULL(vdc, VDP_CTX_MAGIC);
423 435
        wrk = vdc->wrk;
424 435
        CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
425 435
        (void)act;
426
427 435
        CAST_OBJ_NOTNULL(vg, *priv, VGZ_MAGIC);
428 435
        AN(vg->m_buf);
429
430 435
        if (len == 0)
431 117
                return (0);
432
433 318
        VGZ_Ibuf(vg, ptr, len);
434 318
        do {
435 318
                vr = VGZ_Gunzip(vg, &dp, &dl);
436 318
                if (vr == VGZ_END && !VGZ_IbufEmpty(vg)) {
437 2
                        VSLb(vg->vsl, SLT_Gzip, "G(un)zip error: %d (%s)",
438 1
                             vr, "junk after VGZ_END");
439 1
                        return (-1);
440
                }
441 317
                vg->m_len += dl;
442 317
                if (vr < VGZ_OK)
443 0
                        return (-1);
444
                // END or STUCK
445 317
                if (vg->m_len == vg->m_sz || vr != VGZ_OK) {
446 146
                        if (VDP_bytes(vdc, vr == VGZ_END ? VDP_END : VDP_FLUSH,
447 73
                            vg->m_buf, vg->m_len))
448 4
                                return (vdc->retval);
449 69
                        vg->m_len = 0;
450 69
                        VGZ_Obuf(vg, vg->m_buf, vg->m_sz);
451 69
                }
452 313
        } while (!VGZ_IbufEmpty(vg));
453 313
        assert(vr == VGZ_STUCK || vr == VGZ_OK || vr == VGZ_END);
454 313
        return (0);
455 435
}
456
457
#ifdef LATER
458
/*
459
 * XXX does it make sense to work on more than one buffer?
460
 */
461
static int v_matchproto_(vdpio_init_f)
462
vdpio_gunzip_init(VRT_CTX, struct vdp_ctx *vdc, void **priv, int capacity)
463
{
464
        struct vgz *vg;
465
        struct vscarab *in;
466
467
        CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
468
        CHECK_OBJ_NOTNULL(vdc, VDP_CTX_MAGIC);
469
        CHECK_OBJ_ORNULL(vdc->oc, OBJCORE_MAGIC);
470
        CHECK_OBJ_NOTNULL(vdc->hp, HTTP_MAGIC);
471
        AN(vdc->clen);
472
        AN(priv);
473
        assert(capacity >= 1);
474
475
        if (capacity < 4)
476
                capacity = 4;
477
478
        in = WS_Alloc(ctx->ws, VSCARAB_SIZE(capacity));
479
        if (in == NULL)
480
                return (-1);
481
482
        vg = VGZ_NewGunzip(vdc->vsl, "U D -");
483
        if (vg == NULL)
484
                return (-1);
485
486
        AZ(vg->m_buf);
487
        vg->m_buf = (void *)in;
488
489
        *priv = vg;
490
        AZ(vdp_gunzip_init_common(vdc));
491
        return (1);
492
}
493
#endif
494
495
static int v_matchproto_(vdpio_init_f)
496 16
vdpio_gunzip_upgrade(VRT_CTX, struct vdp_ctx *vdc, void **priv, int capacity)
497
{
498
        struct vgz *vg;
499
        struct vscarab *in;
500
501 16
        CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
502 16
        CHECK_OBJ_NOTNULL(vdc, VDP_CTX_MAGIC);
503 16
        AN(priv);
504 16
        assert(capacity >= 1);
505
506 16
        if (capacity < 4)
507 16
                capacity = 4;
508
509
        /* done in vdp_gunzip_init */
510 16
        CAST_OBJ_NOTNULL(vg, *priv, VGZ_MAGIC);
511
512 16
        in = WS_Alloc(ctx->ws, VSCARAB_SIZE(capacity));
513 16
        if (in == NULL)
514 0
                return (-1);
515 16
        VSCARAB_INIT(in, capacity);
516
517
        // XXX duplicate work - remove when completing transition to VAI
518 16
        AN(vg->m_buf);
519 16
        AN(vg->stvbuf);
520 16
        STV_FreeBuf(vdc->wrk, &vg->stvbuf);
521 16
        vg->stvbuf = NULL;
522 16
        vg->m_buf = (void *)in;
523
524 16
        return (1);
525 16
}
526
527
static int v_matchproto_(vdpio_lease_f)
528 34
vdpio_gunzip_lease(struct vdp_ctx *vdc, struct vdp_entry *this, struct vscarab *out)
529
{
530
        struct vscarab *in;
531
        enum vgzret_e vr;
532
        struct vgz *vg;
533
        struct viov *v, *b, *o;
534
        int r;
535
536 34
        CHECK_OBJ_NOTNULL(vdc, VDP_CTX_MAGIC);
537 34
        CHECK_OBJ_NOTNULL(this, VDP_ENTRY_MAGIC);
538 34
        CAST_OBJ_NOTNULL(vg, this->priv, VGZ_MAGIC);
539 34
        CAST_OBJ_NOTNULL(in, (void*)vg->m_buf, VSCARAB_MAGIC);
540
541 34
        this->calls++;
542
543 34
        if (out->used == out->capacity)
544 16
                return (0);
545
546 18
        if (in->used < in->capacity && (in->flags & VSCARAB_F_END) == 0)
547 18
                r = vdpio_pull(vdc, this, in);
548
        else
549 0
                r = 0;
550
551 18
        if (in->used == 0) {
552 2
                out->flags = in->flags & VSCARAB_F_END;
553 2
                return (r);
554
        }
555
556
        // XXX more than one buffer?
557 16
        VSCARAB_LOCAL(buf, 1);
558 16
        b = VSCARAB_GET(buf);
559 16
        AN(b);
560 16
        b->iov.iov_len = cache_param->gzip_buffer;
561 16
        r = ObjVAIbuffer(vdc->wrk, vdc->vai_hdl, buf);
562 16
        if (r < 0)
563 0
                return (out->used ? 0 : r);
564
565 16
        o = VSCARAB_GET(out);
566 16
        AN(o);
567 16
        r = 0;
568
569 16
        VSCARAB_FOREACH(v, in) {
570 16
                this->bytes_in += v->iov.iov_len;
571 16
                vr = vgz_gunzip_iovec(vg, &v->iov, &b->iov, &o->iov);
572 16
                if (vr == VGZ_END && v->iov.iov_len > 0) {
573 0
                        VSLb(vg->vsl, SLT_Gzip, "G(un)zip error: %d (%s)",
574 0
                             vr, "junk after VGZ_END");
575 0
                        r = -EMSGSIZE;
576 0
                        break;
577
                }
578 16
                if (vr < VGZ_OK)
579 0
                        break;
580
581 16
                if (b->iov.iov_len == 0 || vr != VGZ_OK) {
582 16
                        r = 1;
583 16
                        break;
584
                }
585 0
        }
586
587 16
        if (r <= 0) {
588 0
                o->iov.iov_base = NULL;
589 0
                o->iov.iov_len = 0;
590 0
                vdpio_return_lease(vdc, b->lease);
591 0
                return (r);
592
        }
593
594 16
        o->lease = b->lease;
595 16
        b->lease = 0;
596
597 16
        vdpio_consolidate_vscarab(vdc, in);
598 16
        if (in->used == 0)
599 16
                out->flags = in->flags & VSCARAB_F_END;
600
601 16
        return (r);
602 34
}
603
604
static void v_matchproto_(vdpio_fini_f)
605 16
vdpio_gunzip_fini(struct vdp_ctx *vdc, void **priv)
606
{
607
        struct vscarab *in;
608
        struct vgz *vg;
609
610 16
        (void)vdc;
611 16
        TAKE_OBJ_NOTNULL(vg, priv, VGZ_MAGIC);
612 16
        CAST_OBJ_NOTNULL(in, (void *)vg->m_buf, VSCARAB_MAGIC);
613 16
        vg->m_buf = NULL;
614
615 16
        (void)VGZ_Destroy(vdc->wrk, &vg);
616 16
}
617
618
const struct vdp VDP_gunzip = {
619
        .name =         "gunzip",
620
        .init =         vdp_gunzip_init,
621
        .bytes =        vdp_gunzip_bytes,
622
        .fini =         vdp_gunzip_fini,
623
624
#ifdef LATER
625
        .io_init =      vdpio_gunzip_init,
626
#endif
627
        .io_upgrade =   vdpio_gunzip_upgrade,
628
        .io_lease =     vdpio_gunzip_lease,
629
        .io_fini =      vdpio_gunzip_fini,
630
};
631
632
/*--------------------------------------------------------------------*/
633
634
void
635 1268
VGZ_UpdateObj(const struct vfp_ctx *vc, struct vgz *vg, enum vgzret_e e)
636
{
637
        char *p;
638
        intmax_t ii;
639
640 1268
        CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
641 1268
        if (e < VGZ_OK)
642 0
                return;
643 1268
        ii = vg->vz.start_bit + vg->vz.last_bit + vg->vz.stop_bit;
644 1268
        if (e != VGZ_END && ii == vg->bits)
645 935
                return;
646 333
        vg->bits = ii;
647 333
        p = ObjSetAttr(vc->wrk, vc->oc, OA_GZIPBITS, 32, NULL);
648 333
        AN(p);
649 333
        vbe64enc(p, vg->vz.start_bit);
650 333
        vbe64enc(p + 8, vg->vz.last_bit);
651 333
        vbe64enc(p + 16, vg->vz.stop_bit);
652 333
        if (e != VGZ_END)
653 127
                return;
654 206
        if (vg->dir == VGZ_GZ)
655 174
                vbe64enc(p + 24, vg->vz.total_in);
656 206
        if (vg->dir == VGZ_UN)
657 32
                vbe64enc(p + 24, vg->vz.total_out);
658 1268
}
659
660
/*--------------------------------------------------------------------
661
 */
662
663
enum vgzret_e
664 340
VGZ_Destroy(struct worker *wrk, struct vgz **vgp)
665
{
666
        struct vgz *vg;
667
        enum vgzret_e vr;
668
        int i;
669
670 340
        CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
671 340
        TAKE_OBJ_NOTNULL(vg, vgp, VGZ_MAGIC);
672 340
        AN(vg->id);
673 680
        VSLb(vg->vsl, SLT_Gzip, "%s %jd %jd %jd %jd %jd",
674 340
            vg->id,
675 340
            (intmax_t)vg->vz.total_in,
676 340
            (intmax_t)vg->vz.total_out,
677 340
            (intmax_t)vg->vz.start_bit,
678 340
            (intmax_t)vg->vz.last_bit,
679 340
            (intmax_t)vg->vz.stop_bit);
680 340
        if (vg->dir == VGZ_GZ)
681 92
                i = deflateEnd(&vg->vz);
682
        else
683 248
                i = inflateEnd(&vg->vz);
684 340
        if (vg->last_i == Z_STREAM_END && i == Z_OK)
685 275
                i = Z_STREAM_END;
686 340
        if (vg->m_buf != NULL) {
687 248
                AN(vg->stvbuf);
688 248
                STV_FreeBuf(wrk, &vg->stvbuf);
689 248
        }
690 340
        AZ(vg->stvbuf);
691 340
        if (i == Z_OK)
692 63
                vr = VGZ_OK;
693 277
        else if (i == Z_STREAM_END)
694 275
                vr = VGZ_END;
695 2
        else if (i == Z_BUF_ERROR)
696 0
                vr = VGZ_STUCK;
697
        else {
698 4
                VSLb(vg->vsl, SLT_Gzip, "G(un)zip error: %d (%s)",
699 2
                    i, vgz_msg(vg));
700 2
                vr = VGZ_ERROR;
701
        }
702 340
        FREE_OBJ(vg);
703 340
        return (vr);
704
}
705
706
/*--------------------------------------------------------------------*/
707
708
static enum vfp_status v_matchproto_(vfp_init_f)
709 176
vfp_gzip_init(VRT_CTX, struct vfp_ctx *vc, struct vfp_entry *vfe)
710
{
711
        struct vgz *vg;
712
713 176
        CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
714 176
        CHECK_OBJ_NOTNULL(vc, VFP_CTX_MAGIC);
715 176
        CHECK_OBJ_NOTNULL(vfe, VFP_ENTRY_MAGIC);
716
717
        /*
718
         * G(un)zip makes no sence on partial responses, but since
719
         * it is an pure 1:1 transform, we can just ignore it.
720
         */
721 176
        if (http_GetStatus(vc->resp) == 206)
722 4
                return (VFP_NULL);
723
724 172
        if (vfe->vfp == &VFP_gzip) {
725 16
                if (http_GetHdr(vc->resp, H_Content_Encoding, NULL))
726 0
                        return (VFP_NULL);
727 16
                vg = VGZ_NewGzip(vc->wrk->vsl, vfe->vfp->priv1);
728 16
                vc->obj_flags |= OF_GZIPED | OF_CHGCE;
729 16
        } else {
730 156
                if (!http_HdrIs(vc->resp, H_Content_Encoding, "gzip"))
731 0
                        return (VFP_NULL);
732 156
                if (vfe->vfp == &VFP_gunzip) {
733 125
                        vg = VGZ_NewGunzip(vc->wrk->vsl, vfe->vfp->priv1);
734 125
                        vc->obj_flags &= ~OF_GZIPED;
735 125
                        vc->obj_flags |= OF_CHGCE;
736 125
                } else {
737 31
                        vg = VGZ_NewTestGunzip(vc->wrk->vsl, vfe->vfp->priv1);
738 31
                        vc->obj_flags |= OF_GZIPED;
739
                }
740
        }
741 172
        AN(vg);
742 172
        vfe->priv1 = vg;
743 172
        if (vgz_getmbuf(vc->wrk, vg))
744 0
                return (VFP_ERROR);
745 172
        VGZ_Ibuf(vg, vg->m_buf, 0);
746 172
        AZ(vg->m_len);
747
748 172
        if (vfe->vfp == &VFP_gunzip || vfe->vfp == &VFP_gzip) {
749 141
                http_Unset(vc->resp, H_Content_Encoding);
750 141
                http_Unset(vc->resp, H_Content_Length);
751 141
                RFC2616_Weaken_Etag(vc->resp);
752 141
        }
753
754 172
        if (vfe->vfp == &VFP_gzip)
755 16
                http_SetHeader(vc->resp, "Content-Encoding: gzip");
756
757 172
        if (vfe->vfp == &VFP_gzip || vfe->vfp == &VFP_testgunzip)
758 47
                RFC2616_Vary_AE(vc->resp);
759
760 172
        return (VFP_OK);
761 176
}
762
763
/*--------------------------------------------------------------------
764
 * VFP_GUNZIP
765
 *
766
 * A VFP for gunzip'ing an object as we receive it from the backend
767
 */
768
769
static enum vfp_status v_matchproto_(vfp_pull_f)
770 468
vfp_gunzip_pull(struct vfp_ctx *vc, struct vfp_entry *vfe, void *p,
771
    ssize_t *lp)
772
{
773
        ssize_t l;
774
        struct vgz *vg;
775 468
        enum vgzret_e vr = VGZ_ERROR;
776
        const void *dp;
777
        ssize_t dl;
778 468
        enum vfp_status vp = VFP_OK;
779
780 468
        CHECK_OBJ_NOTNULL(vc, VFP_CTX_MAGIC);
781 468
        CHECK_OBJ_NOTNULL(vfe, VFP_ENTRY_MAGIC);
782 468
        CAST_OBJ_NOTNULL(vg, vfe->priv1, VGZ_MAGIC);
783 468
        AN(p);
784 468
        AN(lp);
785 468
        l = *lp;
786 468
        *lp = 0;
787 468
        VGZ_Obuf(vg, p, l);
788 468
        do {
789 470
                if (VGZ_IbufEmpty(vg)) {
790 145
                        l = vg->m_sz;
791 145
                        vp = VFP_Suck(vc, vg->m_buf, &l);
792 145
                        if (vp == VFP_ERROR)
793 0
                                return (vp);
794 145
                        VGZ_Ibuf(vg, vg->m_buf, l);
795 145
                }
796 470
                if (!VGZ_IbufEmpty(vg) || vp == VFP_END) {
797 470
                        vr = VGZ_Gunzip(vg, &dp, &dl);
798 470
                        if (vr == VGZ_END && !VGZ_IbufEmpty(vg))
799 1
                                return (VFP_Error(vc, "Junk after gzip data"));
800 469
                        if (vr < VGZ_OK)
801 4
                                return (VFP_Error(vc,
802 2
                                    "Invalid Gzip data: %s", vgz_msg(vg)));
803 467
                        if (dl > 0) {
804 395
                                *lp = dl;
805 395
                                assert(dp == p);
806 395
                                return (VFP_OK);
807
                        }
808 72
                }
809 72
                AN(VGZ_IbufEmpty(vg));
810 72
        } while (vp == VFP_OK);
811 70
        if (vr != VGZ_END)
812 1
                return (VFP_Error(vc, "Gunzip error at the very end"));
813 69
        return (vp);
814 468
}
815
816
817
/*--------------------------------------------------------------------
818
 * VFP_GZIP
819
 *
820
 * A VFP for gzip'ing an object as we receive it from the backend
821
 */
822
823
static enum vfp_status v_matchproto_(vfp_pull_f)
824 63
vfp_gzip_pull(struct vfp_ctx *vc, struct vfp_entry *vfe, void *p,
825
    ssize_t *lp)
826
{
827
        ssize_t l;
828
        struct vgz *vg;
829 63
        enum vgzret_e vr = VGZ_ERROR;
830
        const void *dp;
831
        ssize_t dl;
832 63
        enum vfp_status vp = VFP_ERROR;
833
834 63
        CHECK_OBJ_NOTNULL(vc, VFP_CTX_MAGIC);
835 63
        CHECK_OBJ_NOTNULL(vfe, VFP_ENTRY_MAGIC);
836 63
        CAST_OBJ_NOTNULL(vg, vfe->priv1, VGZ_MAGIC);
837 63
        AN(p);
838 63
        AN(lp);
839 63
        l = *lp;
840 63
        *lp = 0;
841 63
        VGZ_Obuf(vg, p, l);
842 63
        do {
843 67
                if (VGZ_IbufEmpty(vg)) {
844 63
                        l = vg->m_sz;
845 63
                        vp = VFP_Suck(vc, vg->m_buf, &l);
846 63
                        if (vp == VFP_ERROR)
847 0
                                break;
848 63
                        if (vp == VFP_END)
849 22
                                vg->flag = VGZ_FINISH;
850 63
                        VGZ_Ibuf(vg, vg->m_buf, l);
851 63
                }
852 67
                if (!VGZ_IbufEmpty(vg) || vg->flag == VGZ_FINISH) {
853 67
                        vr = VGZ_Gzip(vg, &dp, &dl, vg->flag);
854 67
                        if (vr < VGZ_OK)
855 0
                                return (VFP_Error(vc, "Gzip failed"));
856 67
                        if (dl > 0) {
857 63
                                VGZ_UpdateObj(vc, vg, vr);
858 63
                                *lp = dl;
859 63
                                assert(dp == p);
860 63
                                if (vr != VGZ_END || !VGZ_IbufEmpty(vg))
861 48
                                        return (VFP_OK);
862 15
                        }
863 19
                }
864 19
                AN(VGZ_IbufEmpty(vg));
865 19
        } while (vg->flag != VGZ_FINISH);
866
867 15
        if (vr != VGZ_END)
868 0
                return (VFP_Error(vc, "Gzip failed"));
869 15
        VGZ_UpdateObj(vc, vg, VGZ_END);
870 15
        return (VFP_END);
871 63
}
872
873
/*--------------------------------------------------------------------
874
 * VFP_TESTGZIP
875
 *
876
 * A VFP for testing that received gzip data is valid, and for
877
 * collecting the magic bits while we're at it.
878
 */
879
880
static enum vfp_status v_matchproto_(vfp_pull_f)
881 40
vfp_testgunzip_pull(struct vfp_ctx *vc, struct vfp_entry *vfe, void *p,
882
    ssize_t *lp)
883
{
884
        struct vgz *vg;
885 40
        enum vgzret_e vr = VGZ_ERROR;
886
        const void *dp;
887
        ssize_t dl;
888
        enum vfp_status vp;
889
890 40
        CHECK_OBJ_NOTNULL(vc, VFP_CTX_MAGIC);
891 40
        CHECK_OBJ_NOTNULL(vfe, VFP_ENTRY_MAGIC);
892 40
        CAST_OBJ_NOTNULL(vg, vfe->priv1, VGZ_MAGIC);
893 40
        AN(p);
894 40
        AN(lp);
895 40
        vp = VFP_Suck(vc, p, lp);
896 40
        if (vp == VFP_ERROR)
897 2
                return (vp);
898 38
        if (*lp > 0 || vp == VFP_END) {
899 38
                VGZ_Ibuf(vg, p, *lp);
900 38
                do {
901 38
                        VGZ_Obuf(vg, vg->m_buf, vg->m_sz);
902 38
                        vr = VGZ_Gunzip(vg, &dp, &dl);
903 38
                        if (vr == VGZ_END && !VGZ_IbufEmpty(vg))
904 3
                                return (VFP_Error(vc, "Junk after gzip data"));
905 35
                        if (vr < VGZ_OK)
906 0
                                return (VFP_Error(vc,
907 0
                                    "Invalid Gzip data: %s", vgz_msg(vg)));
908 35
                } while (!VGZ_IbufEmpty(vg));
909 35
        }
910 35
        VGZ_UpdateObj(vc, vg, vr);
911 35
        if (vp == VFP_END && vr != VGZ_END)
912 1
                return (VFP_Error(vc, "tGunzip failed"));
913 34
        return (vp);
914 40
}
915
916
/*--------------------------------------------------------------------*/
917
918
static void v_matchproto_(vfp_fini_f)
919 187
vfp_gzip_fini(struct vfp_ctx *vc, struct vfp_entry *vfe)
920
{
921
        struct vgz *vg;
922
923 187
        CHECK_OBJ_NOTNULL(vc, VFP_CTX_MAGIC);
924 187
        CHECK_OBJ_NOTNULL(vfe, VFP_ENTRY_MAGIC);
925
926 187
        if (vfe->priv1 != NULL) {
927 172
                TAKE_OBJ_NOTNULL(vg, &vfe->priv1, VGZ_MAGIC);
928 172
                (void)VGZ_Destroy(vc->wrk, &vg);
929 172
        }
930 187
}
931
932
/*--------------------------------------------------------------------*/
933
934
const struct vfp VFP_gunzip = {
935
        .name = "gunzip",
936
        .init = vfp_gzip_init,
937
        .pull = vfp_gunzip_pull,
938
        .fini = vfp_gzip_fini,
939
        .priv1 = "U F -",
940
};
941
942
const struct vfp VFP_gzip = {
943
        .name = "gzip",
944
        .init = vfp_gzip_init,
945
        .pull = vfp_gzip_pull,
946
        .fini = vfp_gzip_fini,
947
        .priv1 = "G F -",
948
};
949
950
const struct vfp VFP_testgunzip = {
951
        .name = "testgunzip",
952
        .init = vfp_gzip_init,
953
        .pull = vfp_testgunzip_pull,
954
        .fini = vfp_gzip_fini,
955
        .priv1 = "u F -",
956
};