varnish-cache/bin/varnishd/cache/cache_gzip.c
0
/*-
1
 * Copyright (c) 2013-2015 Varnish Software AS
2
 * All rights reserved.
3
 *
4
 * Author: Poul-Henning Kamp <phk@phk.freebsd.dk>
5
 *
6
 * SPDX-License-Identifier: BSD-2-Clause
7
 *
8
 * Redistribution and use in source and binary forms, with or without
9
 * modification, are permitted provided that the following conditions
10
 * are met:
11
 * 1. Redistributions of source code must retain the above copyright
12
 *    notice, this list of conditions and the following disclaimer.
13
 * 2. Redistributions in binary form must reproduce the above copyright
14
 *    notice, this list of conditions and the following disclaimer in the
15
 *    documentation and/or other materials provided with the distribution.
16
 *
17
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20
 * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
21
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27
 * SUCH DAMAGE.
28
 *
29
 * Interaction with the libvgz (zlib) library.
30
 *
31
 * The zlib library pollutes namespace a LOT when you include the "vgz.h"
32
 * (aka "zlib.h") file so we contain the damage by vectoring all access
33
 * to libz through this source file.
34
 *
35
 * The API defined by this file, will also insulate the rest of the code,
36
 * should we find a better gzip library at a later date.
37
 *
38
 */
39
40
#include "config.h"
41
42
#include <stdlib.h>
43
44
#include "cache_varnishd.h"
45
#include "cache_filter.h"
46
#include "cache_objhead.h"
47
#include "cache_vgz.h"
48
49
#include "storage/storage.h"
50
51
#include "vend.h"
52
53
#include "vgz.h"
54
55
struct vgz {
56
        unsigned                magic;
57
#define VGZ_MAGIC               0x162df0cb
58
        enum {VGZ_GZ,VGZ_UN}    dir;
59
        struct vsl_log          *vsl;
60
        const char              *id;
61
        int                     last_i;
62
        enum vgz_flag           flag;
63
64
        struct stv_buffer       *stvbuf;
65
        char                    *m_buf;
66
        ssize_t                 m_sz;
67
        ssize_t                 m_len;
68
69
        intmax_t                bits;
70
71
        z_stream                vz;
72
};
73
74
static const char *
75 12
vgz_msg(const struct vgz *vg)
76
{
77 12
        CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
78 12
        return (vg->vz.msg ? vg->vz.msg : "(null)");
79
}
80
81
/*--------------------------------------------------------------------
82
 * Set up a gunzip instance
83
 */
84
85
static struct vgz *
86 512
vgz_gunzip(struct vsl_log *vsl, const char *id)
87
{
88
        struct vgz *vg;
89
90 512
        ALLOC_OBJ(vg, VGZ_MAGIC);
91 512
        AN(vg);
92 512
        vg->vsl = vsl;
93 512
        vg->id = id;
94 512
        vg->dir = VGZ_UN;
95
96
        /*
97
         * Max memory usage according to zonf.h:
98
         *      mem_needed = "a few kb" + (1 << (windowBits))
99
         * Since we don't control windowBits, we have to assume
100
         * it is 15, so 34-35KB or so.
101
         */
102 512
        assert(Z_OK == inflateInit2(&vg->vz, 31));
103 512
        return (vg);
104
}
105
106
static struct vgz *
107 450
VGZ_NewGunzip(struct vsl_log *vsl, const char *id)
108
{
109 450
        VSC_C_main->n_gunzip++;
110 450
        return (vgz_gunzip(vsl, id));
111
}
112
113
static struct vgz *
114 62
VGZ_NewTestGunzip(struct vsl_log *vsl, const char *id)
115
{
116 62
        VSC_C_main->n_test_gunzip++;
117 62
        return (vgz_gunzip(vsl, id));
118
}
119
120
struct vgz *
121 184
VGZ_NewGzip(struct vsl_log *vsl, const char *id)
122
{
123
        struct vgz *vg;
124
        int i;
125
126 184
        VSC_C_main->n_gzip++;
127 184
        ALLOC_OBJ(vg, VGZ_MAGIC);
128 184
        AN(vg);
129 184
        vg->vsl = vsl;
130 184
        vg->id = id;
131 184
        vg->dir = VGZ_GZ;
132
133
        /*
134
         * From zconf.h:
135
         *
136
         *      mem_needed = "a few kb"
137
         *              + (1 << (windowBits+2))
138
         *              +  (1 << (memLevel+9))
139
         *
140
         * windowBits [8..15] (-> 1K..128K)
141
         * memLevel [1..9] (-> 1K->256K)
142
         */
143 184
        i = deflateInit2(&vg->vz,
144
            cache_param->gzip_level,            /* Level */
145
            Z_DEFLATED,                         /* Method */
146
            16 + 15,                            /* Window bits (16=gzip) */
147
            cache_param->gzip_memlevel,         /* memLevel */
148
            Z_DEFAULT_STRATEGY);
149 184
        assert(Z_OK == i);
150 184
        return (vg);
151
}
152
153
/*--------------------------------------------------------------------
154
 */
155
156
static int
157 544
vgz_getmbuf(struct worker *wrk, struct vgz *vg)
158
{
159
        size_t sz;
160
161 544
        CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
162 544
        CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
163 544
        AZ(vg->m_sz);
164 544
        AZ(vg->m_len);
165 544
        AZ(vg->m_buf);
166 544
        AZ(vg->stvbuf);
167
168 544
        vg->stvbuf = STV_AllocBuf(wrk, stv_transient, cache_param->gzip_buffer);
169 544
        if (vg->stvbuf == NULL)
170 0
                return (-1);
171 544
        vg->m_buf = STV_GetBufPtr(vg->stvbuf, &sz);
172 544
        vg->m_sz = sz;
173 544
        AN(vg->m_buf);
174 544
        assert(vg->m_sz > 0);
175 544
        return (0);
176 544
}
177
178
/*--------------------------------------------------------------------*/
179
180
void
181 3364
VGZ_Ibuf(struct vgz *vg, const void *ptr, ssize_t len)
182
{
183
184 3364
        CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
185 3364
        AZ(vg->vz.avail_in);
186 3364
        assert(len >= 0);
187 3364
        if (len > 0)
188 2260
                AN(ptr);
189
190 3364
        vg->vz.next_in = TRUST_ME(ptr);
191 3364
        vg->vz.avail_in = len;
192 3364
}
193
194
int
195 7551
VGZ_IbufEmpty(const struct vgz *vg)
196
{
197
198 7551
        CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
199 7551
        return (vg->vz.avail_in == 0);
200
}
201
202
/*--------------------------------------------------------------------*/
203
204
void
205 3606
VGZ_Obuf(struct vgz *vg, void *ptr, ssize_t len)
206
{
207
208 3606
        CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
209 3606
        AN(ptr);
210 3606
        assert(len > 0);
211
212 3606
        vg->vz.next_out = ptr;
213 3606
        vg->vz.avail_out = len;
214 3606
}
215
216
int
217 2133
VGZ_ObufFull(const struct vgz *vg)
218
{
219
220 2133
        CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
221 2133
        return (vg->vz.avail_out == 0);
222
}
223
224
/*--------------------------------------------------------------------*/
225
226
static enum vgzret_e
227 1649
VGZ_Gunzip(struct vgz *vg, const void **pptr, ssize_t *plen)
228
{
229
        int i;
230
        ssize_t l;
231
        const uint8_t *before;
232
233 1649
        CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
234
235 1649
        *pptr = NULL;
236 1649
        *plen = 0;
237 1649
        AN(vg->vz.next_out);
238 1649
        AN(vg->vz.avail_out);
239 1649
        before = vg->vz.next_out;
240 1649
        i = inflate(&vg->vz, 0);
241 1649
        if (i == Z_OK || i == Z_STREAM_END) {
242 1641
                *pptr = before;
243 1641
                l = (const uint8_t *)vg->vz.next_out - before;
244 1641
                *plen = l;
245 1641
        }
246 1649
        vg->last_i = i;
247 1649
        if (i == Z_OK)
248 1127
                return (VGZ_OK);
249 522
        if (i == Z_STREAM_END)
250 514
                return (VGZ_END);
251 8
        if (i == Z_BUF_ERROR)
252 4
                return (VGZ_STUCK);
253 4
        VSLb(vg->vsl, SLT_Gzip, "Gunzip error: %d (%s)", i, vgz_msg(vg));
254 4
        return (VGZ_ERROR);
255 1649
}
256
257
/* set vz pointers for in and out iovecs */
258
static inline void
259 32
vgz_iovec_update(struct vgz *vg, const struct iovec *in, const struct iovec *buf)
260
{
261
        /* in: either fully consumed or the same */
262 32
        assert(vg->vz.avail_in == 0 || vg->vz.next_in == TRUST_ME(in->iov_base));
263 32
        vg->vz.next_in = TRUST_ME(in->iov_base);
264 32
        vg->vz.avail_in = in->iov_len;
265 32
        vg->vz.next_out = TRUST_ME(buf->iov_base);
266 32
        vg->vz.avail_out = buf->iov_len;
267 32
}
268
269
static enum vgzret_e
270 32
vgz_gunzip_iovec(struct vgz *vg, struct iovec *in, struct iovec *buf, struct iovec *out)
271
{
272
        int i;
273
274 32
        CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
275 32
        AN(in && buf && out);
276 32
        vgz_iovec_update(vg, in, buf);
277
278 32
        i = inflate(&vg->vz, 0);
279 32
        if (i == Z_OK || i == Z_STREAM_END) {
280 32
                iovec_collect(buf, out, pdiff(buf->iov_base, vg->vz.next_out));
281 32
                in->iov_base = TRUST_ME(vg->vz.next_in);
282 32
                in->iov_len = vg->vz.avail_in;
283 32
        }
284 32
        vg->last_i = i;
285 32
        if (i == Z_OK)
286 0
                return (VGZ_OK);
287 32
        if (i == Z_STREAM_END)
288 32
                return (VGZ_END);
289 0
        if (i == Z_BUF_ERROR)
290 0
                return (VGZ_STUCK);
291 0
        VSLb(vg->vsl, SLT_Gzip, "Gunzip error: %d (%s)", i, vgz_msg(vg));
292 0
        return (VGZ_ERROR);
293 32
}
294
295
/*--------------------------------------------------------------------*/
296
297
enum vgzret_e
298 2267
VGZ_Gzip(struct vgz *vg, const void **pptr, ssize_t *plen, enum vgz_flag flags)
299
{
300
        int i;
301
        int zflg;
302
        ssize_t l;
303
        const uint8_t *before;
304
305 2267
        CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
306
307 2267
        *pptr = NULL;
308 2267
        *plen = 0;
309 2267
        AN(vg->vz.next_out);
310 2267
        AN(vg->vz.avail_out);
311 2267
        before = vg->vz.next_out;
312 2267
        switch (flags) {
313 1491
        case VGZ_NORMAL:        zflg = Z_NO_FLUSH; break;
314 250
        case VGZ_ALIGN:         zflg = Z_SYNC_FLUSH; break;
315 252
        case VGZ_RESET:         zflg = Z_FULL_FLUSH; break;
316 274
        case VGZ_FINISH:        zflg = Z_FINISH; break;
317 0
        default:                WRONG("Invalid VGZ flag");
318 0
        }
319 2267
        i = deflate(&vg->vz, zflg);
320 2267
        if (i == Z_OK || i == Z_STREAM_END) {
321 2267
                *pptr = before;
322 2267
                l = (const uint8_t *)vg->vz.next_out - before;
323 2267
                *plen = l;
324 2267
        }
325 2267
        vg->last_i = i;
326 2267
        if (i == Z_OK)
327 2093
                return (VGZ_OK);
328 174
        if (i == Z_STREAM_END)
329 174
                return (VGZ_END);
330 0
        if (i == Z_BUF_ERROR)
331 0
                return (VGZ_STUCK);
332 0
        VSLb(vg->vsl, SLT_Gzip, "Gzip error: %d (%s)", i, vgz_msg(vg));
333 0
        return (VGZ_ERROR);
334 2267
}
335
336
/*--------------------------------------------------------------------
337
 * VDP for gunzip'ing
338
 */
339
340
// common for traditional interface and vdpio
341
static int vdp_gunzip_init_common(struct vdp_ctx *vdc);
342
343
static int v_matchproto_(vdp_init_f)
344 200
vdp_gunzip_init(VRT_CTX, struct vdp_ctx *vdc, void **priv)
345
{
346
        struct vgz *vg;
347
348 200
        CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
349 200
        CHECK_OBJ_NOTNULL(vdc, VDP_CTX_MAGIC);
350 200
        CHECK_OBJ_ORNULL(vdc->oc, OBJCORE_MAGIC);
351 200
        CHECK_OBJ_NOTNULL(vdc->hp, HTTP_MAGIC);
352 200
        AN(vdc->clen);
353 200
        AN(priv);
354
355 200
        vg = VGZ_NewGunzip(vdc->vsl, "U D -");
356 200
        AN(vg);
357 200
        if (vgz_getmbuf(vdc->wrk, vg)) {
358 0
                (void)VGZ_Destroy(vdc->wrk, &vg);
359 0
                return (-1);
360
        }
361
362 200
        VGZ_Obuf(vg, vg->m_buf, vg->m_sz);
363 200
        *priv = vg;
364 200
        return (vdp_gunzip_init_common(vdc));
365 200
}
366
367
static int
368 200
vdp_gunzip_init_common(struct vdp_ctx *vdc)
369
{
370
        struct boc *boc;
371
        enum boc_state_e bos;
372
        const char *p;
373
        ssize_t dl;
374
        uint64_t u;
375 200
        http_Unset(vdc->hp, H_Content_Encoding);
376
377 200
        *vdc->clen = -1;
378
379 200
        if (vdc->oc == NULL)
380 34
                return (0);
381
382 166
        boc = HSH_RefBoc(vdc->oc);
383 166
        if (boc != NULL) {
384 26
                CHECK_OBJ(boc, BOC_MAGIC);
385 26
                bos = boc->state;
386 26
                HSH_DerefBoc(vdc->wrk, vdc->oc);
387 26
                if (bos < BOS_FINISHED)
388 26
                        return (0); /* OA_GZIPBITS is not stable yet */
389 0
        }
390
391 140
        p = ObjGetAttr(vdc->wrk, vdc->oc, OA_GZIPBITS, &dl);
392 140
        if (p != NULL && dl == 32) {
393 140
                u = vbe64dec(p + 24);
394 140
                if (u != 0)
395 138
                        *vdc->clen = u;
396 140
        }
397 140
        return (0);
398 200
}
399
400
static int v_matchproto_(vdp_fini_f)
401 168
vdp_gunzip_fini(struct vdp_ctx *vdc, void **priv)
402
{
403
        struct vgz *vg;
404
405 168
        (void)vdc;
406 168
        TAKE_OBJ_NOTNULL(vg, priv, VGZ_MAGIC);
407 168
        AN(vg->m_buf);
408 168
        (void)VGZ_Destroy(vdc->wrk, &vg);
409 168
        return (0);
410
}
411
412
static int v_matchproto_(vdp_bytes_f)
413 885
vdp_gunzip_bytes(struct vdp_ctx *vdc, enum vdp_action act, void **priv,
414
    const void *ptr, ssize_t len)
415
{
416
        enum vgzret_e vr;
417
        ssize_t dl;
418
        const void *dp;
419
        struct worker *wrk;
420
        struct vgz *vg;
421
422 885
        CHECK_OBJ_NOTNULL(vdc, VDP_CTX_MAGIC);
423 885
        wrk = vdc->wrk;
424 885
        CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
425 885
        (void)act;
426
427 885
        CAST_OBJ_NOTNULL(vg, *priv, VGZ_MAGIC);
428 885
        AN(vg->m_buf);
429
430 885
        if (len == 0)
431 233
                return (0);
432
433 652
        VGZ_Ibuf(vg, ptr, len);
434 652
        do {
435 652
                vr = VGZ_Gunzip(vg, &dp, &dl);
436 652
                if (vr == VGZ_END && !VGZ_IbufEmpty(vg)) {
437 4
                        VSLb(vg->vsl, SLT_Gzip, "G(un)zip error: %d (%s)",
438 2
                             vr, "junk after VGZ_END");
439 2
                        return (-1);
440
                }
441 650
                vg->m_len += dl;
442 650
                if (vr < VGZ_OK)
443 0
                        return (-1);
444
                // END or STUCK
445 650
                if (vg->m_len == vg->m_sz || vr != VGZ_OK) {
446 324
                        if (VDP_bytes(vdc, vr == VGZ_END ? VDP_END : VDP_FLUSH,
447 162
                            vg->m_buf, vg->m_len))
448 8
                                return (vdc->retval);
449 154
                        vg->m_len = 0;
450 154
                        VGZ_Obuf(vg, vg->m_buf, vg->m_sz);
451 154
                }
452 642
        } while (!VGZ_IbufEmpty(vg));
453 642
        assert(vr == VGZ_STUCK || vr == VGZ_OK || vr == VGZ_END);
454 642
        return (0);
455 885
}
456
457
#ifdef LATER
458
/*
459
 * XXX does it make sense to work on more than one buffer?
460
 */
461
static int v_matchproto_(vdpio_init_f)
462
vdpio_gunzip_init(VRT_CTX, struct vdp_ctx *vdc, void **priv, int capacity)
463
{
464
        struct vgz *vg;
465
        struct vscarab *in;
466
467
        CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
468
        CHECK_OBJ_NOTNULL(vdc, VDP_CTX_MAGIC);
469
        CHECK_OBJ_ORNULL(vdc->oc, OBJCORE_MAGIC);
470
        CHECK_OBJ_NOTNULL(vdc->hp, HTTP_MAGIC);
471
        AN(vdc->clen);
472
        AN(priv);
473
        assert(capacity >= 1);
474
475
        if (capacity < 4)
476
                capacity = 4;
477
478
        in = WS_Alloc(ctx->ws, VSCARAB_SIZE(capacity));
479
        if (in == NULL)
480
                return (-1);
481
482
        vg = VGZ_NewGunzip(vdc->vsl, "U D -");
483
        if (vg == NULL)
484
                return (-1);
485
486
        AZ(vg->m_buf);
487
        vg->m_buf = (void *)in;
488
489
        *priv = vg;
490
        AZ(vdp_gunzip_init_common(vdc));
491
        return (1);
492
}
493
#endif
494
495
static int v_matchproto_(vdpio_init_f)
496 32
vdpio_gunzip_upgrade(VRT_CTX, struct vdp_ctx *vdc, void **priv, int capacity)
497
{
498
        struct vgz *vg;
499
        struct vscarab *in;
500
501 32
        CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
502 32
        CHECK_OBJ_NOTNULL(vdc, VDP_CTX_MAGIC);
503 32
        AN(priv);
504 32
        assert(capacity >= 1);
505
506 32
        if (capacity < 4)
507 32
                capacity = 4;
508
509
        /* done in vdp_gunzip_init */
510 32
        CAST_OBJ_NOTNULL(vg, *priv, VGZ_MAGIC);
511
512 32
        in = WS_Alloc(ctx->ws, VSCARAB_SIZE(capacity));
513 32
        if (in == NULL)
514 0
                return (-1);
515 32
        VSCARAB_INIT(in, capacity);
516
517
        // XXX duplicate work - remove when completing transition to VAI
518 32
        AN(vg->m_buf);
519 32
        AN(vg->stvbuf);
520 32
        STV_FreeBuf(vdc->wrk, &vg->stvbuf);
521 32
        vg->stvbuf = NULL;
522 32
        vg->m_buf = (void *)in;
523
524 32
        return (1);
525 32
}
526
527
static int v_matchproto_(vdpio_lease_f)
528 74
vdpio_gunzip_lease(struct vdp_ctx *vdc, struct vdp_entry *this, struct vscarab *out)
529
{
530
        struct vscarab *in;
531
        enum vgzret_e vr;
532
        struct vgz *vg;
533
        struct viov *v, *b, *o;
534
        int r;
535
536 74
        CHECK_OBJ_NOTNULL(vdc, VDP_CTX_MAGIC);
537 74
        CHECK_OBJ_NOTNULL(this, VDP_ENTRY_MAGIC);
538 74
        CAST_OBJ_NOTNULL(vg, this->priv, VGZ_MAGIC);
539 74
        CAST_OBJ_NOTNULL(in, (void*)vg->m_buf, VSCARAB_MAGIC);
540
541 74
        this->calls++;
542
543 74
        if (out->used == out->capacity)
544 32
                return (0);
545
546 42
        if (in->used < in->capacity && (in->flags & VSCARAB_F_END) == 0)
547 40
                r = vdpio_pull(vdc, this, in);
548
        else
549 2
                r = 0;
550
551 42
        if (in->used == 0) {
552 8
                out->flags = in->flags & VSCARAB_F_END;
553 8
                return (r);
554
        }
555
556
        // XXX more than one buffer?
557 30
        VSCARAB_LOCAL(buf, 1);
558 30
        b = VSCARAB_GET(buf);
559 30
        AN(b);
560 30
        b->iov.iov_len = cache_param->gzip_buffer;
561 30
        r = ObjVAIbuffer(vdc->wrk, vdc->vai_hdl, buf);
562 30
        if (r < 0)
563 0
                return (out->used ? 0 : r);
564
565 30
        o = VSCARAB_GET(out);
566 30
        AN(o);
567 30
        r = 0;
568
569 31
        VSCARAB_FOREACH(v, in) {
570 31
                this->bytes_in += v->iov.iov_len;
571 31
                vr = vgz_gunzip_iovec(vg, &v->iov, &b->iov, &o->iov);
572 31
                if (vr == VGZ_END && v->iov.iov_len > 0) {
573 0
                        VSLb(vg->vsl, SLT_Gzip, "G(un)zip error: %d (%s)",
574 0
                             vr, "junk after VGZ_END");
575 0
                        r = -EMSGSIZE;
576 0
                        break;
577
                }
578 33
                if (vr < VGZ_OK)
579 0
                        break;
580
581 33
                if (b->iov.iov_len == 0 || vr != VGZ_OK) {
582 32
                        r = 1;
583 32
                        break;
584
                }
585 1
        }
586
587 32
        if (r <= 0) {
588 0
                o->iov.iov_base = NULL;
589 0
                o->iov.iov_len = 0;
590 0
                vdpio_return_lease(vdc, b->lease);
591 0
                return (r);
592
        }
593
594 32
        o->lease = b->lease;
595 32
        b->lease = 0;
596
597 32
        vdpio_consolidate_vscarab(vdc, in);
598 32
        if (in->used == 0)
599 32
                out->flags = in->flags & VSCARAB_F_END;
600
601 32
        return (r);
602 72
}
603
604
static void v_matchproto_(vdpio_fini_f)
605 32
vdpio_gunzip_fini(struct vdp_ctx *vdc, void **priv)
606
{
607
        struct vscarab *in;
608
        struct vgz *vg;
609
610 32
        (void)vdc;
611 32
        TAKE_OBJ_NOTNULL(vg, priv, VGZ_MAGIC);
612 32
        CAST_OBJ_NOTNULL(in, (void *)vg->m_buf, VSCARAB_MAGIC);
613 32
        vg->m_buf = NULL;
614
615 32
        (void)VGZ_Destroy(vdc->wrk, &vg);
616 32
}
617
618
const struct vdp VDP_gunzip = {
619
        .name =         "gunzip",
620
        .init =         vdp_gunzip_init,
621
        .bytes =        vdp_gunzip_bytes,
622
        .fini =         vdp_gunzip_fini,
623
624
#ifdef LATER
625
        .io_init =      vdpio_gunzip_init,
626
#endif
627
        .io_upgrade =   vdpio_gunzip_upgrade,
628
        .io_lease =     vdpio_gunzip_lease,
629
        .io_fini =      vdpio_gunzip_fini,
630
};
631
632
/*--------------------------------------------------------------------*/
633
634
void
635 2504
VGZ_UpdateObj(const struct vfp_ctx *vc, struct vgz *vg, enum vgzret_e e)
636
{
637
        char *p;
638
        intmax_t ii;
639
640 2504
        CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
641 2504
        if (e < VGZ_OK)
642 0
                return;
643 2504
        ii = vg->vz.start_bit + vg->vz.last_bit + vg->vz.stop_bit;
644 2504
        if (e != VGZ_END && ii == vg->bits)
645 1837
                return;
646 667
        vg->bits = ii;
647 667
        p = ObjSetAttr(vc->wrk, vc->oc, OA_GZIPBITS, 32, NULL);
648 667
        AN(p);
649 667
        vbe64enc(p, vg->vz.start_bit);
650 667
        vbe64enc(p + 8, vg->vz.last_bit);
651 667
        vbe64enc(p + 16, vg->vz.stop_bit);
652 667
        if (e != VGZ_END)
653 255
                return;
654 412
        if (vg->dir == VGZ_GZ)
655 348
                vbe64enc(p + 24, vg->vz.total_in);
656 412
        if (vg->dir == VGZ_UN)
657 64
                vbe64enc(p + 24, vg->vz.total_out);
658 2504
}
659
660
/*--------------------------------------------------------------------
661
 */
662
663
enum vgzret_e
664 696
VGZ_Destroy(struct worker *wrk, struct vgz **vgp)
665
{
666
        struct vgz *vg;
667
        enum vgzret_e vr;
668
        int i;
669
670 696
        CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
671 696
        TAKE_OBJ_NOTNULL(vg, vgp, VGZ_MAGIC);
672 696
        AN(vg->id);
673 1392
        VSLb(vg->vsl, SLT_Gzip, "%s %jd %jd %jd %jd %jd",
674 696
            vg->id,
675 696
            (intmax_t)vg->vz.total_in,
676 696
            (intmax_t)vg->vz.total_out,
677 696
            (intmax_t)vg->vz.start_bit,
678 696
            (intmax_t)vg->vz.last_bit,
679 696
            (intmax_t)vg->vz.stop_bit);
680 696
        if (vg->dir == VGZ_GZ)
681 184
                i = deflateEnd(&vg->vz);
682
        else
683 512
                i = inflateEnd(&vg->vz);
684 696
        if (vg->last_i == Z_STREAM_END && i == Z_OK)
685 566
                i = Z_STREAM_END;
686 696
        if (vg->m_buf != NULL) {
687 512
                AN(vg->stvbuf);
688 512
                STV_FreeBuf(wrk, &vg->stvbuf);
689 512
        }
690 696
        AZ(vg->stvbuf);
691 696
        if (i == Z_OK)
692 126
                vr = VGZ_OK;
693 570
        else if (i == Z_STREAM_END)
694 566
                vr = VGZ_END;
695 4
        else if (i == Z_BUF_ERROR)
696 0
                vr = VGZ_STUCK;
697
        else {
698 8
                VSLb(vg->vsl, SLT_Gzip, "G(un)zip error: %d (%s)",
699 4
                    i, vgz_msg(vg));
700 4
                vr = VGZ_ERROR;
701
        }
702 696
        FREE_OBJ(vg);
703 696
        return (vr);
704
}
705
706
/*--------------------------------------------------------------------*/
707
708
static enum vfp_status v_matchproto_(vfp_init_f)
709 352
vfp_gzip_init(VRT_CTX, struct vfp_ctx *vc, struct vfp_entry *vfe)
710
{
711
        struct vgz *vg;
712
713 352
        CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
714 352
        CHECK_OBJ_NOTNULL(vc, VFP_CTX_MAGIC);
715 352
        CHECK_OBJ_NOTNULL(vfe, VFP_ENTRY_MAGIC);
716
717
        /*
718
         * G(un)zip makes no sence on partial responses, but since
719
         * it is an pure 1:1 transform, we can just ignore it.
720
         */
721 352
        if (http_GetStatus(vc->resp) == 206)
722 8
                return (VFP_NULL);
723
724 344
        if (vfe->vfp == &VFP_gzip) {
725 32
                if (http_GetHdr(vc->resp, H_Content_Encoding, NULL))
726 0
                        return (VFP_NULL);
727 32
                vg = VGZ_NewGzip(vc->wrk->vsl, vfe->vfp->priv1);
728 32
                vc->obj_flags |= OF_GZIPED | OF_CHGCE;
729 32
        } else {
730 312
                if (!http_HdrIs(vc->resp, H_Content_Encoding, "gzip"))
731 0
                        return (VFP_NULL);
732 312
                if (vfe->vfp == &VFP_gunzip) {
733 250
                        vg = VGZ_NewGunzip(vc->wrk->vsl, vfe->vfp->priv1);
734 250
                        vc->obj_flags &= ~OF_GZIPED;
735 250
                        vc->obj_flags |= OF_CHGCE;
736 250
                } else {
737 62
                        vg = VGZ_NewTestGunzip(vc->wrk->vsl, vfe->vfp->priv1);
738 62
                        vc->obj_flags |= OF_GZIPED;
739
                }
740
        }
741 344
        AN(vg);
742 344
        vfe->priv1 = vg;
743 344
        if (vgz_getmbuf(vc->wrk, vg))
744 0
                return (VFP_ERROR);
745 344
        VGZ_Ibuf(vg, vg->m_buf, 0);
746 344
        AZ(vg->m_len);
747
748 344
        if (vfe->vfp == &VFP_gunzip || vfe->vfp == &VFP_gzip) {
749 282
                http_Unset(vc->resp, H_Content_Encoding);
750 282
                http_Unset(vc->resp, H_Content_Length);
751 282
                RFC2616_Weaken_Etag(vc->resp);
752 282
        }
753
754 344
        if (vfe->vfp == &VFP_gzip)
755 32
                http_SetHeader(vc->resp, "Content-Encoding: gzip");
756
757 344
        if (vfe->vfp == &VFP_gzip || vfe->vfp == &VFP_testgunzip)
758 94
                RFC2616_Vary_AE(vc->resp);
759
760 344
        return (VFP_OK);
761 352
}
762
763
/*--------------------------------------------------------------------
764
 * VFP_GUNZIP
765
 *
766
 * A VFP for gunzip'ing an object as we receive it from the backend
767
 */
768
769
static enum vfp_status v_matchproto_(vfp_pull_f)
770 916
vfp_gunzip_pull(struct vfp_ctx *vc, struct vfp_entry *vfe, void *p,
771
    ssize_t *lp)
772
{
773
        ssize_t l;
774
        struct vgz *vg;
775 916
        enum vgzret_e vr = VGZ_ERROR;
776
        const void *dp;
777
        ssize_t dl;
778 916
        enum vfp_status vp = VFP_OK;
779
780 916
        CHECK_OBJ_NOTNULL(vc, VFP_CTX_MAGIC);
781 916
        CHECK_OBJ_NOTNULL(vfe, VFP_ENTRY_MAGIC);
782 916
        CAST_OBJ_NOTNULL(vg, vfe->priv1, VGZ_MAGIC);
783 916
        AN(p);
784 916
        AN(lp);
785 916
        l = *lp;
786 916
        *lp = 0;
787 916
        VGZ_Obuf(vg, p, l);
788 916
        do {
789 920
                if (VGZ_IbufEmpty(vg)) {
790 290
                        l = vg->m_sz;
791 290
                        vp = VFP_Suck(vc, vg->m_buf, &l);
792 290
                        if (vp == VFP_ERROR)
793 0
                                return (vp);
794 290
                        VGZ_Ibuf(vg, vg->m_buf, l);
795 290
                }
796 920
                if (!VGZ_IbufEmpty(vg) || vp == VFP_END) {
797 920
                        vr = VGZ_Gunzip(vg, &dp, &dl);
798 920
                        if (vr == VGZ_END && !VGZ_IbufEmpty(vg))
799 2
                                return (VFP_Error(vc, "Junk after gzip data"));
800 918
                        if (vr < VGZ_OK)
801 8
                                return (VFP_Error(vc,
802 4
                                    "Invalid Gzip data: %s", vgz_msg(vg)));
803 914
                        if (dl > 0) {
804 770
                                *lp = dl;
805 770
                                assert(dp == p);
806 770
                                return (VFP_OK);
807
                        }
808 144
                }
809 144
                AN(VGZ_IbufEmpty(vg));
810 144
        } while (vp == VFP_OK);
811 140
        if (vr != VGZ_END)
812 2
                return (VFP_Error(vc, "Gunzip error at the very end"));
813 138
        return (vp);
814 916
}
815
816
817
/*--------------------------------------------------------------------
818
 * VFP_GZIP
819
 *
820
 * A VFP for gzip'ing an object as we receive it from the backend
821
 */
822
823
static enum vfp_status v_matchproto_(vfp_pull_f)
824 126
vfp_gzip_pull(struct vfp_ctx *vc, struct vfp_entry *vfe, void *p,
825
    ssize_t *lp)
826
{
827
        ssize_t l;
828
        struct vgz *vg;
829 126
        enum vgzret_e vr = VGZ_ERROR;
830
        const void *dp;
831
        ssize_t dl;
832 126
        enum vfp_status vp = VFP_ERROR;
833
834 126
        CHECK_OBJ_NOTNULL(vc, VFP_CTX_MAGIC);
835 126
        CHECK_OBJ_NOTNULL(vfe, VFP_ENTRY_MAGIC);
836 126
        CAST_OBJ_NOTNULL(vg, vfe->priv1, VGZ_MAGIC);
837 126
        AN(p);
838 126
        AN(lp);
839 126
        l = *lp;
840 126
        *lp = 0;
841 126
        VGZ_Obuf(vg, p, l);
842 126
        do {
843 134
                if (VGZ_IbufEmpty(vg)) {
844 126
                        l = vg->m_sz;
845 126
                        vp = VFP_Suck(vc, vg->m_buf, &l);
846 126
                        if (vp == VFP_ERROR)
847 0
                                break;
848 126
                        if (vp == VFP_END)
849 44
                                vg->flag = VGZ_FINISH;
850 126
                        VGZ_Ibuf(vg, vg->m_buf, l);
851 126
                }
852 134
                if (!VGZ_IbufEmpty(vg) || vg->flag == VGZ_FINISH) {
853 134
                        vr = VGZ_Gzip(vg, &dp, &dl, vg->flag);
854 134
                        if (vr < VGZ_OK)
855 0
                                return (VFP_Error(vc, "Gzip failed"));
856 134
                        if (dl > 0) {
857 126
                                VGZ_UpdateObj(vc, vg, vr);
858 126
                                *lp = dl;
859 126
                                assert(dp == p);
860 126
                                if (vr != VGZ_END || !VGZ_IbufEmpty(vg))
861 96
                                        return (VFP_OK);
862 30
                        }
863 38
                }
864 38
                AN(VGZ_IbufEmpty(vg));
865 38
        } while (vg->flag != VGZ_FINISH);
866
867 30
        if (vr != VGZ_END)
868 0
                return (VFP_Error(vc, "Gzip failed"));
869 30
        VGZ_UpdateObj(vc, vg, VGZ_END);
870 30
        return (VFP_END);
871 126
}
872
873
/*--------------------------------------------------------------------
874
 * VFP_TESTGZIP
875
 *
876
 * A VFP for testing that received gzip data is valid, and for
877
 * collecting the magic bits while we're at it.
878
 */
879
880
static enum vfp_status v_matchproto_(vfp_pull_f)
881 81
vfp_testgunzip_pull(struct vfp_ctx *vc, struct vfp_entry *vfe, void *p,
882
    ssize_t *lp)
883
{
884
        struct vgz *vg;
885 81
        enum vgzret_e vr = VGZ_ERROR;
886
        const void *dp;
887
        ssize_t dl;
888
        enum vfp_status vp;
889
890 81
        CHECK_OBJ_NOTNULL(vc, VFP_CTX_MAGIC);
891 81
        CHECK_OBJ_NOTNULL(vfe, VFP_ENTRY_MAGIC);
892 81
        CAST_OBJ_NOTNULL(vg, vfe->priv1, VGZ_MAGIC);
893 81
        AN(p);
894 81
        AN(lp);
895 81
        vp = VFP_Suck(vc, p, lp);
896 81
        if (vp == VFP_ERROR)
897 4
                return (vp);
898 77
        if (*lp > 0 || vp == VFP_END) {
899 77
                VGZ_Ibuf(vg, p, *lp);
900 77
                do {
901 77
                        VGZ_Obuf(vg, vg->m_buf, vg->m_sz);
902 77
                        vr = VGZ_Gunzip(vg, &dp, &dl);
903 77
                        if (vr == VGZ_END && !VGZ_IbufEmpty(vg))
904 6
                                return (VFP_Error(vc, "Junk after gzip data"));
905 71
                        if (vr < VGZ_OK)
906 0
                                return (VFP_Error(vc,
907 0
                                    "Invalid Gzip data: %s", vgz_msg(vg)));
908 71
                } while (!VGZ_IbufEmpty(vg));
909 71
        }
910 71
        VGZ_UpdateObj(vc, vg, vr);
911 71
        if (vp == VFP_END && vr != VGZ_END)
912 2
                return (VFP_Error(vc, "tGunzip failed"));
913 69
        return (vp);
914 81
}
915
916
/*--------------------------------------------------------------------*/
917
918
static void v_matchproto_(vfp_fini_f)
919 374
vfp_gzip_fini(struct vfp_ctx *vc, struct vfp_entry *vfe)
920
{
921
        struct vgz *vg;
922
923 374
        CHECK_OBJ_NOTNULL(vc, VFP_CTX_MAGIC);
924 374
        CHECK_OBJ_NOTNULL(vfe, VFP_ENTRY_MAGIC);
925
926 374
        if (vfe->priv1 != NULL) {
927 344
                TAKE_OBJ_NOTNULL(vg, &vfe->priv1, VGZ_MAGIC);
928 344
                (void)VGZ_Destroy(vc->wrk, &vg);
929 344
        }
930 374
}
931
932
/*--------------------------------------------------------------------*/
933
934
const struct vfp VFP_gunzip = {
935
        .name = "gunzip",
936
        .init = vfp_gzip_init,
937
        .pull = vfp_gunzip_pull,
938
        .fini = vfp_gzip_fini,
939
        .priv1 = "U F -",
940
};
941
942
const struct vfp VFP_gzip = {
943
        .name = "gzip",
944
        .init = vfp_gzip_init,
945
        .pull = vfp_gzip_pull,
946
        .fini = vfp_gzip_fini,
947
        .priv1 = "G F -",
948
};
949
950
const struct vfp VFP_testgunzip = {
951
        .name = "testgunzip",
952
        .init = vfp_gzip_init,
953
        .pull = vfp_testgunzip_pull,
954
        .fini = vfp_gzip_fini,
955
        .priv1 = "u F -",
956
};