varnish-cache/bin/varnishhist/varnishhist.c
0
/*-
1
 * Copyright (c) 2006 Verdens Gang AS
2
 * Copyright (c) 2006-2015 Varnish Software AS
3
 * All rights reserved.
4
 *
5
 * Author: Poul-Henning Kamp <phk@phk.freebsd.dk>
6
 * Author: Dag-Erling Smørgrav <des@des.no>
7
 * Author: Guillaume Quintard <guillaume.quintard@gmail.com>
8
 *
9
 * SPDX-License-Identifier: BSD-2-Clause
10
 *
11
 * Redistribution and use in source and binary forms, with or without
12
 * modification, are permitted provided that the following conditions
13
 * are met:
14
 * 1. Redistributions of source code must retain the above copyright
15
 *    notice, this list of conditions and the following disclaimer.
16
 * 2. Redistributions in binary form must reproduce the above copyright
17
 *    notice, this list of conditions and the following disclaimer in the
18
 *    documentation and/or other materials provided with the distribution.
19
 *
20
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
21
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23
 * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
24
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
25
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
26
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
27
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
28
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
29
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30
 * SUCH DAMAGE.
31
 *
32
 * Log tailer for Varnish
33
 */
34
35
#include "config.h"
36
37
#include <limits.h>
38
#include <math.h>
39
#include <pthread.h>
40
#include <stdarg.h>
41
#include <stdlib.h>
42
#include <string.h>
43
#include <unistd.h>
44
45
#define VOPT_DEFINITION
46
#define VOPT_INC "varnishhist_options.h"
47
48
#include "vdef.h"
49
#include "vcurses.h"
50
#include "vapi/vsl.h"
51
#include "vapi/vsm.h"
52
#include "vapi/voptget.h"
53
#include "vapi/vsig.h"
54
#include "vas.h"
55
#include "vut.h"
56
#include "vtim.h"
57
58
#define HIST_N 2000             /* how far back we remember */
59
#define HIST_RES 100            /* bucket resolution */
60
61
static struct VUT *vut;
62
63
static int hist_low;
64
static int hist_high;
65
static int hist_range;
66
static unsigned hist_buckets;
67
68
static pthread_mutex_t mtx = PTHREAD_MUTEX_INITIALIZER;
69
70
static int end_of_file = 0;
71
static unsigned ms_delay = 1000;
72
static unsigned rr_hist[HIST_N];
73
static unsigned nhist;
74
static unsigned next_hist;
75
static unsigned *bucket_miss;
76
static unsigned *bucket_hit;
77
static char *format;
78
static int match_tag;
79
static double timebend = 0, t0;
80
static double vsl_t0 = 0, vsl_to, vsl_ts = 0;
81
static pthread_cond_t timebend_cv;
82
static double log_ten;
83
static char *ident;
84
85
static const unsigned scales[] = {
86
        1,
87
        2,
88
        3,
89
        4,
90
        5,
91
        10,
92
        15,
93
        20,
94
        25,
95
        50,
96
        100,
97
        250,
98
        500,
99
        1000,
100
        2500,
101
        5000,
102
        10000,
103
        25000,
104
        50000,
105
        100000,
106
        UINT_MAX
107
};
108
109
struct profile {
110
        const char *name;
111
        char VSL_arg;
112
        enum VSL_tag_e tag;
113
        const char *prefix;
114
        int field;
115
        int hist_low;
116
        int hist_high;
117
};
118
119
#define HIS_PROF(name,vsl_arg,tag,prefix,field,hist_low,high_high,doc)  \
120
        {name,vsl_arg,tag,prefix,field,hist_low,high_high},
121
#define HIS_NO_PREFIX   NULL
122
#define HIS_CLIENT      'c'
123
#define HIS_BACKEND     'b'
124
static const struct profile profiles[] = {
125
#include "varnishhist_profiles.h"
126
        { NULL }
127
};
128
#undef HIS_NO_PREFIX
129
#undef HIS_BACKEND
130
#undef HIS_CLIENT
131
#undef HIS_PROF
132
133
static const struct profile *active_profile;
134
135
static void
136 9349
update(void)
137
{
138
        char t[VTIM_FORMAT_SIZE];
139 9349
        const unsigned w = COLS / hist_range;
140 9349
        const unsigned n = w * hist_range;
141 9349
        unsigned bm[n], bh[n];
142
        unsigned max;
143
        unsigned scale;
144
        int i, j;
145
        unsigned k, l;
146
147
        /* Draw horizontal axis */
148 682989
        for (k = 0; k < n; ++k)
149 673640
                (void)mvaddch(LINES - 2, k, '-');
150 93042
        for (i = 0, j = hist_low; i < hist_range; ++i, ++j) {
151 83693
                (void)mvaddch(LINES - 2, w * i, '+');
152 83693
                IC(mvprintw(LINES - 1, w * i, "|1e%d", j));
153 83693
        }
154
155 9349
        if (end_of_file)
156 0
                IC(mvprintw(0, 0, "%*s", COLS - 1, "EOF"));
157
        else
158 9349
                IC(mvprintw(0, 0, "%*s", COLS - 1, ident));
159
160
        /* count our flock */
161 9349
        memset(bm, 0, sizeof bm);
162 9349
        memset(bh, 0, sizeof bh);
163 8378649
        for (k = 0, max = 1; k < hist_buckets; ++k) {
164 8369300
                l = k * n / hist_buckets;
165 8369300
                assert(l < n);
166 8369300
                bm[l] += bucket_miss[k];
167 8369300
                bh[l] += bucket_hit[k];
168 8369300
                max = vmax(max, bm[l] + bh[l]);
169 8369300
        }
170
171
        /* scale,time */
172 9349
        assert(LINES - 3 >= 0);
173 9349
        for (i = 0; max / scales[i] > (unsigned)(LINES - 3); ++i)
174
                /* nothing */ ;
175 9349
        scale = scales[i];
176
177 9349
        if (vsl_t0 > 0) {
178 9158
                VTIM_format(vsl_ts, t);
179
180 9158
                IC(mvprintw(0, 0, "1:%u, n = %u, d = %g @ %s x %g",
181
                    scale, nhist, 1e-3 * ms_delay, t, timebend));
182 9158
        } else {
183 191
                IC(mvprintw(0, 0, "1:%u, n = %u, d = %g",
184
                    scale, nhist, 1e-3 * ms_delay));
185
        }
186
187 46745
        for (j = 5; j < LINES - 2; j += 5)
188 37396
                IC(mvprintw((LINES - 2) - j, 0, "%u_", j * scale));
189
190
        /* show them */
191 682989
        for (k = 0; k < n; ++k) {
192 682942
                for (l = 0; l < bm[k] / scale; ++l)
193 9302
                        (void)mvaddch((LINES - 3) - l, k, '#');
194 673728
                for (; l < (bm[k] + bh[k]) / scale; ++l)
195 88
                        (void)mvaddch((LINES - 3) - l, k, '|');
196 673640
        }
197 9349
}
198
199
inline static void
200 32
upd_vsl_ts(const char *p)
201
{
202
203 32
        if (timebend == 0)
204 24
                return;
205
206 8
        p = strchr(p, ' ');
207
208 8
        if (p == NULL)
209 0
                return;
210
211 8
        vsl_ts = vmax_t(double, vsl_ts, strtod(p + 1, NULL));
212 32
}
213
214
static void
215 32
delorean(void)
216
{
217
        int i;
218 32
        double t = VTIM_mono();
219
220 32
        if (vsl_t0 == 0)
221 32
                vsl_to = vsl_t0 = vsl_ts;
222
223 32
        assert(t > t0);
224 32
        vsl_to = vsl_t0 + (t - t0) * timebend;
225
226 32
        if (vsl_ts > vsl_to) {
227 0
                double when = VTIM_real() + vsl_ts - vsl_to;
228 0
                struct timespec ts = VTIM_timespec(when);
229 0
                i = pthread_cond_timedwait(&timebend_cv, &mtx, &ts);
230 0
                assert(i == 0 || i == ETIMEDOUT);
231 0
        }
232 32
}
233
234
static int v_matchproto_ (VSLQ_dispatch_f)
235 48
accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[],
236
    void *priv)
237
{
238
        int i, tag, skip, match, hit;
239
        unsigned u;
240 48
        double value = 0;
241
        struct VSL_transaction *tr;
242
        const char *tsp;
243
        enum vsl_status stat;
244
245 48
        (void)vsl;
246 48
        (void)priv;
247
248 96
        for (tr = pt[0]; tr != NULL; tr = *++pt) {
249 48
                if (VSIG_int || VSIG_term || VSIG_hup)
250 0
                        return (-1);
251
252 48
                if (tr->reason == VSL_r_esi) {
253
                        /* Skip ESI requests */
254 0
                        continue;
255
                }
256
257 48
                hit = 0;
258 48
                skip = 0;
259 48
                match = 0;
260 48
                tsp = NULL;
261 1480
                while (skip == 0) {
262 1480
                        stat = VSL_Next(tr->c);
263 1480
                        if (stat == vsl_e_overrun) {
264
                                /* need to skip forward */
265 0
                                PTOK(pthread_mutex_lock(&mtx));
266 0
                                vsl_to = vsl_t0 = vsl_ts = 0;
267 0
                                t0 = VTIM_mono();
268 0
                                PTOK(pthread_mutex_unlock(&mtx));
269 0
                                break;
270
                        }
271 1480
                        if (stat != vsl_more)
272 48
                                break;
273
274
                        /* get the value we want and register if it's a hit */
275 1432
                        tag = VSL_TAG(tr->c->rec.ptr);
276
277 1432
                        if (VSL_tagflags[tag])
278 160
                                continue;
279
280 1272
                        switch (tag) {
281
                        case SLT_Hit:
282 8
                                hit = 1;
283 8
                                break;
284
                        case SLT_VCL_return:
285 96
                                if (!strcasecmp(VSL_CDATA(tr->c->rec.ptr),
286 96
                                    "restart") ||
287 96
                                    !strcasecmp(VSL_CDATA(tr->c->rec.ptr),
288
                                    "retry"))
289 0
                                        skip = 1;
290 96
                                break;
291
                        case SLT_Timestamp:
292 184
                                tsp = VSL_CDATA(tr->c->rec.ptr);
293
                                /* FALLTHROUGH */
294
                        default:
295 1168
                                if (tag != match_tag)
296 1032
                                        break;
297
298 136
                                if (active_profile->prefix &&
299 256
                                    strncmp(VSL_CDATA(tr->c->rec.ptr),
300 128
                                    active_profile->prefix,
301 256
                                    strlen(active_profile->prefix)) != 0)
302 104
                                        break;
303
304 64
                                i = sscanf(VSL_CDATA(tr->c->rec.ptr),
305 32
                                    format, &value);
306 32
                                if (i != 1)
307 0
                                        break;
308 32
                                match = 1;
309 32
                                break;
310
                        }
311
                }
312
313 48
                if (skip || !match || value <= 0)
314 16
                        continue;
315
316
                /* select bucket */
317 32
                i = vlimit_t(int, lround(HIST_RES * log(value) / log_ten),
318 32
                    hist_low * HIST_RES, hist_high * HIST_RES - 1) -
319 32
                        hist_low * HIST_RES;
320 32
                assert(i >= 0);
321 32
                assert((unsigned)i < hist_buckets);
322
323 32
                PTOK(pthread_mutex_lock(&mtx));
324
325
                /*
326
                 * only parse the last tsp seen in this transaction -
327
                 * it should be the latest.
328
                 */
329 32
                if (tsp)
330 32
                        upd_vsl_ts(tsp);
331
332
                /* phase out old data */
333 32
                if (nhist == HIST_N) {
334 0
                        u = rr_hist[next_hist];
335 0
                        if (u >= hist_buckets) {
336 0
                                u -= hist_buckets;
337 0
                                assert(u < hist_buckets);
338 0
                                assert(bucket_hit[u] > 0);
339 0
                                bucket_hit[u]--;
340 0
                        } else {
341 0
                                assert(bucket_miss[u] > 0);
342 0
                                bucket_miss[u]--;
343
                        }
344 0
                } else {
345 32
                        ++nhist;
346
                }
347
348
                /* phase in new data */
349 32
                if (hit) {
350 8
                        bucket_hit[i]++;
351 8
                        rr_hist[next_hist] = i + hist_buckets;
352 8
                } else {
353 24
                        bucket_miss[i]++;
354 24
                        rr_hist[next_hist] = i;
355
                }
356 32
                if (++next_hist == HIST_N) {
357 0
                        next_hist = 0;
358 0
                }
359 32
                if (vsl_ts >= vsl_to)
360 32
                        delorean();
361 32
                PTOK(pthread_mutex_unlock(&mtx));
362 32
        }
363 48
        return (0);
364 48
}
365
366
static void * v_matchproto_(pthread_t)
367 24
do_curses(void *arg)
368
{
369
        int ch;
370 24
        (void)arg;
371
372 24
        (void)initscr();
373 24
        AC(raw());
374 24
        AC(noecho());
375 24
        AC(nonl());
376 24
        AC(intrflush(stdscr, FALSE));
377 24
        IC(curs_set(0));
378 24
        AC(erase());
379 9373
        while (!VSIG_int && !VSIG_term && !VSIG_hup) {
380
381 9349
                AC(erase());
382 9349
                PTOK(pthread_mutex_lock(&mtx));
383 9349
                update();
384 9349
                PTOK(pthread_mutex_unlock(&mtx));
385 9349
                AC(refresh());
386
387 9349
                assert(ms_delay > 0);
388 9349
                timeout(ms_delay);
389 9349
                switch ((ch = getch())) {
390
                case ERR:
391 9157
                        break;
392
#ifdef KEY_RESIZE
393
                case KEY_RESIZE:
394 8
                        AC(erase());
395 8
                        break;
396
#endif
397
                case '\014':    /* Ctrl-L */
398
                case '\024':    /* Ctrl-T */
399 8
                        AC(redrawwin(stdscr));
400 8
                        AC(refresh());
401 8
                        break;
402
                case '\032':    /* Ctrl-Z */
403 0
                        AC(endwin());
404 0
                        AZ(raise(SIGTSTP));
405 0
                        break;
406
                case '\003':    /* Ctrl-C */
407
                case '\021':    /* Ctrl-Q */
408
                case 'Q':
409
                case 'q':
410 8
                        AZ(raise(SIGINT));
411 8
                        break;
412
                case '0':
413
                case '1':
414
                case '2':
415
                case '3':
416
                case '4':
417
                case '5':
418
                case '6':
419
                case '7':
420
                case '8':
421
                case '9':
422 16
                        ms_delay = 1000U << (ch - '0');
423 16
                        break;
424
                case '+':
425 96
                        ms_delay = vmax(ms_delay >> 1, 1U);
426 96
                        break;
427
                case '-':
428 16
                        ms_delay *= 2;
429 16
                        break;
430
                case '>':
431
                case '<':
432
                        /* see below */
433 32
                        break;
434
                default:
435 8
                        AC(beep());
436 8
                        break;
437
                }
438
439 9349
                if (ch == '<' || ch == '>') {
440 32
                        PTOK(pthread_mutex_lock(&mtx));
441 32
                        vsl_to = vsl_t0 = vsl_ts;
442 32
                        t0 = VTIM_mono();
443 32
                        if (timebend == 0)
444 8
                                timebend = 1;
445 24
                        else if (ch == '<')
446 16
                                timebend /= 2;
447
                        else
448 8
                                timebend *= 2;
449 32
                        PTOK(pthread_cond_broadcast(&timebend_cv));
450 32
                        PTOK(pthread_mutex_unlock(&mtx));
451 32
                }
452
        }
453 24
        AC(endwin());
454 24
        return (NULL);
455
}
456
457
/*--------------------------------------------------------------------*/
458
459
static void v_noreturn_
460 40
profile_error(const char *s)
461
{
462 80
        fprintf(stderr, "-P: '%s' is not a valid"
463 40
            " profile name or definition\n", s);
464 40
        exit(1);
465
}
466
467
int
468 152
main(int argc, char **argv)
469
{
470
        int i;
471
        char *colon;
472 152
        const char *ptag, *profile = "responsetime";
473
        pthread_t thr;
474
        int fnum;
475 152
        struct profile cli_p = {0};
476
477 152
        vut = VUT_InitProg(argc, argv, &vopt_spec);
478 152
        AN(vut);
479 152
        PTOK(pthread_cond_init(&timebend_cv, NULL));
480
481 224
        while ((i = getopt(argc, argv, vopt_spec.vopt_optstring)) != -1) {
482 176
                switch (i) {
483
                case 'h':
484
                        /* Usage help */
485 8
                        VUT_Usage(vut, &vopt_spec, 0);
486
                case 'p':
487 24
                        ms_delay = lround(1e3 * strtod(optarg, NULL));
488 24
                        if (ms_delay == 0)
489 16
                                VUT_Error(vut, 1, "-p: invalid '%s'", optarg);
490 8
                        break;
491
                case 'P':
492 88
                        colon = strchr(optarg, ':');
493
                        /* no colon, take the profile as a name */
494 88
                        if (colon == NULL) {
495 16
                                profile = optarg;
496 16
                                break;
497
                        }
498
                        /* else check if valid definition */
499 72
                        if (colon == optarg + 1 && (*optarg == 'b' ||
500 0
                            *optarg == 'c' || *optarg == 'E')) {
501 56
                                cli_p.VSL_arg = *optarg;
502 56
                                ptag = colon + 1;
503 56
                                colon = strchr(colon + 1, ':');
504 56
                                if (colon == NULL)
505 8
                                        profile_error(optarg);
506 48
                        } else {
507 16
                                ptag = optarg;
508 16
                                cli_p.VSL_arg = 'c';
509
                        }
510
511 64
                        assert(colon);
512
513 64
                        match_tag = VSL_Name2Tag(ptag, colon - ptag);
514 64
                        if (match_tag < 0)
515 16
                                VUT_Error(vut, 1,
516
                                    "-P: '%s' is not a valid tag name",
517 8
                                    optarg);
518
519 56
                        if (VSL_tagflags[match_tag])
520 16
                                VUT_Error(vut, 1,
521
                                    "-P: '%s' is an unsafe or binary record",
522 8
                                    optarg);
523
524 48
                        cli_p.prefix = colon + 1;
525
526 48
                        colon = strchr(colon + 1, ':');
527 48
                        if (colon == NULL)
528 8
                                profile_error(optarg);
529
530 40
                        *colon = '\0';
531 40
                        if (*cli_p.prefix == '\0')
532 32
                                cli_p.prefix = NULL;
533
534 40
                        if (sscanf(colon + 1, "%d",  &cli_p.field) != 1)
535 16
                                profile_error(optarg);
536
537 24
                        cli_p.name = "custom";
538 24
                        cli_p.tag = (enum VSL_tag_e)match_tag;
539 24
                        cli_p.hist_low = -6;
540 24
                        cli_p.hist_high = 3;
541 24
                        profile = NULL;
542 24
                        active_profile = &cli_p;
543
544 24
                        colon = strchr(colon + 1, ':');
545 24
                        if (colon == NULL)
546 8
                                break;
547
548 48
                        if (sscanf(colon + 1, "%d:%d",  &cli_p.hist_low,
549 32
                                   &cli_p.hist_high) != 2)
550 8
                                profile_error(optarg);
551
552 8
                        break;
553
                case 'B':
554 24
                        timebend = strtod(optarg, NULL);
555 24
                        if (timebend == 0)
556 16
                                VUT_Error(vut, 1,
557
                                    "-B: being able to bend time does not"
558
                                    " mean we can stop it"
559 8
                                    " (invalid factor '%s')", optarg);
560 16
                        if (timebend < 0)
561 16
                                VUT_Error(vut, 1,
562
                                    "-B: being able to bend time does not"
563
                                    " mean we can make it go backwards"
564 8
                                    " (invalid factor '%s')", optarg);
565 8
                        break;
566
                default:
567 32
                        if (!VUT_Arg(vut, i, optarg))
568 8
                                VUT_Usage(vut, &vopt_spec, 1);
569 24
                }
570
        }
571
572 48
        if (optind != argc)
573 8
                VUT_Usage(vut, &vopt_spec, 1);
574
575
        /* Check for valid grouping mode */
576 40
        assert(vut->g_arg < VSL_g__MAX);
577 40
        if (vut->g_arg != VSL_g_vxid && vut->g_arg != VSL_g_request)
578 16
                VUT_Error(vut, 1, "Invalid grouping mode: %s"
579
                    " (only vxid and request are supported)",
580 8
                    VSLQ_grouping[vut->g_arg]);
581
582 32
        if (profile) {
583 120
                for (active_profile = profiles; active_profile->name;
584 96
                     active_profile++) {
585 112
                        if (strcmp(active_profile->name, profile) == 0)
586 16
                                break;
587 96
                }
588 24
        }
589 32
        AN(active_profile);
590 32
        if (!active_profile->name)
591 8
                VUT_Error(vut, 1, "-P: No such profile '%s'", profile);
592
593 24
        assert(active_profile->VSL_arg == 'b' ||
594
            active_profile->VSL_arg == 'c' ||
595
            active_profile->VSL_arg == 'E');
596 24
        assert(VUT_Arg(vut, active_profile->VSL_arg, NULL));
597 24
        match_tag = active_profile->tag;
598 24
        fnum = active_profile->field;
599 24
        hist_low = active_profile->hist_low;
600 24
        hist_high = active_profile->hist_high;
601
602 24
        hist_range = hist_high - hist_low;
603 24
        hist_buckets = hist_range * HIST_RES;
604 24
        bucket_hit = calloc(hist_buckets, sizeof *bucket_hit);
605 24
        bucket_miss = calloc(hist_buckets, sizeof *bucket_miss);
606
607 24
        if (timebend > 0)
608 8
                t0 = VTIM_mono();
609
610 24
        format = malloc(4L * fnum);
611 24
        AN(format);
612 88
        for (i = 0; i < fnum - 1; i++)
613 64
                strcpy(format + 4 * i, "%*s ");
614 24
        strcpy(format + 4 * (fnum - 1), "%lf");
615
616 24
        log_ten = log(10.0);
617
618 24
        VUT_Setup(vut);
619 24
        if (vut->vsm)
620 24
                ident = VSM_Dup(vut->vsm, "Arg", "-i");
621
        else
622 0
                ident = strdup("");
623 24
        PTOK(pthread_create(&thr, NULL, do_curses, NULL));
624 24
        vut->dispatch_f = accumulate;
625 24
        vut->dispatch_priv = NULL;
626 24
        (void)VUT_Main(vut);
627 24
        end_of_file = 1;
628 24
        PTOK(pthread_join(thr, NULL));
629 24
        VUT_Fini(&vut);
630 24
        exit(0);
631
}