1 /* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2 
3 /** \file
4  * The main memcached header holding commonly used data
5  * structures and function prototypes.
6  */
7 
8 #include "src/config.h"
9 
10 #include <stdio.h>
11 #include <sys/types.h>
12 #include <event.h>
13 #include <stdbool.h>
14 #include <stdint.h>
15 #include <platform/platform.h>
16 
17 #include "work.h"
18 #include "genhash.h"
19 
20 #include "protocol_binary.h"
21 #include "cache.h"
22 
23 /** Maximum length of a key. */
24 #define KEY_MAX_LENGTH 250
25 
26 /** Size of an incr buf. */
27 #define INCR_MAX_STORAGE_LEN 24
28 
29 #define DATA_BUFFER_SIZE 2048
30 #define UDP_READ_BUFFER_SIZE 65536
31 #define UDP_MAX_PAYLOAD_SIZE 1400
32 #define UDP_HEADER_SIZE 8
33 #define MAX_SENDBUF_SIZE (256 * 1024 * 1024)
34 
35 /* Port values */
36 #define EPHEMERAL -1
37 #define UNSPECIFIED -2
38 
39 /* No. of seconds in 30 days - largest possible delta exptime. */
40 #define REALTIME_MAXDELTA 60*60*24*30
41 
42 /* I'm told the max length of a 64-bit num converted to string is 20 bytes.
43  * Plus a few for spaces, \r\n, \0 */
44 #define SUFFIX_SIZE 200
45 
46 /** Initial size of list of items being returned by "get". */
47 #define ITEM_LIST_INITIAL 200
48 
49 /** Initial size of list of CAS suffixes appended to "gets" lines. */
50 #define SUFFIX_LIST_INITIAL 20
51 
52 /** Initial size of the sendmsg() scatter/gather array. */
53 #define IOV_LIST_INITIAL 400
54 
55 /** Initial number of sendmsg() argument structures to allocate. */
56 #define MSG_LIST_INITIAL 10
57 
58 /** High water marks for buffer shrinking */
59 #define READ_BUFFER_HIGHWAT 8192
60 #define ITEM_LIST_HIGHWAT 400
61 #define IOV_LIST_HIGHWAT 600
62 #define MSG_LIST_HIGHWAT 100
63 
64 /* Binary protocol stuff */
65 #define MIN_BIN_PKT_LENGTH 16
66 #define BIN_PKT_HDR_WORDS (MIN_BIN_PKT_LENGTH/sizeof(uint32_t))
67 
68 /* unistd.h is here */
69 #if HAVE_UNISTD_H
70 # include <unistd.h>
71 #endif
72 
73 /* Slab sizing definitions. */
74 #define POWER_SMALLEST 1
75 #define POWER_LARGEST  200
76 #define POWER_BLOCK 1048576
77 #define CHUNK_ALIGN_BYTES 8
78 #define DONT_PREALLOC_SLABS
79 #define MAX_NUMBER_OF_SLAB_CLASSES (POWER_LARGEST + 1)
80 
81 /** How long an object can reasonably be assumed to be locked before
82     harvesting it on a low memory condition. */
83 #define TAIL_REPAIR_TIME (3 * 3600)
84 
85 /* warning: don't use these macros with a function, as it evals its arg twice */
86 #define ITEM_get_cas(i) ((uint64_t)(((i)->it_flags & ITEM_CAS) ? \
87                                     *(uint64_t*)&((i)->end[0]) : 0x0))
88 #define ITEM_set_cas(i,v) { if ((i)->it_flags & ITEM_CAS) { \
89                           *(uint64_t*)&((i)->end[0]) = v; } }
90 
91 #define ITEM_key(item) (((char*)&((item)->end[0])) \
92          + (((item)->it_flags & ITEM_CAS) ? sizeof(uint64_t) : 0))
93 
94 #define ITEM_suffix(item) ((char*) &((item)->end[0]) + (item)->nkey + 1 \
95          + (((item)->it_flags & ITEM_CAS) ? sizeof(uint64_t) : 0))
96 
97 #define ITEM_data(item) ((char*) &((item)->end[0]) + (item)->nkey + 1 \
98          + (item)->nsuffix \
99          + (((item)->it_flags & ITEM_CAS) ? sizeof(uint64_t) : 0))
100 
101 #define ITEM_ntotal(item) (sizeof(struct _stritem) + (item)->nkey + 1 \
102          + (item)->nsuffix + (item)->nbytes \
103          + (((item)->it_flags & ITEM_CAS) ? sizeof(uint64_t) : 0))
104 
105 #define STAT_KEY_LEN 256
106 #define STAT_VAL_LEN 512
107 
108 /** Append a simple stat with a stat name, value format and value */
109 #define APPEND_STAT(name, fmt, val) \
110     append_stat(name, add_stats, c, fmt, val);
111 
112 #define APPEND_PREFIX_STAT(name, fmt, val) \
113     append_prefix_stat(prefix, name, add_stats, c, fmt, val);
114 
115 /** Append an indexed stat with a stat name (with format), value format
116     and value */
117 #define APPEND_NUM_FMT_STAT(name_fmt, num, name, fmt, val)          \
118     klen = snprintf(key_str, STAT_KEY_LEN, name_fmt, num, name);    \
119     vlen = snprintf(val_str, STAT_VAL_LEN, fmt, val);               \
120     add_stats(key_str, klen, val_str, vlen, c);
121 
122 /** Common APPEND_NUM_FMT_STAT format. */
123 #define APPEND_NUM_STAT(num, name, fmt, val) \
124     APPEND_NUM_FMT_STAT("%d:%s", num, name, fmt, val)
125 
126 /**
127  * Callback for any function producing stats.
128  *
129  * @param key the stat's key
130  * @param klen length of the key
131  * @param val the stat's value in an ascii form (e.g. text form of a number)
132  * @param vlen length of the value
133  * @parm cookie magic callback cookie
134  */
135 typedef void (*ADD_STAT)(const char *key, const uint16_t klen,
136                          const char *val, const uint32_t vlen,
137                          const void *cookie);
138 
139 /*
140  * NOTE: If you modify this table you _MUST_ update the function state_text
141  */
142 /**
143  * Possible states of a connection.
144  */
145 enum conn_states {
146     conn_listening,  /**< the socket which listens for connections */
147     conn_new_cmd,    /**< Prepare connection for next command */
148     conn_waiting,    /**< waiting for a readable socket */
149     conn_read,       /**< reading in a command line */
150     conn_parse_cmd,  /**< try to parse a command from the input buffer */
151     conn_write,      /**< writing out a simple response */
152     conn_nread,      /**< reading in a fixed number of bytes */
153     conn_swallow,    /**< swallowing unnecessary bytes w/o storing */
154     conn_closing,    /**< closing this connection */
155     conn_mwrite,     /**< writing out many items sequentially */
156     conn_pause,      /**< waiting for asynchronous event */
157     conn_connecting, /**< the socket is in connecting state*/
158     conn_max_state   /**< Max state value (used for assertion) */
159 };
160 
161 enum bin_substates {
162     bin_no_state,
163     bin_reading_set_header,
164     bin_reading_cas_header,
165     bin_read_set_value,
166     bin_reading_get_key,
167     bin_reading_stat,
168     bin_reading_del_header,
169     bin_reading_incr_header,
170     bin_read_flush_exptime
171 };
172 
173 enum protocol {
174     ascii_prot = 3, /* arbitrary value. */
175     binary_prot,
176     proxy_upstream_ascii_prot,
177     proxy_upstream_binary_prot,
178     proxy_downstream_ascii_prot,
179     proxy_downstream_binary_prot,
180     negotiating_proxy_prot,
181     negotiating_prot /* Discovering the protocol */
182 };
183 
184 enum network_transport {
185     local_transport, /* Unix sockets*/
186     tcp_transport,
187     udp_transport
188 };
189 
190 int IS_UDP(enum network_transport protocol);
191 #define IS_ASCII(x) (x == ascii_prot || \
192                      x == proxy_upstream_ascii_prot || \
193                      x == proxy_downstream_ascii_prot)
194 #define IS_BINARY(x) (x == binary_prot || \
195                       x == proxy_upstream_binary_prot || \
196                       x == proxy_downstream_binary_prot)
197 #define IS_NEGOTIATING(x) (x == negotiating_prot || \
198                            x == negotiating_proxy_prot)
199 #define IS_PROXY(x) (x == proxy_upstream_ascii_prot || \
200                      x == proxy_upstream_binary_prot || \
201                      x == proxy_downstream_ascii_prot || \
202                      x == proxy_downstream_binary_prot || \
203                      x == negotiating_proxy_prot)
204 #define IS_DOWNSTREAM(x) (x == proxy_downstream_ascii_prot || \
205                           x == proxy_downstream_binary_prot)
206 
207 #define NREAD_ADD 1
208 #define NREAD_SET 2
209 #define NREAD_REPLACE 3
210 #define NREAD_APPEND 4
211 #define NREAD_PREPEND 5
212 #define NREAD_CAS 6
213 
214 enum store_item_type {
215     NOT_STORED=0, STORED, EXISTS, NOT_FOUND
216 };
217 
218 enum delta_result_type {
219     OK, NON_NUMERIC, EOM
220 };
221 
222 /** Time relative to server start. Smaller than time_t on 64-bit systems. */
223 typedef unsigned int rel_time_t;
224 
225 /** Stats stored per slab (and per thread). */
226 struct slab_stats {
227     uint64_t  set_cmds;
228     uint64_t  get_hits;
229     uint64_t  delete_hits;
230     uint64_t  cas_hits;
231     uint64_t  cas_badval;
232     uint64_t  incr_hits;
233     uint64_t  decr_hits;
234 };
235 
236 /**
237  * Stats stored per-thread.
238  */
239 struct thread_stats {
240     cb_mutex_t        mutex;
241     uint64_t          get_cmds;
242     uint64_t          get_misses;
243     uint64_t          delete_misses;
244     uint64_t          incr_misses;
245     uint64_t          decr_misses;
246     uint64_t          cas_misses;
247     uint64_t          bytes_read;
248     uint64_t          bytes_written;
249     uint64_t          flush_cmds;
250     uint64_t          conn_yields; /* # of yields for connections (-R option)*/
251     struct slab_stats slab_stats[MAX_NUMBER_OF_SLAB_CLASSES];
252 };
253 
254 /**
255  * Global stats.
256  */
257 struct stats {
258     cb_mutex_t    mutex;
259     unsigned int  curr_items;
260     unsigned int  total_items;
261     uint64_t      curr_bytes;
262     unsigned int  curr_conns;
263     unsigned int  total_conns;
264     unsigned int  conn_structs;
265     uint64_t      get_cmds;
266     uint64_t      set_cmds;
267     uint64_t      get_hits;
268     uint64_t      get_misses;
269     uint64_t      evictions;
270     time_t        started;          /* when the process was started */
271     bool          accepting_conns;  /* whether we are currently accepting */
272     uint64_t      listen_disabled_num;
273 };
274 
275 struct proxy_stats_cmd_info {
276     bool do_info;
277     bool do_settings;
278     bool do_behaviors;
279     bool do_frontcache;
280     bool do_keystats;
281     bool do_stats;
282     bool do_zeros; /* might be used later */
283 };
284 
285 #define MAX_VERBOSITY_LEVEL 4
286 
287 /* When adding a setting, be sure to update process_stat_settings */
288 /**
289  * Globally accessible settings as derived from the commandline.
290  */
291 struct settings {
292     size_t maxbytes;
293     int maxconns;
294     int port;
295     int udpport;
296     char *inter;
297     int verbose;
298     rel_time_t oldest_live; /* ignore existing items older than this */
299     int evict_to_free;
300     char *socketpath;   /* path to unix socket if using local socket */
301     int access;  /* access mask (a la chmod) for unix domain socket */
302     double factor;          /* chunk size growth factor */
303     int chunk_size;
304     int num_threads;        /* number of libevent threads to run */
305     char prefix_delimiter;  /* character that marks a key prefix (for stats) */
306     int detail_enabled;     /* nonzero if we're collecting detailed stats */
307     int reqs_per_event;     /* Maximum number of io to process on each
308                                io-event. */
309     bool use_cas;
310     enum protocol binding_protocol;
311     int backlog;
312     bool enable_mcmux_mode; /* enable mcmux compatiblity mode, disables libvbucket/libmemcached support */
313 };
314 
315 extern struct stats stats;
316 extern time_t process_started;
317 extern struct settings settings;
318 
319 #define ITEM_LINKED 1
320 #define ITEM_CAS 2
321 
322 /* temp */
323 #define ITEM_SLABBED 4
324 
325 /**
326  * Structure for storing items within memcached.
327  */
328 typedef struct _stritem {
329     struct _stritem *next;
330     struct _stritem *prev;
331     struct _stritem *h_next;    /* hash chain next */
332     rel_time_t      time;       /* least recent access */
333     rel_time_t      exptime;    /* expire time */
334     int             nbytes;     /* size of data */
335     unsigned short  refcount;
336     uint8_t         nsuffix;    /* length of flags-and-length string */
337     uint8_t         it_flags;   /* ITEM_* above */
338     uint8_t         slabs_clsid;/* which slab class we're in */
339     uint8_t         nkey;       /* key length, w/terminating null and padding */
340     void * end[];
341     /* if it_flags & ITEM_CAS we have 8 bytes CAS */
342     /* then null-terminated key */
343     /* then " flags length\r\n" (no terminating null) */
344     /* then data with terminating \r\n (no terminating null; it's binary!) */
345 } item;
346 
347 typedef struct bin_cmd bin_cmd;
348 
349 struct bin_cmd {
350     item *request_item;  /* Has 1 refcount. */
351     item *response_item; /* Has 1 refcount. */
352     bin_cmd *next;
353 };
354 
355 typedef struct {
356     cb_thread_t thread_id;        /* unique ID of this thread */
357     struct event_base *base;    /* libevent handle this thread uses */
358     struct event notify_event;  /* listen event for notify pipe */
359     SOCKET notify_receive_fd;      /* receiving end of notify pipe */
360     SOCKET notify_send_fd;         /* sending end of notify pipe */
361     struct thread_stats stats;  /* Stats generated by this thread */
362     struct conn_queue *new_conn_queue; /* queue of new connections to handle */
363     cache_t *suffix_cache;      /* suffix cache */
364     work_queue *work_queue;
365     genhash_t *conn_hash;       /* per thread connection hash, keyed by host_ident */
366 } LIBEVENT_THREAD;
367 
368 /**
369  * The structure representing a connection into memcached.
370  */
371 typedef struct conn conn;
372 typedef struct conn_funcs conn_funcs;
373 
374 struct conn_funcs {
375     /* Function pointers so that drive_machine loop is reusable. */
376     bool (*conn_init)(conn *c);
377     void (*conn_close)(conn *c);
378     bool (*conn_connect)(conn *c);
379     void (*conn_process_ascii_command)(conn *c, char *command);
380     void (*conn_process_binary_command)(conn *c);
381     void (*conn_complete_nread_ascii)(conn *c);
382     void (*conn_complete_nread_binary)(conn *c);
383     void (*conn_pause)(conn *c);
384     rel_time_t (*conn_realtime)(const time_t exptime);
385     void (*conn_state_change)(conn *c, enum conn_states next_state);
386 
387     /* PROTOCOL_BINARY_REQ/RES */
388     uint8_t conn_binary_command_magic;
389 };
390 
391 struct conn {
392     SOCKET sfd;
393     enum conn_states  state;
394     enum bin_substates substate;
395     struct event event;
396     short  ev_flags;
397     short  which;   /** which events were just triggered */
398 
399     char   *rbuf;   /** buffer to read commands into */
400     char   *rcurr;  /** but if we parsed some already, this is where we stopped */
401     int    rsize;   /** total allocated size of rbuf */
402     int    rbytes;  /** how much data, starting from rcur, do we have unparsed */
403 
404     char   *wbuf;
405     char   *wcurr;
406     int    wsize;
407     int    wbytes;
408     /** which state to go into after finishing current write */
409     enum conn_states  write_and_go;
410     void   *write_and_free; /** free this memory after finishing writing */
411 
412     char   *ritem;  /** when we read in an item's value, it goes here */
413     int    rlbytes;
414 
415     /* data for the nread state */
416 
417     /**
418      * item is used to hold an item structure created after reading the command
419      * line of set/add/replace commands, but before we finished reading the actual
420      * data. The data is read into ITEM_data(item) to avoid extra copying.
421      */
422 
423     void   *item;     /* for commands set/add/replace  */
424 
425     /* data for the swallow state */
426     int    sbytes;    /* how many bytes to swallow */
427 
428     /* data for the mwrite state */
429     struct iovec *iov;
430     int    iovsize;   /* number of elements allocated in iov[] */
431     int    iovused;   /* number of elements used in iov[] */
432 
433     struct msghdr *msglist;
434     int    msgsize;   /* number of elements allocated in msglist[] */
435     int    msgused;   /* number of elements used in msglist[] */
436     int    msgcurr;   /* element in msglist[] being transmitted now */
437     int    msgbytes;  /* number of bytes in current msg */
438 
439     item   **ilist;   /* list of items to write out */
440     int    isize;
441     item   **icurr;
442     int    ileft;
443 
444     char   **suffixlist;
445     int    suffixsize;
446     char   **suffixcurr;
447     int    suffixleft;
448 
449     enum protocol protocol;   /* which protocol this connection speaks */
450     enum network_transport transport; /* what transport is used by this connection */
451 
452     /* data for UDP clients */
453     int    request_id; /* Incoming UDP request ID, if this is a UDP "connection" */
454     struct sockaddr request_addr; /* Who sent the most recent request */
455     socklen_t request_addr_size;
456     unsigned char *hdrbuf; /* udp packet headers */
457     int    hdrsize;   /* number of headers' worth of space is allocated */
458 
459     bool   noreply;   /* True if the reply should not be sent. */
460     /* current stats command */
461     struct {
462         char *buffer;
463         size_t size;
464         size_t offset;
465     } stats;
466 
467     /* Binary protocol stuff */
468     /* This is where the binary header goes */
469     protocol_binary_request_header binary_header;
470     uint64_t cas; /* the cas to return */
471     short cmd; /* current command being processed */
472     int opaque;
473     int keylen;
474     conn *next;     /* Used for generating a list of conn structures */
475     LIBEVENT_THREAD *thread; /* Pointer to the thread object serving this connection */
476 
477     conn_funcs *funcs;
478     void *extra;
479 
480     protocol_binary_command cmd_curr;
481 
482     char     *cmd_start;      /* Pointer into rbuf, snapshot of rcurr. */
483     uint64_t  cmd_start_time; /* Snapshot of usec_now or msec_current_time. */
484     int       cmd_retries;
485 
486     bool      hit_local;
487     bool      cmd_unpaused;
488     uint64_t  cmd_arrive_time;
489 
490     bin_cmd *corked;
491 
492     char *host_ident; /* Uniquely identifies a memcached server, including */
493                       /* address:port and possibly optional bucket/usr/pwd info. */
494     char *peer_host;    /* this and the following two paramters are used for mcmux */
495     unsigned int peer_protocol;  /* compatiblity mode */
496     int peer_port;
497 
498     const char *update_diag;
499 };
500 
501 extern conn *listen_conn;
502 
503 typedef struct token_s {
504     char *value;
505     size_t length;
506 } token_t;
507 
508 size_t tokenize_command(char *command, token_t *tokens,
509                         const size_t max_tokens);
510 
511 /* current time of day (updated periodically) */
512 extern volatile rel_time_t current_time;
513 
514 /*
515  * Functions
516  */
517 void do_accept_new_conns(const bool do_accept);
518 enum delta_result_type do_add_delta(conn *c, item *item, const bool incr,
519                                     const int64_t delta, char *buf);
520 enum store_item_type do_store_item(item *item, int comm, conn* c);
521 
522 conn *conn_new(const SOCKET sfd, const enum conn_states init_state,
523                const int event_flags, const int read_buffer_size,
524                enum network_transport transport,
525                struct event_base *base,
526                conn_funcs *funcs, void *extra);
527 
528 void conn_set_state(conn* c, enum conn_states state);
529 void add_bytes_read(conn *c, int bytes_read);
530 void out_string(conn *c, const char *str);
531 
532 #define ___update_event_str(a) #a
533 #define __update_event_str(a) ___update_event_str(a)
534 
535 #define update_event(c, new_flags) update_event_real(c, new_flags, (__FILE__ ":" __update_event_str(__LINE__)))
536 #define update_event_timed(c, new_flags, timeout) update_event_timed_real(c, new_flags, timeout, (__FILE__ ":" __update_event_str(__LINE__)))
537 
538 bool update_event_real(conn *c, const int new_flags, const char *update_diag);
539 bool update_event_timed_real(conn *c, const int new_flags, struct timeval *timeout, const char *update_diag);
540 
541 int try_read_command(conn *c);
542 void process_command(conn *c, char *command);
543 void process_update_command(conn *c, token_t *tokens, const size_t ntokens, int comm, bool handle_cas);
544 void process_stats_proxy_command(conn *c, token_t *tokens, const size_t ntokens);
545 void process_verbosity_command(conn *c, token_t *tokens, const size_t ntokens);
546 void dispatch_bin_command(conn *c);
547 void process_bin_noreply(conn *c);
548 void bin_read_key(conn *c, enum bin_substates next_substate, int extra);
549 char* binary_get_key(conn *c);
550 void* binary_get_request(conn *c);
551 uint64_t mc_swap64(uint64_t in);
552 void reset_cmd_handler(conn *c);
553 void complete_nread(conn *c);
554 void complete_nread_binary(conn *c);
555 void complete_nread_ascii(conn *c);
556 int ensure_iov_space(conn *c);
557 int add_iov(conn *c, const void *buf, int len);
558 int add_msghdr(conn *c);
559 void set_noreply_maybe(conn *c, token_t *tokens, size_t ntokens);
560 
561 const char *state_text(enum conn_states state);
562 
563 #ifndef WIN32
564 extern int daemonize(int nochdir, int noclose);
565 #endif
566 
567 int server_socket(int port,
568                   enum network_transport transport,
569                   FILE *portnum_file);
570 
571 #ifdef HAVE_SYS_UN_H
572 int server_socket_unix(const char *path, int access_mask);
573 #endif
574 
575 void drive_machine(conn *c);
576 
577 void write_bin_response(conn *c, void *d, int hlen, int keylen, int dlen);
578 void write_bin_error(conn *c, protocol_binary_response_status err, int swallow);
579 
580 #include "stats.h"
581 #include "slabs.h"
582 #include "assoc.h"
583 #include "items.h"
584 #include "trace.h"
585 #include "hash.h"
586 #include "util.h"
587 
588 /*
589  * Functions such as the libevent-related calls that need to do cross-thread
590  * communication in multithreaded mode (rather than actually doing the work
591  * in the current thread) are called via "dispatch_" frontends, which are
592  * also #define-d to directly call the underlying code in singlethreaded mode.
593  */
594 
595 void thread_init(int nthreads, struct event_base *main_base);
596 int  thread_index(cb_thread_t thread_id);
597 LIBEVENT_THREAD *thread_by_index(int i);
598 
599 int  dispatch_event_add(int thread, conn *c);
600 
601 void dispatch_conn_new(SOCKET sfd, enum conn_states init_state,
602                        int event_flags,
603                        int read_buffer_size,
604                        enum protocol prot,
605                        enum network_transport transport,
606                        conn_funcs *funcs, void *extra);
607 
608 void dispatch_conn_new_to_thread(int tid, SOCKET sfd, enum conn_states init_state,
609                                  int event_flags,
610                                  int read_buffer_size,
611                                  enum protocol prot,
612                                  enum network_transport transport,
613                                  conn_funcs *funcs, void *extra);
614 
615 /* Lock wrappers for cache functions that are called from main loop. */
616 enum delta_result_type add_delta(conn *c, item *item, const int incr,
617                                  const int64_t delta, char *buf);
618 void accept_new_conns(const bool do_accept);
619 conn *conn_from_freelist(void);
620 bool  conn_add_to_freelist(conn *c);
621 int   is_listen_thread(void);
622 item *item_alloc(char *key, size_t nkey, int flags, rel_time_t exptime, int nbytes);
623 char *item_cachedump(const unsigned int slabs_clsid, const unsigned int limit, unsigned int *bytes);
624 void  item_flush_expired(void);
625 item *item_get(const char *key, const size_t nkey);
626 int   item_link(item *it);
627 void  item_remove(item *it);
628 int   item_replace(item *it, item *new_it);
629 void  item_stats(ADD_STAT add_stats, void *c);
630 void  item_stats_sizes(ADD_STAT add_stats, void *c);
631 void  item_unlink(item *it);
632 void  item_update(item *it);
633 
634 void STATS_LOCK(void);
635 void STATS_UNLOCK(void);
636 void threadlocal_stats_reset(void);
637 void threadlocal_stats_aggregate(struct thread_stats *stats);
638 void slab_stats_aggregate(struct thread_stats *stats, struct slab_stats *out);
639 
640 /* Stat processing functions */
641 void append_stat(const char *name, ADD_STAT add_stats, void *c,
642                  const char *fmt, ...);
643 
644 void append_prefix_stat(const char *prefix, const char *name, ADD_STAT add_stats, void *c,
645                         const char *fmt, ...);
646 
647 void server_stats(ADD_STAT add_stats, void *c, const char *prefix);
648 void process_stat_settings(ADD_STAT add_stats, void *c, const char *prefix);
649 
650 enum store_item_type store_item(item *item, int comm, conn *c);
651 void initialize_conn_lock(void);
652 
653 #ifdef HAVE_DROP_PRIVILEGES
654 extern void drop_privileges(void);
655 #else
656 #define drop_privileges()
657 #endif
658 
659 /* If supported, give compiler hints for branch prediction. */
660 #if !defined(__GNUC__) || (__GNUC__ == 2 && __GNUC_MINOR__ < 96)
661 #define __builtin_expect(x, expected_value) (x)
662 #endif
663 
664 #define likely(x)       __builtin_expect((x),1)
665 #define unlikely(x)     __builtin_expect((x),0)
666 
667 /* used by unit tests */
668 void start_main(char *arg0, ...);
669 
670 void process_bin_proxy_stats(conn *c);
671