1 /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2 
3 #include "testapp.h"
4 
5 #include "ssl_impl.h"
6 
7 #include <JSON_checker.h>
8 #include <cbsasl/client.h>
9 #include <folly/portability/GTest.h>
10 #include <folly/portability/SysTypes.h>
11 #include <mcbp/protocol/framebuilder.h>
12 #include <platform/backtrace.h>
13 #include <platform/cb_malloc.h>
14 #include <platform/cbassert.h>
15 #include <platform/dirutils.h>
16 #include <platform/socket.h>
17 #include <platform/strerror.h>
18 #include <platform/string_hex.h>
19 #include <gsl/gsl>
20 
21 #include <getopt.h>
22 #include <platform/compress.h>
23 #include <fstream>
24 
25 #include <include/memcached/protocol_binary.h>
26 #include <nlohmann/json.hpp>
27 #include <protocol/mcbp/ewb_encode.h>
28 #include <utilities/json_utilities.h>
29 #include <atomic>
30 #include <chrono>
31 #include <csignal>
32 #include <thread>
33 
34 McdEnvironment* mcd_env = nullptr;
35 
36 /* test phases (bitmasks) */
37 #define phase_plain 0x2
38 #define phase_ssl 0x4
39 
40 #define phase_max 4
41 static int current_phase = 0;
42 
43 pid_t server_pid = -1;
44 in_port_t port = -1;
45 in_port_t ssl_port = -1;
46 SOCKET sock = INVALID_SOCKET;
47 SOCKET sock_ssl;
48 static std::atomic<bool> allow_closed_read;
49 static time_t server_start_time = 0;
50 
51 // used in embedded mode to shutdown memcached
52 extern void shutdown_server();
53 
54 std::set<cb::mcbp::Feature> enabled_hello_features;
55 
56 int memcached_verbose = 0;
57 // State variable if we're running the memcached server in a
58 // thread in the same process or not
59 static bool embedded_memcached_server;
60 
61 /* static storage for the different environment variables set by
62  * putenv().
63  *
64  * (These must be static as putenv() essentially 'takes ownership' of
65  * the provided array, so it is unsafe to use an automatic variable.
66  * However, if we use the result of cb_malloc() (i.e. the heap) then
67  * memory leak checkers (e.g. Valgrind) will report the memory as
68  * leaked as it's impossible to free it).
69  */
70 static char mcd_parent_monitor_env[80];
71 static char mcd_port_filename_env[80];
72 
73 static SOCKET connect_to_server_ssl(in_port_t ssl_port);
74 
set_allow_closed_read(bool enabled)75 void set_allow_closed_read(bool enabled) {
76     allow_closed_read = enabled;
77 }
78 
sock_is_ssl()79 bool sock_is_ssl() {
80     return current_phase == phase_ssl;
81 }
82 
get_server_start_time()83 time_t get_server_start_time() {
84     return server_start_time;
85 }
86 
operator <<(std::ostream& os, const TransportProtocols& t)87 std::ostream& operator<<(std::ostream& os, const TransportProtocols& t) {
88     os << to_string(t);
89     return os;
90 }
91 
to_string(const TransportProtocols& transport)92 std::string to_string(const TransportProtocols& transport) {
93 #ifdef JETBRAINS_CLION_IDE
94     // CLion don't properly parse the output when the
95     // output gets written as the string instead of the
96     // number. This makes it harder to debug the tests
97     // so let's just disable it while we're waiting
98     // for them to supply a fix.
99     // See https://youtrack.jetbrains.com/issue/CPP-6039
100     return std::to_string(static_cast<int>(transport));
101 #else
102     switch (transport) {
103     case TransportProtocols::McbpPlain:
104         return "Mcbp";
105     case TransportProtocols::McbpIpv6Plain:
106         return "McbpIpv6";
107     case TransportProtocols::McbpSsl:
108         return "McbpSsl";
109     case TransportProtocols::McbpIpv6Ssl:
110         return "McbpIpv6Ssl";
111     }
112     throw std::logic_error("Unknown transport");
113 #endif
114 }
115 
to_string(ClientJSONSupport json)116 std::string to_string(ClientJSONSupport json) {
117     switch (json) {
118     case ClientJSONSupport::Yes:
119         return "JsonYes";
120     case ClientJSONSupport::No:
121         return "JsonNo";
122     }
123     throw std::logic_error("Unknown JSON support");
124 }
125 
to_string(ClientSnappySupport snappy)126 std::string to_string(ClientSnappySupport snappy) {
127     switch (snappy) {
128     case ClientSnappySupport::Yes:
129         return "SnappyYes";
130     case ClientSnappySupport::No:
131         return "SnappyNo";
132     }
133     throw std::logic_error("Unknown ClientSnappySupport value: " +
134                            std::to_string(int(snappy)));
135 }
136 
CreateTestBucket()137 void TestappTest::CreateTestBucket() {
138     auto& conn = connectionMap.getConnection(false);
139 
140     // Reconnect to the server so we know we're on a "fresh" connection
141     // to the server (and not one that might have been cached by the
142     // idle-timer, but not yet noticed on the client side)
143     conn.reconnect();
144     conn.authenticate("@admin", "password", "PLAIN");
145 
146     mcd_env->getTestBucket().setUpBucket(bucketName, "", conn);
147 
148     // Reconnect the object to avoid others to reuse the admin creds
149     conn.reconnect();
150 }
151 
DeleteTestBucket()152 void TestappTest::DeleteTestBucket() {
153     auto& conn = connectionMap.getConnection(false);
154     conn.reconnect();
155     conn.authenticate("@admin", "password", "PLAIN");
156     try {
157         conn.deleteBucket(bucketName);
158     } catch (const ConnectionError& error) {
159         EXPECT_FALSE(error.isNotFound()) << "Delete bucket [" << bucketName
160                                          << "] failed with: " << error.what();
161     }
162 }
163 
GetTestBucket()164 TestBucketImpl& TestappTest::GetTestBucket() {
165     return mcd_env->getTestBucket();
166 }
167 
168 // Per-test-case set-up.
169 // Called before the first test in this test case.
SetUpTestCase()170 void TestappTest::SetUpTestCase() {
171     token = 0xdeadbeef;
172     memcached_cfg = generate_config(0);
173     start_memcached_server();
174 
175     if (HasFailure()) {
176         std::cerr << "Error in TestappTest::SetUpTestCase, terminating process"
177                   << std::endl;
178 
179         exit(EXIT_FAILURE);
180     } else {
181         CreateTestBucket();
182     }
183 }
184 
185 // Per-test-case tear-down.
186 // Called after the last test in this test case.
TearDownTestCase()187 void TestappTest::TearDownTestCase() {
188     if (sock != INVALID_SOCKET) {
189         cb::net::closesocket(sock);
190     }
191 
192     if (server_pid != -1) {
193         DeleteTestBucket();
194     }
195     stop_memcached_server();
196 }
197 
reconfigure_client_cert_auth(const std::string& state, const std::string& path, const std::string& prefix, const std::string& delimiter)198 void TestappTest::reconfigure_client_cert_auth(const std::string& state,
199                                                const std::string& path,
200                                                const std::string& prefix,
201                                                const std::string& delimiter) {
202     memcached_cfg["client_cert_auth"] = {};
203     memcached_cfg["client_cert_auth"]["state"] = state;
204     memcached_cfg["client_cert_auth"]["path"] = path;
205     memcached_cfg["client_cert_auth"]["prefix"] = prefix;
206     memcached_cfg["client_cert_auth"]["delimiter"] = delimiter;
207     // update the server to use this!
208     reconfigure();
209 }
210 
setClientCertData(MemcachedConnection& connection)211 void TestappTest::setClientCertData(MemcachedConnection& connection) {
212     connection.setSslCertFile(SOURCE_ROOT +
213                               std::string("/tests/cert/client.pem"));
214     connection.setSslKeyFile(SOURCE_ROOT +
215                              std::string("/tests/cert/client.key"));
216 }
217 
get_sasl_mechs()218 std::string get_sasl_mechs() {
219     using namespace cb::mcbp;
220     Request request = {};
221     request.setMagic(Magic::ClientRequest);
222     request.setOpcode(ClientOpcode::SaslListMechs);
223     request.setOpaque(0xdeadbeef);
224 
225     safe_send(request.getFrame());
226 
227     std::vector<uint8_t> blob;
228     safe_recv_packet(blob);
229     auto& response = *reinterpret_cast<Response*>(blob.data());
230     mcbp_validate_response_header(response,
231                                   cb::mcbp::ClientOpcode::SaslListMechs,
232                                   cb::mcbp::Status::Success);
233 
234     auto val = response.getValue();
235     return std::string{reinterpret_cast<const char*>(val.data()), val.size()};
236 }
237 
sasl_auth(const char* username, const char* password)238 cb::mcbp::Status TestappTest::sasl_auth(const char* username,
239                                         const char* password) {
240     cb::sasl::client::ClientContext client(
241             [username]() -> std::string { return username; },
242             [password]() -> std::string { return password; },
243             get_sasl_mechs());
244 
245     auto client_data = client.start();
246     EXPECT_EQ(cb::sasl::Error::OK, client_data.first);
247     if (::testing::Test::HasFailure()) {
248         // Can't continue if we didn't suceed in starting SASL auth.
249         return cb::mcbp::Status::Einternal;
250     }
251 
252     std::vector<uint8_t> blob;
253     {
254         BinprotSaslAuthCommand cmd;
255         cmd.setMechanism(client.getName());
256         cmd.setChallenge(client_data.second);
257         cmd.encode(blob);
258         safe_send(blob);
259     }
260 
261     blob.resize(0);
262     safe_recv_packet(blob);
263 
264     bool stepped = false;
265     auto* response = reinterpret_cast<cb::mcbp::Response*>(blob.data());
266     while (response->getStatus() == cb::mcbp::Status::AuthContinue) {
267         stepped = true;
268         auto challenge = response->getValue();
269         client_data = client.step(cb::const_char_buffer{
270                 reinterpret_cast<const char*>(challenge.data()),
271                 challenge.size()});
272         EXPECT_EQ(cb::sasl::Error::CONTINUE, client_data.first);
273         BinprotSaslStepCommand cmd;
274         cmd.setMechanism(client.getName());
275         cmd.setChallenge(client_data.second);
276         blob.resize(0);
277         cmd.encode(blob);
278 
279         safe_send(blob);
280         blob.resize(0);
281         safe_recv_packet(blob);
282         response = reinterpret_cast<cb::mcbp::Response*>(blob.data());
283     }
284 
285     if (stepped) {
286         mcbp_validate_response_header(*response,
287                                       cb::mcbp::ClientOpcode::SaslStep,
288                                       response->getStatus());
289     } else {
290         mcbp_validate_response_header(*response,
291                                       cb::mcbp::ClientOpcode::SaslAuth,
292                                       response->getStatus());
293     }
294 
295     return response->getStatus();
296 }
297 
isJSON(cb::const_char_buffer value)298 bool TestappTest::isJSON(cb::const_char_buffer value) {
299     JSON_checker::Validator validator;
300     const auto* ptr = reinterpret_cast<const uint8_t*>(value.data());
301     return validator.validate(ptr, value.size());
302 }
303 
304 // per test setup function.
SetUp()305 void TestappTest::SetUp() {
306     verify_server_running();
307     current_phase = phase_plain;
308     sock = connect_to_server_plain(port);
309     ASSERT_NE(INVALID_SOCKET, sock);
310 
311     // Set ewouldblock_engine test harness to default mode.
312     ewouldblock_engine_configure(ENGINE_EWOULDBLOCK,
313                                  EWBEngineMode::First,
314                                  /*unused*/ 0);
315 
316     enabled_hello_features.clear();
317 
318     const auto* info = ::testing::UnitTest::GetInstance()->current_test_info();
319     name.assign(info->test_case_name());
320     name.append("_");
321     name.append(info->name());
322     std::replace(name.begin(), name.end(), '/', '_');
323 }
324 
325 // per test tear-down function.
TearDown()326 void TestappTest::TearDown() {
327     cb::net::closesocket(sock);
328 }
329 
330 const std::string TestappTest::bucketName = "default";
331 
332 // per test setup function.
SetUp()333 void McdTestappTest::SetUp() {
334     verify_server_running();
335     if (getProtocolParam() == TransportProtocols::McbpPlain) {
336         current_phase = phase_plain;
337         sock = connect_to_server_plain(port);
338         ASSERT_NE(INVALID_SOCKET, sock);
339     } else {
340         current_phase = phase_ssl;
341         sock_ssl = connect_to_server_ssl(ssl_port);
342         ASSERT_NE(INVALID_SOCKET, sock_ssl);
343     }
344 
345     set_json_feature(hasJSONSupport() == ClientJSONSupport::Yes);
346 
347     // Set ewouldblock_engine test harness to default mode.
348     ewouldblock_engine_configure(ENGINE_EWOULDBLOCK,
349                                  EWBEngineMode::First,
350                                  /*unused*/ 0);
351 
352     setCompressionMode("off");
353 }
354 
355 // per test tear-down function.
TearDown()356 void McdTestappTest::TearDown() {
357     if (getProtocolParam() == TransportProtocols::McbpPlain) {
358         cb::net::closesocket(sock);
359     } else {
360         cb::net::closesocket(sock_ssl);
361         sock_ssl = INVALID_SOCKET;
362         destroy_ssl_socket();
363     }
364 }
365 
hasJSONSupport() const366 ClientJSONSupport McdTestappTest::hasJSONSupport() const {
367     return getJSONParam();
368 }
369 
setCompressionMode(const std::string& compression_mode)370 void TestappTest::setCompressionMode(const std::string& compression_mode) {
371     mcd_env->getTestBucket().setCompressionMode(
372             getAdminConnection(), bucketName, compression_mode);
373 }
374 
setMinCompressionRatio(float min_compression_ratio)375 void TestappTest::setMinCompressionRatio(float min_compression_ratio) {
376     mcd_env->getTestBucket().setMinCompressionRatio(
377             getAdminConnection(),
378             bucketName,
379             std::to_string(min_compression_ratio));
380 }
381 
waitForAtLeastSeqno(Vbid vbid, uint64_t uuid, uint64_t seqno)382 void TestappTest::waitForAtLeastSeqno(Vbid vbid,
383                                       uint64_t uuid,
384                                       uint64_t seqno) {
385     ASSERT_TRUE(mcd_env->getTestBucket().supportsPersistence())
386             << "Error: your bucket does not support persistence";
387 
388     // Poll for that sequence number to be persisted.
389     ObserveInfo observe;
390     MemcachedConnection& conn = getConnection();
391     do {
392         observe = conn.observeSeqno(vbid, uuid);
393         EXPECT_EQ(0, observe.formatType);
394         EXPECT_EQ(vbid, observe.vbId);
395         EXPECT_EQ(uuid, observe.uuid);
396 
397         if (observe.lastPersistedSeqno < seqno) {
398             // Don't busy wait, yield a little. A 2019 poll of sleeps in KV
399             // reveals 100us to be our most popular wait time.
400             std::this_thread::sleep_for(std::chrono::microseconds(100));
401         } else {
402             break;
403         }
404     } while (true);
405 }
406 
storeAndPersistItem(Vbid vbid, std::string key)407 Document TestappTest::storeAndPersistItem(Vbid vbid, std::string key) {
408     MemcachedConnection& conn = getConnection();
409     conn.setMutationSeqnoSupport(true);
410     Document doc;
411     doc.info.id = key;
412     doc.value = "persist me";
413     auto mutation = conn.mutate(doc, vbid, MutationType::Set);
414     EXPECT_NE(0, mutation.seqno);
415     EXPECT_NE(0, mutation.vbucketuuid);
416     doc.info.cas = mutation.cas;
417 
418     waitForAtLeastSeqno(vbid, mutation.vbucketuuid, mutation.seqno);
419 
420     return doc;
421 }
422 
PrintToStringCombinedName( const ::testing::TestParamInfo< ::testing::tuple<TransportProtocols, ClientJSONSupport>>& info)423 std::string McdTestappTest::PrintToStringCombinedName(
424         const ::testing::TestParamInfo<
425                 ::testing::tuple<TransportProtocols, ClientJSONSupport>>&
426                 info) {
427     return to_string(::testing::get<0>(info.param)) + "_" +
428            to_string(::testing::get<1>(info.param));
429 }
430 
CERTIFICATE_PATH(const std::string& file)431 std::string CERTIFICATE_PATH(const std::string& file) {
432 #ifdef WIN32
433     return std::string("\\tests\\cert\\") + file;
434 #else
435     return std::string("/tests/cert/") + file;
436 #endif
437 }
get_errmaps_dir()438 static std::string get_errmaps_dir() {
439     std::string dir(SOURCE_ROOT);
440     dir += "/etc/couchbase/kv/error_maps";
441     cb::io::sanitizePath(dir);
442     return dir;
443 }
444 
generate_config(uint16_t ssl_port)445 nlohmann::json TestappTest::generate_config(uint16_t ssl_port) {
446     const std::string cwd = cb::io::getcwd();
447     const std::string pem_path = cwd + CERTIFICATE_PATH("testapp.pem");
448     const std::string cert_path = cwd + CERTIFICATE_PATH("testapp.cert");
449 
450     nlohmann::json ret = {
451             {"max_connections", Testapp::MAX_CONNECTIONS},
452             {"system_connections", Testapp::MAX_CONNECTIONS / 4},
453             {"stdin_listener", false},
454             {"datatype_json", true},
455             {"datatype_snappy", true},
456             {"xattr_enabled", true},
457             {"dedupe_nmvb_maps", false},
458             {"active_external_users_push_interval", "30 m"},
459             {"error_maps_dir", get_errmaps_dir()},
460             {"audit_file", mcd_env->getAuditFilename()},
461             {"rbac_file", mcd_env->getRbacFilename()},
462             {"ssl_cipher_list",
463              {{"tls 1.2", "HIGH"},
464               {"tls 1.3",
465                "TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256:TLS_AES_"
466                "128_GCM_SHA256:TLS_AES_128_CCM_8_SHA256:TLS_AES_128_CCM_"
467                "SHA256"}}},
468             {"ssl_minimum_protocol", "tlsv1"},
469             {"opcode_attributes_override",
470              {{"version", 1}, {"EWB_CTL", {{"slow", 50}}}}},
471             {"logger", {{"unit_test", true}}},
472     };
473 
474     if (memcached_verbose == 0) {
475         ret["logger"]["console"] = false;
476     } else {
477         ret["verbosity"] = memcached_verbose - 1;
478     }
479 
480     // For simplicity in the test to check for max connections we mark the
481     // SSL port as an admin port
482     ret["interfaces"][0] = {{"tag", "plain"},
483                             {"system", false},
484                             {"port", 0},
485                             {"ipv4", "required"},
486                             {"ipv6", "required"},
487                             {"host", "*"}};
488 
489     ret["interfaces"][1] = {{"tag", "ssl"},
490                             {"system", true},
491                             {"port", ssl_port},
492                             {"ipv4", "required"},
493                             {"ipv6", "required"},
494                             {"host", "*"},
495                             {"ssl", {{"key", pem_path}, {"cert", cert_path}}}};
496 
497     return ret;
498 }
499 
write_config_to_file(const std::string& config, const std::string& fname)500 void write_config_to_file(const std::string& config, const std::string& fname) {
501     FILE* fp = fopen(fname.c_str(), "w");
502 
503     if (fp == nullptr) {
504         throw std::system_error(errno,
505                                 std::system_category(),
506                                 "Failed to open file \"" + fname + "\"");
507     } else {
508         fprintf(fp, "%s", config.c_str());
509         fclose(fp);
510     }
511 }
512 
513 #ifdef WIN32
pidTToHandle(pid_t pid)514 HANDLE TestappTest::pidTToHandle(pid_t pid) {
515     return reinterpret_cast<HANDLE>(static_cast<size_t>(pid));
516 }
517 
handleToPidT(HANDLE handle)518 pid_t TestappTest::handleToPidT(HANDLE handle) {
519     return static_cast<pid_t>(reinterpret_cast<size_t>(handle));
520 }
521 #endif
522 
verify_server_running()523 void TestappTest::verify_server_running() {
524     if (embedded_memcached_server) {
525         // we don't monitor this thread...
526         return;
527     }
528 
529     if (-1 == server_pid) {
530         std::cerr << "Server not running (server_pid == -1)" << std::endl;
531         exit(EXIT_FAILURE);
532     }
533 
534 #ifdef WIN32
535     DWORD status;
536 
537     if (!GetExitCodeProcess(pidTToHandle(server_pid), &status)) {
538         std::cerr << "GetExitCodeProcess: failed: " << cb_strerror()
539                   << std::endl;
540         exit(EXIT_FAILURE);
541     }
542     if (status != STILL_ACTIVE) {
543         std::cerr << "memcached process is not active: Exit code " << status
544                   << std::endl;
545         exit(EXIT_FAILURE);
546     }
547 #else
548     int status;
549     pid_t ret = waitpid(server_pid, &status, WNOHANG);
550 
551     if (ret == static_cast<pid_t>(-1)) {
552         std::cerr << "waitpid() failed with: " << strerror(errno) << std::endl;
553         exit(EXIT_FAILURE);
554     }
555 
556     if (server_pid == ret) {
557         std::cerr << "waitpid status     : " << status << std::endl
558                   << "WIFEXITED(status)  : " << WIFEXITED(status) << std::endl
559                   << "WEXITSTATUS(status): " << WEXITSTATUS(status) << std::endl
560                   << "WIFSIGNALED(status): " << WIFSIGNALED(status) << std::endl
561                   << "WTERMSIG(status)   : " << WTERMSIG(status) << std::endl
562                   << "WCOREDUMP(status)  : " << WCOREDUMP(status) << std::endl;
563         exit(EXIT_FAILURE);
564     }
565 #endif
566 }
567 
parse_portnumber_file()568 void TestappTest::parse_portnumber_file() {
569     // I've seen that running under valgrind startup of the processes
570     // might be slow, and getting even worse if the machine is under
571     // load. Instead of having a "false timeout" just because the
572     // server is slow, lets set the deadline to a high value so that
573     // if we hit it we have a real problem and not just a loaded
574     // server (rebuilding all of the source one more time is just
575     // putting more load on the servers).
576     using std::chrono::steady_clock;
577     const auto timeout = steady_clock::now() + std::chrono::minutes(5);
578 
579     do {
580         std::this_thread::sleep_for(std::chrono::milliseconds(10));
581         if (cb::io::isFile(portnumber_file)) {
582             break;
583         }
584 
585         verify_server_running();
586     } while (steady_clock::now() < timeout);
587 
588     ASSERT_TRUE(cb::io::isFile(portnumber_file))
589             << "Timed out after 5 minutes waiting for memcached port file '"
590             << portnumber_file << "' to be created.";
591 
592     // We'll throw here if anything goes wrong
593     const auto json = nlohmann::json::parse(cb::io::loadFile(portnumber_file));
594 
595     connectionMap.initialize(json);
596 
597     // The tests which don't use the MemcachedConnection class needs the
598     // global variables port and ssl_port to be set
599     port = (in_port_t)-1;
600     ssl_port = (in_port_t)-1;
601 
602     connectionMap.iterate([](MemcachedConnection& connection) {
603         if (connection.getFamily() == AF_INET) {
604             if (connection.isSsl()) {
605                 ssl_port = connection.getPort();
606             } else {
607                 port = connection.getPort();
608             }
609         }
610     });
611 
612     if (port == in_port_t(-1)) {
613         throw std::runtime_error(
614                 "parse_portnumber_file: Failed to locate an plain IPv4 "
615                 "connection");
616     }
617 
618     if (ssl_port == in_port_t(-1)) {
619         throw std::runtime_error(
620                 "parse_portnumber_file: Failed to locate a SSL IPv4 "
621                 "connection");
622     }
623 
624     EXPECT_EQ(0, remove(portnumber_file.c_str()));
625 }
626 
627 extern "C" int memcached_main(int argc, char** argv);
628 
memcached_server_thread_main(void* arg)629 extern "C" void memcached_server_thread_main(void* arg) {
630     char* argv[4];
631     int argc = 0;
632     argv[argc++] = const_cast<char*>("./memcached");
633     argv[argc++] = const_cast<char*>("-C");
634     argv[argc++] = reinterpret_cast<char*>(arg);
635 
636     // Reset getopt()'s optind so memcached_main starts from the first
637     // argument.
638     optind = 1;
639 
640     memcached_main(argc, argv);
641 }
642 
spawn_embedded_server()643 void TestappTest::spawn_embedded_server() {
644     char* filename = mcd_port_filename_env + strlen("MEMCACHED_PORT_FILENAME=");
645     snprintf(mcd_port_filename_env,
646              sizeof(mcd_port_filename_env),
647              "MEMCACHED_PORT_FILENAME=memcached_ports.%u.%lu",
648              (int)getpid(),
649              (unsigned long)time(NULL));
650     remove(filename);
651     portnumber_file.assign(filename);
652     putenv(mcd_port_filename_env);
653 
654     ASSERT_EQ(0,
655               cb_create_thread(&memcached_server_thread,
656                                memcached_server_thread_main,
657                                const_cast<char*>(config_file.c_str()),
658                                0));
659 }
660 
start_external_server()661 void TestappTest::start_external_server() {
662     char* filename = mcd_port_filename_env + strlen("MEMCACHED_PORT_FILENAME=");
663     snprintf(mcd_parent_monitor_env,
664              sizeof(mcd_parent_monitor_env),
665              "MEMCACHED_PARENT_MONITOR=%u",
666              (int)getpid());
667     putenv(mcd_parent_monitor_env);
668 
669     snprintf(mcd_port_filename_env,
670              sizeof(mcd_port_filename_env),
671              "MEMCACHED_PORT_FILENAME=memcached_ports.%u.%lu",
672              (int)getpid(),
673              (unsigned long)time(NULL));
674     remove(filename);
675     portnumber_file.assign(filename);
676     static char topkeys_env[] = "MEMCACHED_TOP_KEYS=10";
677     putenv(topkeys_env);
678 
679 #ifdef WIN32
680     STARTUPINFO sinfo;
681     PROCESS_INFORMATION pinfo;
682     memset(&sinfo, 0, sizeof(sinfo));
683     memset(&pinfo, 0, sizeof(pinfo));
684     sinfo.cb = sizeof(sinfo);
685 
686     char commandline[1024];
687     sprintf(commandline, "memcached.exe -C %s", config_file.c_str());
688 
689     putenv(mcd_port_filename_env);
690 
691     if (!CreateProcess("memcached.exe",
692                        commandline,
693                        NULL,
694                        NULL,
695                        /*bInheritHandles*/ FALSE,
696                        0,
697                        NULL,
698                        NULL,
699                        &sinfo,
700                        &pinfo)) {
701         std::cerr << "Failed to start process: " << cb_strerror() << std::endl;
702         exit(EXIT_FAILURE);
703     }
704 
705     server_pid = handleToPidT(pinfo.hProcess);
706 #else
707     server_pid = fork();
708     ASSERT_NE(reinterpret_cast<pid_t>(-1), server_pid);
709 
710     if (server_pid == 0) {
711         /* Child */
712         const char* argv[20];
713         int arg = 0;
714         putenv(mcd_port_filename_env);
715 
716         if (getenv("RUN_UNDER_VALGRIND") != nullptr) {
717             argv[arg++] = "valgrind";
718             argv[arg++] = "--log-file=valgrind.%p.log";
719             argv[arg++] = "--leak-check=full";
720 #if defined(__APPLE__)
721             /* Needed to ensure debugging symbols are up-to-date. */
722             argv[arg++] = "--dsymutil=yes";
723 #endif
724         }
725 
726         if (getenv("RUN_UNDER_PERF") != nullptr) {
727             argv[arg++] = "perf";
728             argv[arg++] = "record";
729             argv[arg++] = "--call-graph";
730             argv[arg++] = "dwarf";
731         }
732 
733         argv[arg++] = "./memcached";
734         argv[arg++] = "-C";
735         argv[arg++] = config_file.c_str();
736 
737         argv[arg++] = nullptr;
738         cb_assert(execvp(argv[0], const_cast<char**>(argv)) != -1);
739     }
740 #endif // !WIN32
741 }
742 
create_connect_plain_socket(in_port_t port)743 SOCKET create_connect_plain_socket(in_port_t port) {
744     if (port == in_port_t(-1)) {
745         throw std::runtime_error(
746                 "create_connect_plain_socket: Can't connect to port == -1");
747     }
748     auto sock = cb::net::new_socket("", port, AF_INET);
749     if (sock == INVALID_SOCKET) {
750         ADD_FAILURE() << "Failed to connect socket to 127.0.0.1:" << port;
751     }
752     return sock;
753 }
754 
connect_to_server_plain(in_port_t port)755 SOCKET connect_to_server_plain(in_port_t port) {
756     return create_connect_plain_socket(port);
757 }
758 
connect_to_server_ssl(in_port_t ssl_port)759 static SOCKET connect_to_server_ssl(in_port_t ssl_port) {
760     if (ssl_port == in_port_t(-1)) {
761         throw std::runtime_error(
762                 "connect_to_server_ssl: Can't connect to port == -1");
763     }
764     SOCKET sock = create_connect_ssl_socket(ssl_port);
765     if (sock == INVALID_SOCKET) {
766         ADD_FAILURE() << "Failed to connect SSL socket to port" << ssl_port;
767         return INVALID_SOCKET;
768     }
769 
770     return sock;
771 }
772 
773 /*
774     re-connect to server.
775     Uses global port and ssl_port values.
776     New socket-fd written to global "sock" and "ssl_bio"
777 */
reconnect_to_server()778 void reconnect_to_server() {
779     if (current_phase == phase_ssl) {
780         cb::net::closesocket(sock_ssl);
781         destroy_ssl_socket();
782 
783         sock_ssl = connect_to_server_ssl(ssl_port);
784         ASSERT_NE(INVALID_SOCKET, sock_ssl);
785     } else {
786         cb::net::closesocket(sock);
787         sock = connect_to_server_plain(port);
788         ASSERT_NE(INVALID_SOCKET, sock);
789     }
790 }
791 
set_feature(cb::mcbp::Feature feature, bool enable)792 static void set_feature(cb::mcbp::Feature feature, bool enable) {
793     // First update the currently enabled features.
794     if (enable) {
795         enabled_hello_features.insert(feature);
796     } else {
797         enabled_hello_features.erase(feature);
798     }
799 
800     BinprotHelloCommand command("testapp");
801     for (auto f : enabled_hello_features) {
802         command.enableFeature(f, true);
803     }
804 
805     std::vector<uint8_t> blob;
806     command.encode(blob);
807 
808     safe_send(blob);
809 
810     blob.resize(0);
811     safe_recv_packet(blob);
812     const auto& response = *reinterpret_cast<cb::mcbp::Response*>(blob.data());
813     ASSERT_EQ(cb::mcbp::Status::Success, response.getStatus());
814 
815     BinprotHelloResponse rsp;
816     rsp.assign(std::move(blob));
817 
818     auto enabled = rsp.getFeatures();
819     EXPECT_EQ(enabled_hello_features.size(), enabled.size());
820     for (auto f : enabled_hello_features) {
821         EXPECT_NE(enabled.end(), std::find(enabled.begin(), enabled.end(), f));
822     }
823 }
824 
set_datatype_feature(bool enable)825 void set_datatype_feature(bool enable) {
826     set_feature(cb::mcbp::Feature::JSON, enable);
827     set_feature(cb::mcbp::Feature::SNAPPY, enable);
828 }
829 
fetch_value(const std::string& key)830 std::pair<cb::mcbp::Status, std::string> fetch_value(const std::string& key) {
831     std::vector<uint8_t> blob;
832     BinprotGetCommand cmd;
833     cmd.setKey(key);
834     cmd.encode(blob);
835     safe_send(blob);
836 
837     blob.resize(0);
838     safe_recv_packet(blob);
839     BinprotGetResponse rsp;
840     rsp.assign(std::move(blob));
841     return std::make_pair(rsp.getStatus(), rsp.getDataString());
842 }
843 
validate_datatype_is_json(const std::string& key, bool isJson)844 void validate_datatype_is_json(const std::string& key, bool isJson) {
845     std::vector<uint8_t> blob;
846     BinprotGetCommand cmd;
847     cmd.setKey(key);
848     cmd.encode(blob);
849     safe_send(blob.data(), blob.size(), false);
850 
851     blob.resize(0);
852     safe_recv_packet(blob);
853     BinprotGetResponse rsp;
854     rsp.assign(std::move(blob));
855     ASSERT_EQ(cb::mcbp::Status::Success, rsp.getStatus());
856     EXPECT_EQ(isJson, rsp.getDatatype() & PROTOCOL_BINARY_DATATYPE_JSON);
857 }
858 
validate_json_document(const std::string& key, const std::string& expected_value)859 void validate_json_document(const std::string& key,
860                             const std::string& expected_value) {
861     auto pair = fetch_value(key);
862     EXPECT_EQ(cb::mcbp::Status::Success, pair.first);
863     const auto exp = nlohmann::json::parse(expected_value).dump();
864     const auto val = nlohmann::json::parse(pair.second).dump();
865     EXPECT_EQ(exp, val);
866 }
867 
validate_flags(const std::string& key, uint32_t expected_flags)868 void validate_flags(const std::string& key, uint32_t expected_flags) {
869     std::vector<uint8_t> blob;
870     BinprotGetCommand cmd;
871     cmd.setKey(key);
872     cmd.encode(blob);
873     safe_send(blob);
874 
875     blob.resize(0);
876     safe_recv_packet(blob);
877     BinprotGetResponse rsp;
878     rsp.assign(std::move(blob));
879     EXPECT_EQ(expected_flags, rsp.getDocumentFlags());
880 }
881 
delete_object(const std::string& key, bool ignore_missing)882 void delete_object(const std::string& key, bool ignore_missing) {
883     std::vector<uint8_t> blob;
884     BinprotRemoveCommand cmd;
885     cmd.setKey(key);
886     cmd.encode(blob);
887     safe_send(blob);
888 
889     blob.resize(0);
890     safe_recv_packet(blob);
891     BinprotRemoveResponse rsp;
892     rsp.assign(std::move(blob));
893 
894     if (ignore_missing && rsp.getStatus() == cb::mcbp::Status::KeyEnoent) {
895         /* Ignore. Just using this for cleanup then */
896         return;
897     }
898     mcbp_validate_response_header(
899             const_cast<cb::mcbp::Response&>(rsp.getResponse()),
900             cb::mcbp::ClientOpcode::Delete,
901             cb::mcbp::Status::Success);
902 }
903 
start_memcached_server()904 void TestappTest::start_memcached_server() {
905     config_file = cb::io::mktemp("memcached_testapp.json");
906     write_config_to_file(memcached_cfg.dump(2), config_file);
907 
908     server_start_time = time(0);
909 
910     if (embedded_memcached_server) {
911         spawn_embedded_server();
912     } else {
913         start_external_server();
914     }
915     parse_portnumber_file();
916 }
917 
store_object_w_datatype(const std::string& key, cb::const_char_buffer value, uint32_t flags, uint32_t expiration, cb::mcbp::Datatype datatype)918 void store_object_w_datatype(const std::string& key,
919                              cb::const_char_buffer value,
920                              uint32_t flags,
921                              uint32_t expiration,
922                              cb::mcbp::Datatype datatype) {
923     cb::mcbp::request::MutationPayload extras;
924     static_assert(sizeof(extras) == 8, "Unexpected extras size");
925     extras.setFlags(flags);
926     extras.setExpiration(expiration);
927 
928     std::vector<uint8_t> buffer(sizeof(cb::mcbp::Request) + sizeof(extras) +
929                                 key.size() + value.size());
930     cb::mcbp::FrameBuilder<cb::mcbp::Request> builder(
931             {buffer.data(), buffer.size()});
932 
933     builder.setMagic(cb::mcbp::Magic::ClientRequest);
934     builder.setOpcode(cb::mcbp::ClientOpcode::Set);
935     builder.setDatatype(datatype);
936     builder.setOpaque(0xdeadbeef);
937     builder.setExtras(extras.getBuffer());
938     builder.setKey({reinterpret_cast<const uint8_t*>(key.data()), key.size()});
939     builder.setValue(
940             {reinterpret_cast<const uint8_t*>(value.data()), value.size()});
941 
942     safe_send(builder.getFrame()->getFrame());
943 
944     std::vector<uint8_t> blob;
945     safe_recv_packet(blob);
946     mcbp_validate_response_header(
947             *reinterpret_cast<cb::mcbp::Response*>(blob.data()),
948             cb::mcbp::ClientOpcode::Set,
949             cb::mcbp::Status::Success);
950 }
951 
store_document(const std::string& key, const std::string& value, uint32_t flags, uint32_t exptime, bool compress)952 void store_document(const std::string& key,
953                     const std::string& value,
954                     uint32_t flags,
955                     uint32_t exptime,
956                     bool compress) {
957     if (compress) {
958         bool disable_snappy = false;
959         if (enabled_hello_features.count(cb::mcbp::Feature::SNAPPY) == 0) {
960             // We need to enable snappy
961             set_feature(cb::mcbp::Feature::SNAPPY, true);
962             disable_snappy = true;
963         }
964         cb::compression::Buffer deflated;
965         cb::compression::deflate(
966                 cb::compression::Algorithm::Snappy, value, deflated);
967 
968         store_object_w_datatype(key.c_str(),
969                                 deflated,
970                                 flags,
971                                 exptime,
972                                 cb::mcbp::Datatype::Snappy);
973         if (disable_snappy) {
974             set_feature(cb::mcbp::Feature::SNAPPY, false);
975         }
976     } else {
977         store_object_w_datatype(
978                 key.c_str(), value, flags, exptime, cb::mcbp::Datatype::Raw);
979     }
980 }
981 
set_json_feature(bool enable)982 void set_json_feature(bool enable) {
983     set_feature(cb::mcbp::Feature::JSON, enable);
984 }
985 
set_mutation_seqno_feature(bool enable)986 void set_mutation_seqno_feature(bool enable) {
987     set_feature(cb::mcbp::Feature::MUTATION_SEQNO, enable);
988 }
989 
set_xerror_feature(bool enable)990 void set_xerror_feature(bool enable) {
991     set_feature(cb::mcbp::Feature::XERROR, enable);
992 }
993 
waitForShutdown(bool killed)994 void TestappTest::waitForShutdown(bool killed) {
995 #ifdef WIN32
996     ASSERT_EQ(WAIT_OBJECT_0,
997               WaitForSingleObject(pidTToHandle(server_pid), 60000));
998     DWORD exit_code = NULL;
999     GetExitCodeProcess(pidTToHandle(server_pid), &exit_code);
1000     EXPECT_EQ(0, exit_code);
1001 #else
1002     int status;
1003     pid_t ret;
1004     while (true) {
1005         ret = waitpid(server_pid, &status, 0);
1006         if (ret == reinterpret_cast<pid_t>(-1) && errno == EINTR) {
1007             // Just loop again
1008             continue;
1009         }
1010         break;
1011     }
1012     ASSERT_NE(reinterpret_cast<pid_t>(-1), ret)
1013             << "waitpid failed: " << strerror(errno);
1014     bool correctShutdown = killed ? WIFSIGNALED(status) : WIFEXITED(status);
1015     EXPECT_TRUE(correctShutdown)
1016             << "waitpid status     : " << status << std::endl
1017             << "WIFEXITED(status)  : " << WIFEXITED(status) << std::endl
1018             << "WEXITSTATUS(status): " << WEXITSTATUS(status) << std::endl
1019             << "WIFSIGNALED(status): " << WIFSIGNALED(status) << std::endl
1020             << "WTERMSIG(status)   : " << WTERMSIG(status) << " ("
1021             << strsignal(WTERMSIG(status)) << ")" << std::endl
1022             << "WCOREDUMP(status)  : " << WCOREDUMP(status) << std::endl;
1023     EXPECT_EQ(0, WEXITSTATUS(status));
1024 #endif
1025     server_pid = pid_t(-1);
1026 }
1027 
stop_memcached_server()1028 void TestappTest::stop_memcached_server() {
1029     connectionMap.invalidate();
1030     if (sock != INVALID_SOCKET) {
1031         cb::net::closesocket(sock);
1032         sock = INVALID_SOCKET;
1033     }
1034 
1035     if (embedded_memcached_server) {
1036         shutdown_server();
1037         cb_join_thread(memcached_server_thread);
1038     }
1039 
1040     if (server_pid != pid_t(-1)) {
1041 #ifdef WIN32
1042         TerminateProcess(pidTToHandle(server_pid), 0);
1043         waitForShutdown();
1044 #else
1045         if (kill(server_pid, SIGTERM) == 0) {
1046             waitForShutdown();
1047         }
1048 #endif
1049     }
1050 
1051     if (!config_file.empty()) {
1052         EXPECT_NE(-1, remove(config_file.c_str()));
1053         config_file.clear();
1054     }
1055 }
1056 
socket_send(SOCKET s, const char* buf, size_t len)1057 ssize_t socket_send(SOCKET s, const char* buf, size_t len) {
1058     return cb::net::send(s, buf, len, 0);
1059 }
1060 
phase_send(const void* buf, size_t len)1061 static ssize_t phase_send(const void* buf, size_t len) {
1062     if (current_phase == phase_ssl) {
1063         return phase_send_ssl(buf, len);
1064     } else {
1065         return socket_send(sock, reinterpret_cast<const char*>(buf), len);
1066     }
1067 }
1068 
socket_recv(SOCKET s, char* buf, size_t len)1069 ssize_t socket_recv(SOCKET s, char* buf, size_t len) {
1070     return cb::net::recv(s, buf, len, 0);
1071 }
1072 
phase_recv(void* buf, size_t len)1073 ssize_t phase_recv(void* buf, size_t len) {
1074     if (current_phase == phase_ssl) {
1075         return phase_recv_ssl(buf, len);
1076     } else {
1077         return socket_recv(sock, reinterpret_cast<char*>(buf), len);
1078     }
1079 }
1080 
1081 static bool dump_socket_traffic = getenv("TESTAPP_PACKET_DUMP") != nullptr;
1082 
phase_get_errno()1083 static const std::string phase_get_errno() {
1084     if (current_phase == phase_ssl) {
1085         /* could do with more work here, but so far this has sufficed */
1086         return "SSL error";
1087     }
1088     return cb_strerror();
1089 }
1090 
safe_send(const void* buf, size_t len, bool hickup)1091 void safe_send(const void* buf, size_t len, bool hickup) {
1092     size_t offset = 0;
1093     const char* ptr = reinterpret_cast<const char*>(buf);
1094     do {
1095         size_t num_bytes = len - offset;
1096         ssize_t nw;
1097         if (hickup) {
1098             if (num_bytes > 1024) {
1099                 num_bytes = (rand() % 1023) + 1;
1100             }
1101         }
1102 
1103         nw = phase_send(ptr + offset, num_bytes);
1104 
1105         if (nw == -1) {
1106             if (errno != EINTR) {
1107                 fprintf(stderr,
1108                         "Failed to write: %s\n",
1109                         phase_get_errno().c_str());
1110                 print_backtrace_to_file(stderr);
1111                 abort();
1112             }
1113         } else {
1114             if (hickup) {
1115                 std::this_thread::sleep_for(std::chrono::microseconds(100));
1116             }
1117 
1118             if (dump_socket_traffic) {
1119                 if (current_phase == phase_ssl) {
1120                     std::cerr << "SSL";
1121                 } else {
1122                     std::cerr << "PLAIN";
1123                 }
1124                 std::cerr << "> ";
1125                 for (ssize_t ii = 0; ii < nw; ++ii) {
1126                     std::cerr << "0x" << std::hex << std::setfill('0')
1127                               << std::setw(2)
1128                               << uint32_t(*(uint8_t*)(ptr + offset + ii))
1129                               << ", ";
1130                 }
1131                 std::cerr << std::dec << std::endl;
1132             }
1133             offset += nw;
1134         }
1135     } while (offset < len);
1136 }
1137 
safe_recv(void* buf, size_t len)1138 bool safe_recv(void* buf, size_t len) {
1139     size_t offset = 0;
1140     if (len == 0) {
1141         return true;
1142     }
1143     do {
1144         ssize_t nr = phase_recv(((char*)buf) + offset, len - offset);
1145 
1146         if (nr == -1) {
1147             EXPECT_EQ(EINTR, errno) << "Failed to read: " << phase_get_errno();
1148         } else {
1149             if (nr == 0 && allow_closed_read) {
1150                 return false;
1151             }
1152             EXPECT_NE(0u, nr);
1153             offset += nr;
1154         }
1155 
1156         // Give up if we encountered an error.
1157         if (::testing::Test::HasFailure()) {
1158             return false;
1159         }
1160     } while (offset < len);
1161 
1162     return true;
1163 }
1164 
1165 /**
1166  * Internal function which receives a packet. The type parameter should have
1167  * these three functions:
1168  * - resize(n) ->: ensure the buffer has a total capacity for at least n bytes.
1169  *   This function will usually be called once for the header size (i.e. 24)
1170  *   and then another time for the total packet size (i.e. 24 + bodylen).
1171  * - data() -> char*: get the entire buffer
1172  * - size() -> size_t: get the current size of the buffer
1173  *
1174  * @param info an object conforming to the above.
1175  *
1176  * Once the function has completed, it will return true if no read errors
1177  * occurred. The actual size of the packet can be determined by parsing the
1178  * packet header.
1179  *
1180  * See StaticBufInfo which is an implementation that uses a fixed buffer.
1181  * std::vector naturally conforms to the interface.
1182  */
1183 template <typename T>
safe_recv_packetT(T& info)1184 bool safe_recv_packetT(T& info) {
1185     info.resize(sizeof(cb::mcbp::Response));
1186     auto* header = reinterpret_cast<cb::mcbp::Response*>(info.data());
1187 
1188     if (!safe_recv(header, sizeof(*header))) {
1189         return false;
1190     }
1191 
1192     auto bodylen = header->getBodylen();
1193 
1194     // Set response to NULL, because the underlying buffer may change.
1195     header = nullptr;
1196 
1197     info.resize(sizeof(*header) + bodylen);
1198     auto ret = safe_recv(info.data() + sizeof(*header), bodylen);
1199 
1200     if (dump_socket_traffic) {
1201         if (dump_socket_traffic) {
1202             if (current_phase == phase_ssl) {
1203                 std::cerr << "SSL";
1204             } else {
1205                 std::cerr << "PLAIN";
1206             }
1207             std::cerr << "< ";
1208             for (const auto& val : info) {
1209                 std::cerr << cb::to_hex(uint8_t(val)) << ", ";
1210             }
1211         }
1212         std::cerr << std::endl;
1213     }
1214     return ret;
1215 }
1216 
1217 /**
1218  * Wrapper for a existing buffer which exposes an API suitable for use with
1219  * safe_recv_packetT()
1220  */
1221 struct StaticBufInfo {
StaticBufInfoStaticBufInfo1222     StaticBufInfo(void* buf_, size_t len_)
1223         : buf(reinterpret_cast<char*>(buf_)), len(len_) {
1224     }
sizeStaticBufInfo1225     size_t size(size_t) const {
1226         return len;
1227     }
dataStaticBufInfo1228     char* data() {
1229         return buf;
1230     }
1231 
resizeStaticBufInfo1232     void resize(size_t n) {
1233         if (n > len) {
1234             throw std::runtime_error("Cannot enlarge buffer!");
1235         }
1236     }
1237 
beginStaticBufInfo1238     constexpr char* begin() const {
1239         return buf;
1240     }
1241 
endStaticBufInfo1242     constexpr char* end() const {
1243         return buf + len;
1244     }
1245 
1246     char* buf;
1247     const size_t len;
1248 };
1249 
safe_recv_packet(void* buf, size_t size)1250 bool safe_recv_packet(void* buf, size_t size) {
1251     StaticBufInfo info(buf, size);
1252     return safe_recv_packetT(info);
1253 }
1254 
safe_recv_packet(std::vector<uint8_t>& buf)1255 bool safe_recv_packet(std::vector<uint8_t>& buf) {
1256     return safe_recv_packetT(buf);
1257 }
1258 
safe_recv_packet(std::vector<char>& buf)1259 bool safe_recv_packet(std::vector<char>& buf) {
1260     return safe_recv_packetT(buf);
1261 }
1262 
1263 // Configues the ewouldblock_engine to use the given mode; value
1264 // is a mode-specific parameter.
ewouldblock_engine_configure(ENGINE_ERROR_CODE err_code, const EWBEngineMode& mode, uint32_t value, const std::string& key)1265 void TestappTest::ewouldblock_engine_configure(ENGINE_ERROR_CODE err_code,
1266                                                const EWBEngineMode& mode,
1267                                                uint32_t value,
1268                                                const std::string& key) {
1269     cb::mcbp::request::EWB_Payload payload;
1270     payload.setMode(static_cast<uint32_t>(mode));
1271     payload.setValue(value);
1272     payload.setInjectError(static_cast<uint32_t>(err_code));
1273 
1274     std::vector<uint8_t> buffer(sizeof(cb::mcbp::Request) +
1275                                 sizeof(cb::mcbp::request::EWB_Payload) +
1276                                 key.size());
1277     cb::mcbp::RequestBuilder builder({buffer.data(), buffer.size()});
1278     builder.setMagic(cb::mcbp::Magic::ClientRequest);
1279     builder.setOpcode(cb::mcbp::ClientOpcode::EwouldblockCtl);
1280     builder.setOpaque(0xdeadbeef);
1281     builder.setExtras(
1282             {reinterpret_cast<const uint8_t*>(&payload), sizeof(payload)});
1283     builder.setKey({reinterpret_cast<const uint8_t*>(key.data()), key.size()});
1284     safe_send(buffer);
1285 
1286     buffer.resize(1024);
1287     safe_recv_packet(buffer.data(), buffer.size());
1288     mcbp_validate_response_header(
1289             *reinterpret_cast<cb::mcbp::Response*>(buffer.data()),
1290             cb::mcbp::ClientOpcode::EwouldblockCtl,
1291             cb::mcbp::Status::Success);
1292 }
1293 
ewouldblock_engine_configure( std::vector<cb::engine_errc> sequence)1294 void TestappTest::ewouldblock_engine_configure(
1295         std::vector<cb::engine_errc> sequence) {
1296     ewouldblock_engine_configure(ENGINE_SUCCESS,
1297                                  EWBEngineMode::Sequence,
1298                                  0,
1299                                  ewb::encodeSequence(sequence));
1300 }
1301 
ewouldblock_engine_disable()1302 void TestappTest::ewouldblock_engine_disable() {
1303     // Value for err_code doesn't matter...
1304     ewouldblock_engine_configure(ENGINE_EWOULDBLOCK, EWBEngineMode::Next_N, 0);
1305 }
1306 
reconfigure()1307 void TestappTest::reconfigure() {
1308     write_config_to_file(memcached_cfg.dump(2), config_file);
1309     auto& conn = getAdminConnection();
1310 
1311     BinprotGenericCommand req{cb::mcbp::ClientOpcode::ConfigReload, {}, {}};
1312     const auto resp = conn.execute(req);
1313     ASSERT_TRUE(resp.isSuccess()) << "Failed to reconfigure the server";
1314     conn.reconnect();
1315 }
1316 
runCreateXattr(const std::string& path, const std::string& value, bool macro, cb::mcbp::Status expectedStatus)1317 void TestappTest::runCreateXattr(const std::string& path,
1318                                  const std::string& value,
1319                                  bool macro,
1320                                  cb::mcbp::Status expectedStatus) {
1321     auto& connection = getConnection();
1322 
1323     BinprotSubdocCommand cmd;
1324     cmd.setOp(cb::mcbp::ClientOpcode::SubdocDictAdd);
1325     cmd.setKey(name);
1326     cmd.setPath(path);
1327     cmd.setValue(value);
1328     if (macro) {
1329         cmd.addPathFlags(SUBDOC_FLAG_XATTR_PATH | SUBDOC_FLAG_EXPAND_MACROS |
1330                          SUBDOC_FLAG_MKDIR_P);
1331     } else {
1332         cmd.addPathFlags(SUBDOC_FLAG_XATTR_PATH | SUBDOC_FLAG_MKDIR_P);
1333     }
1334 
1335     connection.sendCommand(cmd);
1336 
1337     BinprotResponse resp;
1338     connection.recvResponse(resp);
1339     EXPECT_EQ(expectedStatus, resp.getStatus());
1340 }
1341 
createXattr(const std::string& path, const std::string& value, bool macro)1342 void TestappTest::createXattr(const std::string& path,
1343                               const std::string& value,
1344                               bool macro) {
1345     runCreateXattr(path, value, macro, cb::mcbp::Status::Success);
1346 }
1347 
runGetXattr( const std::string& path, bool deleted, cb::mcbp::Status expectedStatus)1348 BinprotSubdocResponse TestappTest::runGetXattr(
1349         const std::string& path,
1350         bool deleted,
1351         cb::mcbp::Status expectedStatus) {
1352     auto& connection = getConnection();
1353 
1354     BinprotSubdocCommand cmd;
1355     cmd.setOp(cb::mcbp::ClientOpcode::SubdocGet);
1356     cmd.setKey(name);
1357     cmd.setPath(path);
1358     if (deleted) {
1359         cmd.addPathFlags(SUBDOC_FLAG_XATTR_PATH);
1360         cmd.addDocFlags(mcbp::subdoc::doc_flag::AccessDeleted);
1361     } else {
1362         cmd.addPathFlags(SUBDOC_FLAG_XATTR_PATH);
1363     }
1364     connection.sendCommand(cmd);
1365 
1366     BinprotSubdocResponse resp;
1367     connection.recvResponse(resp);
1368     auto status = resp.getStatus();
1369     if (deleted && status == cb::mcbp::Status::SubdocSuccessDeleted) {
1370         status = cb::mcbp::Status::Success;
1371     }
1372 
1373     if (status != expectedStatus) {
1374         throw ConnectionError("runGetXattr() failed: ", resp);
1375     }
1376     return resp;
1377 }
1378 
getXattr(const std::string& path, bool deleted)1379 BinprotSubdocResponse TestappTest::getXattr(const std::string& path,
1380                                             bool deleted) {
1381     return runGetXattr(path, deleted, cb::mcbp::Status::Success);
1382 }
1383 
getResponseCount(cb::mcbp::Status statusCode)1384 int TestappTest::getResponseCount(cb::mcbp::Status statusCode) {
1385     auto stats = getConnection().stats("responses detailed");
1386     auto responses = stats["responses"];
1387     std::stringstream stream;
1388     stream << std::hex << uint16_t(statusCode);
1389     auto obj = responses.find(stream.str());
1390     if (obj == responses.end()) {
1391         return 0;
1392     }
1393 
1394     return gsl::narrow<int>(obj->get<size_t>());
1395 }
1396 
expectedJSONDatatype() const1397 cb::mcbp::Datatype TestappTest::expectedJSONDatatype() const {
1398     return hasJSONSupport() == ClientJSONSupport::Yes ? cb::mcbp::Datatype::JSON
1399                                                       : cb::mcbp::Datatype::Raw;
1400 }
1401 
getConnection()1402 MemcachedConnection& TestappTest::getConnection() {
1403     // The basic tests should use a plain IPv4 unless something else is
1404     // required (this makes the return value of getConnection predictable
1405     // (rather than returning whatever happened to be stored in "front" of
1406     // the map.
1407     return prepare(connectionMap.getConnection(false, AF_INET));
1408 }
1409 
getAdminConnection()1410 MemcachedConnection& TestappTest::getAdminConnection() {
1411     auto& conn = getConnection();
1412     conn.authenticate("@admin", "password", conn.getSaslMechanisms());
1413     return conn;
1414 }
1415 
prepare(MemcachedConnection& connection)1416 MemcachedConnection& TestappTest::prepare(MemcachedConnection& connection) {
1417     std::vector<cb::mcbp::Feature> features = {
1418             {cb::mcbp::Feature::MUTATION_SEQNO,
1419              cb::mcbp::Feature::XATTR,
1420              cb::mcbp::Feature::XERROR,
1421              cb::mcbp::Feature::SELECT_BUCKET}};
1422     if (hasSnappySupport() == ClientSnappySupport::Yes) {
1423         features.push_back(cb::mcbp::Feature::SNAPPY);
1424     }
1425 
1426     if (hasJSONSupport() == ClientJSONSupport::Yes) {
1427         features.push_back(cb::mcbp::Feature::JSON);
1428     }
1429 
1430     connection.reconnect();
1431     connection.setFeatures("testapp", features);
1432     return connection;
1433 }
1434 
1435 nlohmann::json TestappTest::memcached_cfg;
1436 std::string TestappTest::portnumber_file;
1437 std::string TestappTest::config_file;
1438 ConnectionMap TestappTest::connectionMap;
1439 uint64_t TestappTest::token;
1440 cb_thread_t TestappTest::memcached_server_thread;
1441 
main(int argc, char** argv)1442 int main(int argc, char** argv) {
1443     // We need to set MEMCACHED_UNIT_TESTS to enable the use of
1444     // the ewouldblock engine..
1445     static char envvar[80];
1446     snprintf(envvar, sizeof(envvar), "MEMCACHED_UNIT_TESTS=true");
1447     putenv(envvar);
1448 
1449     setupWindowsDebugCRTAssertHandling();
1450 
1451     ::testing::InitGoogleTest(&argc, argv);
1452 
1453 #ifndef WIN32
1454     /*
1455     ** When running the tests from within CLion it starts the test in
1456     ** another directory than pwd. This cause us to fail to locate the
1457     ** memcached binary to start. To work around that lets just do a
1458     ** chdir(dirname(argv[0])).
1459     */
1460     auto testdir = cb::io::dirname(argv[0]);
1461     if (chdir(testdir.c_str()) != 0) {
1462         std::cerr << "Failed to change directory to " << testdir << std::endl;
1463         exit(EXIT_FAILURE);
1464     }
1465 #endif
1466 
1467     std::string engine_name("default");
1468     std::string engine_config;
1469 
1470     int cmd;
1471     while ((cmd = getopt(argc, argv, "vc:eE:")) != EOF) {
1472         switch (cmd) {
1473         case 'v':
1474             memcached_verbose++;
1475             break;
1476         case 'c':
1477             engine_config = optarg;
1478             break;
1479         case 'e':
1480             embedded_memcached_server = true;
1481             break;
1482         case 'E':
1483             engine_name.assign(optarg);
1484             break;
1485         default:
1486             std::cerr << "Usage: " << argv[0] << " [-v] [-e]" << std::endl
1487                       << std::endl
1488                       << "  -v Verbose - Print verbose memcached output "
1489                       << "to stderr." << std::endl
1490                       << "               (use multiple times to increase the"
1491                       << " verbosity level." << std::endl
1492                       << "  -c CONFIG - Additional configuration to pass to "
1493                       << "bucket creation." << std::endl
1494                       << "  -e Embedded - Run the memcached daemon in the "
1495                       << "same process (for debugging only..)" << std::endl
1496                       << "  -E ENGINE engine type to use. <default|ep>"
1497                       << std::endl;
1498             return 1;
1499         }
1500     }
1501 
1502     /*
1503      * If not running in embedded mode we need the McdEnvironment to manageSSL
1504      * initialization and shutdown.
1505      */
1506     mcd_env = new McdEnvironment(
1507             !embedded_memcached_server, engine_name, engine_config);
1508 
1509     ::testing::AddGlobalTestEnvironment(mcd_env);
1510 
1511     cb_initialize_sockets();
1512 
1513 #if !defined(WIN32)
1514     /*
1515      * When shutting down SSL connections the SSL layer may attempt to
1516      * write to the underlying socket. If the socket has been closed
1517      * on the server side then this will raise a SIGPIPE (and
1518      * terminate the test program). This is Bad.
1519      * Therefore ignore SIGPIPE signals; we can use errno == EPIPE if
1520      * we need that information.
1521      */
1522     if (sigignore(SIGPIPE) == -1) {
1523         std::cerr << "Fatal: failed to ignore SIGPIPE; sigaction" << std::endl;
1524         return 1;
1525     }
1526 #endif
1527 
1528     return RUN_ALL_TESTS();
1529 }
1530 
1531 /* Request stats
1532  * @return a map of stat key & values in the server response.
1533  */
request_stats()1534 stats_response_t request_stats() {
1535     BinprotGenericCommand cmd(cb::mcbp::ClientOpcode::Stat);
1536     std::vector<uint8_t> blob;
1537     cmd.encode(blob);
1538     safe_send(blob);
1539 
1540     stats_response_t result;
1541     while (true) {
1542         safe_recv_packet(blob);
1543         BinprotResponse rsp;
1544         rsp.assign(std::move(blob));
1545         mcbp_validate_response_header(
1546                 const_cast<cb::mcbp::Response&>(rsp.getResponse()),
1547                 cb::mcbp::ClientOpcode::Stat,
1548                 cb::mcbp::Status::Success);
1549         // key length zero indicates end of the stats.
1550         if (rsp.getKeyString().empty()) {
1551             break;
1552         }
1553 
1554         result.insert(std::make_pair(rsp.getKeyString(), rsp.getDataString()));
1555     }
1556 
1557     return result;
1558 }
1559 
1560 // Extracts a single statistic from the set of stats, returning as
1561 // a uint64_t
extract_single_stat(const stats_response_t& stats, const char* name)1562 uint64_t extract_single_stat(const stats_response_t& stats, const char* name) {
1563     auto iter = stats.find(name);
1564     EXPECT_NE(stats.end(), iter);
1565     uint64_t result = 0;
1566     result = std::stoul(iter->second);
1567     return result;
1568 }
1569 
1570 /*
1571     Using a memcached protocol extesnsion, shift the time
1572 */
adjust_memcached_clock( int64_t clock_shift, cb::mcbp::request::AdjustTimePayload::TimeType timeType)1573 void adjust_memcached_clock(
1574         int64_t clock_shift,
1575         cb::mcbp::request::AdjustTimePayload::TimeType timeType) {
1576     cb::mcbp::request::AdjustTimePayload payload;
1577     payload.setOffset(uint64_t(clock_shift));
1578     payload.setTimeType(timeType);
1579 
1580     std::vector<uint8_t> blob(sizeof(cb::mcbp::Request) + sizeof(payload));
1581     cb::mcbp::FrameBuilder<cb::mcbp::Request> builder(
1582             {blob.data(), blob.size()});
1583     builder.setMagic(cb::mcbp::Magic::ClientRequest);
1584     builder.setOpcode(cb::mcbp::ClientOpcode::AdjustTimeofday);
1585     builder.setExtras(payload.getBuffer());
1586     builder.setOpaque(0xdeadbeef);
1587     safe_send(builder.getFrame()->getFrame());
1588 
1589     blob.resize(0);
1590     safe_recv_packet(blob);
1591     BinprotResponse rsp;
1592     rsp.assign(std::move(blob));
1593     mcbp_validate_response_header(
1594             const_cast<cb::mcbp::Response&>(rsp.getResponse()),
1595             cb::mcbp::ClientOpcode::AdjustTimeofday,
1596             cb::mcbp::Status::Success);
1597 }
1598