1 /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2 /*
3  *     Copyright 2017 Couchbase, Inc.
4  *
5  *   Licensed under the Apache License, Version 2.0 (the "License");
6  *   you may not use this file except in compliance with the License.
7  *   You may obtain a copy of the License at
8  *
9  *       http://www.apache.org/licenses/LICENSE-2.0
10  *
11  *   Unless required by applicable law or agreed to in writing, software
12  *   distributed under the License is distributed on an "AS IS" BASIS,
13  *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14  *   See the License for the specific language governing permissions and
15  *   limitations under the License.
16  */
17 #include <string.h>
18 #include <cerrno>
19 #include "testapp_shutdown.h"
20 
21 /// The different types of shutdown we can trigger.
22 enum class ShutdownMode { Clean, Unclean };
23 
operator <<(std::ostream& os, const ShutdownMode& mode)24 std::ostream& operator<<(std::ostream& os, const ShutdownMode& mode) {
25     os << (mode == ShutdownMode::Clean ? "Clean" : "Unclean");
26     return os;
27 }
28 
29 /**
30  * Tests Persist_To functionality for bucket types which support it (i.e. EP
31  * Bucket).
32  *
33  * Reuses the functionality of ShutdownTest to start / stop memcached for each
34  * test instance.
35  */
36 class PersistToTest : public ShutdownTest,
37                       public ::testing::WithParamInterface<ShutdownMode> {
38 protected:
39     void SetUp() override {
40         if (!mcd_env->getTestBucket().supportsPersistence()) {
41             std::cout << "Note: skipping test '"
42                       << ::testing::UnitTest::GetInstance()
43                                  ->current_test_info()
44                                  ->name()
45                       << "' as persistence isn't supported.\n";
46             skipTest = true;
47             return;
48         }
49         ShutdownTest::SetUp();
50     }
51 
52     void TearDown() override {
53         if (skipTest) {
54             return;
55         }
56         ShutdownTest::TearDown();
57     }
58 
59     // Helper functions for tests /////////////////////////////////////////////
60 
storeAndPersistItem(std::string key)61     Document storeAndPersistItem(std::string key) {
62         MemcachedConnection& conn = getConnection();
63         conn.setMutationSeqnoSupport(true);
64         Document doc;
65         doc.info.id = key;
66         doc.value = "persist me";
67         auto mutation = conn.mutate(doc, vbid, MutationType::Set);
68         EXPECT_NE(0, mutation.seqno);
69         EXPECT_NE(0, mutation.vbucketuuid);
70         doc.info.cas = mutation.cas;
71 
72         waitForAtLeastSeqno(mutation.vbucketuuid, mutation.seqno);
73 
74         return doc;
75     }
76 
waitForAtLeastSeqno(uint64_t uuid, uint64_t seqno)77     void waitForAtLeastSeqno(uint64_t uuid, uint64_t seqno) {
78         // Poll for that sequence number to be persisted.
79         ObserveInfo observe;
80         MemcachedConnection& conn = getConnection();
81         do {
82             observe = conn.observeSeqno(vbid, uuid);
83             EXPECT_EQ(0, observe.formatType);
84             EXPECT_EQ(vbid, observe.vbId);
85             EXPECT_EQ(uuid, observe.uuid);
86         } while (observe.lastPersistedSeqno < seqno);
87     }
88 
shutdownMemcached(ShutdownMode mode)89     void shutdownMemcached(ShutdownMode mode) {
90         switch (mode) {
91         case ShutdownMode::Unclean:
92 #ifdef WIN32
93             // There's no direct equivalent of SIGKILL for Windows;
94             // TerminateProcess() behaves like SIGTERM - it allows pending IO
95             // to complete; however it's the best we have...
96             TerminateProcess(server_pid, 0);
97 #else
98             kill(server_pid, SIGKILL);
99 #endif
100             break;
101         case ShutdownMode::Clean: {
102             auto& admin = getAdminConnection();
103             BinprotGenericCommand cmd(PROTOCOL_BINARY_CMD_SHUTDOWN);
104             cmd.setCas(token);
105             admin.sendCommand(cmd);
106 
107             BinprotResponse rsp;
108             admin.recvResponse(rsp);
109             EXPECT_TRUE(rsp.isSuccess());
110             break;
111         }
112         }
113         waitForShutdown(mode == ShutdownMode::Unclean);
114     }
115 
116     uint16_t vbid = 0;
117     bool skipTest = false;
118 };
119 
120 /**
121  * Verify that items are successfully persisted (and can be read) after a
122  * clean / unclean shutdown and restart.
123  */
TEST_P(PersistToTest, PersistedAfterShutdown)124 TEST_P(PersistToTest, PersistedAfterShutdown) {
125     if (skipTest) {
126         return;
127     }
128 
129     // Store 1 item, noting it's sequence number
130     uint16_t vbid = 0;
131     auto doc = storeAndPersistItem("1");
132 
133     // Shutdown of memcached.
134     shutdownMemcached(GetParam());
135 
136     // Restart memcached, and attempt to read the item we persisted.
137     SetUp();
138 
139     MemcachedConnection& conn = getConnection();
140     try {
141         auto doc2 = conn.get(doc.info.id, vbid);
142         EXPECT_EQ(doc, doc2);
143     } catch (const ConnectionError& e) {
144         FAIL() << e.what();
145     }
146 }
147 
148 /**
149  * Verify that the vBucket is in a consistent state after a shutdown;
150  * even if there are missing items.
151  *
152  * By "consistent", what we mean here is that for a sequence of items
153  * {key, seqno} : ({1,a}, {2,b}, {3,c}, ...) written to a vBucket,
154  * that if seqno N was persisted then all seqnos <N were also persisted,
155  * and nonee of seqnos >N were persisted.
156  *
157  * To test this, create a sequence of keys, additionally writing
158  * the current highest key to a additional "high" doc between each key:
159  *
160  *     {1, high=1, 2, high=2, 3, high=3, 4, high=4, ...
161  *
162  * Then shutdown memcached when it's in the processes of persisting these keys
163  * (making sure "high" has been persisted at least once).
164  *
165  * On restart, we read what value "high" has (i.e. how far through the sequence
166  * persistence got). We then verify that the vBucket is in one of two valid
167  * states:
168  *
169  * a) "high" was the very last document persisted - which means that the key
170  *    matching the value of "high", and all proceeding keys should exist.
171  *
172  * b) "high" was the last but one document persisted - which means that there
173  *    is one additional key in existence (named high+1)
174  *
175  * Any other state is invalid and hence a failure.
176  */
TEST_P(PersistToTest, ConsistentStateAfterShutdown)177 TEST_P(PersistToTest, ConsistentStateAfterShutdown) {
178     if (skipTest) {
179         return;
180     }
181 
182     // Start off with persistence disabled.
183     {
184         auto& admin = getAdminConnection();
185         admin.selectBucket("default");
186         admin.disablePersistence();
187     }
188 
189     MemcachedConnection& conn = getConnection();
190     conn.setMutationSeqnoSupport(true);
191 
192     // Store our series of documents:1, high=1, 2, high=2, 3, ...
193     Document high;
194     high.info.id = "high";
195     uint64_t uuid;
196     const size_t docCount = 100;
197 
198     for (size_t i = 0; i < docCount; i++) {
199         Document doc;
200         doc.info.id = std::to_string(i);
201         doc.value = doc.info.id;
202         auto mutation = conn.mutate(doc, vbid, MutationType::Set);
203         uuid = mutation.vbucketuuid;
204 
205         high.value = doc.info.id;
206         conn.mutate(high, vbid, MutationType::Set);
207     }
208 
209     // Re-enable persistence, and check we've stored to at least seqno 2 -
210     // i.e. one iteration of high being written.
211     {
212         auto& admin = getAdminConnection();
213         admin.selectBucket("default");
214         admin.enablePersistence();
215     }
216 
217     waitForAtLeastSeqno(uuid, 2);
218 
219     // Perform a shutdown of memcached.
220     shutdownMemcached(GetParam());
221 
222     // Restart memcached.
223     SetUp();
224 
225     // Read "high" to determine how far we got, and then validate that (1)
226     // all previous documents exist and (2) no more than 1 extra document exists
227     // after high.
228     {
229         MemcachedConnection& conn = getConnection();
230         high = conn.get("high", vbid);
231 
232         // Check that all keys up to "high" exist:
233         size_t highNumber = std::stoi(high.value);
234         for (size_t i = 0; i < highNumber; i++) {
235             conn.get(std::to_string(i), vbid);
236         }
237 
238         // We permit the key one above "high" to not exist - see state (b)
239         // above.
240 
241         // Check that all keys above high+1 do not exist.
242         for (size_t i = highNumber + 1; i < docCount; i++) {
243             auto key = std::to_string(i);
244             try {
245                 conn.get(key, vbid);
246             } catch (ConnectionError&) {
247                 // expect the get to fail.
248                 continue;
249             }
250             FAIL() << "Found key '" << key << "'"
251                    << " which should not exist";
252         }
253     }
254 }
255 
256 // MB-27539: ThreadSanitizer detects false positives on 'Clean' shutdown
257 // tests run after 'Unclean' shutdown tests
258 #if defined(__has_feature)
259 #if __has_feature(thread_sanitizer)
260 #define SKIP_UNCLEAN
261 #endif
262 #endif
263 
264 #if defined(SKIP_UNCLEAN)
265 INSTANTIATE_TEST_CASE_P(Clean,
266                         PersistToTest,
267                         ::testing::Values(ShutdownMode::Clean),
268                         ::testing::PrintToStringParamName());
269 #else
270 INSTANTIATE_TEST_CASE_P(CleanOrUnclean,
271                         PersistToTest,
272                         ::testing::Values(ShutdownMode::Clean,
273                                           ShutdownMode::Unclean),
274                         ::testing::PrintToStringParamName());
275 #endif
276