1/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2/*
3 *     Copyright 2020 Couchbase, Inc
4 *
5 *   Licensed under the Apache License, Version 2.0 (the "License");
6 *   you may not use this file except in compliance with the License.
7 *   You may obtain a copy of the License at
8 *
9 *       http://www.apache.org/licenses/LICENSE-2.0
10 *
11 *   Unless required by applicable law or agreed to in writing, software
12 *   distributed under the License is distributed on an "AS IS" BASIS,
13 *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 *   See the License for the specific language governing permissions and
15 *   limitations under the License.
16 */
17
18#include "dcp/backfill-manager.h"
19#include "evp_store_single_threaded_test.h"
20
21#include <folly/portability/GMock.h>
22#include <folly/portability/GTest.h>
23
24using ::testing::InSequence;
25using ::testing::Return;
26
27class GMockDCPBackfill : public DCPBackfillIface {
28public:
29    MOCK_METHOD0(run, backfill_status_t());
30    MOCK_METHOD0(cancel, void());
31    MOCK_CONST_METHOD0(getVBucketId, Vbid());
32    MOCK_CONST_METHOD0(isStreamDead, bool());
33};
34
35class GMockBackfillTracker : public BackfillTrackingIface {
36public:
37    MOCK_METHOD0(canAddBackfillToActiveQ, bool());
38    MOCK_METHOD0(decrNumActiveSnoozingBackfills, void());
39};
40
41class BackfillManagerTest : public SingleThreadedKVBucketTest {
42protected:
43    void SetUp() {
44        SingleThreadedKVBucketTest::SetUp();
45        backfillMgr =
46                std::make_shared<BackfillManager>(*engine->getKVBucket(),
47                                                  backfillTracker,
48                                                  engine->getConfiguration());
49    }
50
51    void TearDown() {
52        // Need to destroy engine & backfillManager objects before shutting
53        // down ExecutorPool.
54        backfillMgr.reset();
55        SingleThreadedKVBucketTest::TearDown();
56    }
57
58    /**
59     * For tests which are not interested in backfilLTracker, configure it
60     * to accept an arbirary number of concurrent backfills.
61     */
62    void ignoreBackfillTracker() {
63        EXPECT_CALL(backfillTracker, canAddBackfillToActiveQ())
64                .WillRepeatedly(Return(true));
65        EXPECT_CALL(backfillTracker, decrNumActiveSnoozingBackfills())
66                .WillRepeatedly(Return());
67    }
68
69    GMockBackfillTracker backfillTracker;
70    std::shared_ptr<BackfillManager> backfillMgr;
71};
72
73/*
74 * Check that active Backfills are scheduled in round-robin order
75 * (0, 1, 2, 0, 1, 2, ...) until they complete.
76 */
77TEST_F(BackfillManagerTest, RoundRobin) {
78    // Not interested in behaviour of backfillTracker for this test.
79    ignoreBackfillTracker();
80
81    auto backfill0 = std::make_unique<GMockDCPBackfill>();
82    auto backfill1 = std::make_unique<GMockDCPBackfill>();
83    auto backfill2 = std::make_unique<GMockDCPBackfill>();
84
85    // Expectation - two backfills should be scheduled in turn while they
86    // return backfill_success.
87    {
88        InSequence s;
89        EXPECT_CALL(*backfill0, run())
90                .WillOnce(Return(backfill_success))
91                .RetiresOnSaturation();
92        EXPECT_CALL(*backfill1, run())
93                .WillOnce(Return(backfill_success))
94                .RetiresOnSaturation();
95        EXPECT_CALL(*backfill2, run())
96                .WillOnce(Return(backfill_success))
97                .RetiresOnSaturation();
98        EXPECT_CALL(*backfill0, run())
99                .WillOnce(Return(backfill_finished))
100                .RetiresOnSaturation();
101        EXPECT_CALL(*backfill1, run())
102                .WillOnce(Return(backfill_finished))
103                .RetiresOnSaturation();
104        EXPECT_CALL(*backfill2, run())
105                .WillOnce(Return(backfill_finished))
106                .RetiresOnSaturation();
107    }
108
109    // Test: schedule both backfills, then instruct the backfill manager to
110    // backfill 6 times.
111    ASSERT_EQ(BackfillManager::ScheduleResult::Active,
112              backfillMgr->schedule(std::move(backfill0)));
113    ASSERT_EQ(BackfillManager::ScheduleResult::Active,
114              backfillMgr->schedule(std::move(backfill1)));
115    ASSERT_EQ(BackfillManager::ScheduleResult::Active,
116              backfillMgr->schedule(std::move(backfill2)));
117    backfillMgr->backfill();
118    backfillMgr->backfill();
119    backfillMgr->backfill();
120    backfillMgr->backfill();
121    backfillMgr->backfill();
122    backfillMgr->backfill();
123}
124
125/*
126 * MB-37680: Check that active backfills are scheduled in sequential order when
127 * backfillOrder is set to Sequential.
128 */
129TEST_F(BackfillManagerTest, Sequential) {
130    // Not interested in behaviour of backfillTracker for this test.
131    ignoreBackfillTracker();
132
133    auto backfill0 = std::make_unique<GMockDCPBackfill>();
134    auto backfill1 = std::make_unique<GMockDCPBackfill>();
135    auto backfill2 = std::make_unique<GMockDCPBackfill>();
136
137    // Expectation - each Backfill should be run once (to initialise and
138    // generate the snapshot_marker, then the first backfill should be run to
139    // completion before the next one starts.
140    //
141    {
142        InSequence s;
143
144        // each backfill called once for snapshot_marker
145        EXPECT_CALL(*backfill0, run())
146                .WillOnce(Return(backfill_success))
147                .RetiresOnSaturation();
148        EXPECT_CALL(*backfill1, run())
149                .WillOnce(Return(backfill_success))
150                .RetiresOnSaturation();
151        EXPECT_CALL(*backfill2, run())
152                .WillOnce(Return(backfill_success))
153                .RetiresOnSaturation();
154
155        // then each one sequential until finished.
156        EXPECT_CALL(*backfill0, run())
157                .WillOnce(Return(backfill_success))
158                .WillOnce(Return(backfill_finished))
159                .RetiresOnSaturation();
160        EXPECT_CALL(*backfill1, run())
161                .WillOnce(Return(backfill_success))
162                .WillOnce(Return(backfill_finished))
163                .RetiresOnSaturation();
164        EXPECT_CALL(*backfill2, run())
165                .WillOnce(Return(backfill_success))
166                .WillOnce(Return(backfill_finished))
167                .RetiresOnSaturation();
168    }
169
170    // Test: schedule all backfills, then instruct the backfill manager to
171    // backfill 9 times.
172    backfillMgr->setBackfillOrder(BackfillManager::ScheduleOrder::Sequential);
173    ASSERT_EQ(BackfillManager::ScheduleResult::Active,
174              backfillMgr->schedule(std::move(backfill0)));
175    ASSERT_EQ(BackfillManager::ScheduleResult::Active,
176              backfillMgr->schedule(std::move(backfill1)));
177    ASSERT_EQ(BackfillManager::ScheduleResult::Active,
178              backfillMgr->schedule(std::move(backfill2)));
179    for (int i = 0; i < 9; i++) {
180        backfillMgr->backfill();
181    }
182}
183
184/**
185 * Check that if BackfillTracker is full then BackfillManager correctly
186 * puts scheduled Backfills into PendingQ, until space becomes available.
187 */
188TEST_F(BackfillManagerTest, BackfillTrackerFull) {
189    auto backfill0 = std::make_unique<GMockDCPBackfill>();
190    auto backfill1 = std::make_unique<GMockDCPBackfill>();
191
192    // Configure expectations.
193    // (Note: must configure all GMockDCPBackfill expectations before
194    //  schedule() is called, as that takes ownership of the object).
195    {
196        InSequence s;
197        // Calls to schedule() for the two backfills. BackfillManager should
198        // check if it is allowed to add backfill to activeQ. backfill0 should
199        // be allowed, backfill1 should not (and hence be put in pendingQ).
200        EXPECT_CALL(backfillTracker, canAddBackfillToActiveQ())
201                .WillOnce(Return(true))
202                .WillOnce(Return(false))
203                .RetiresOnSaturation();
204
205        // Next, when backfill() is called, it should again check if there are
206        // items in pending queue (which there is 1), and if they they can be
207        // moved to activeQ. Given backfill0 has not finished (and made space),
208        // return false here.
209        EXPECT_CALL(backfillTracker, canAddBackfillToActiveQ())
210                .WillOnce(Return(false))
211                .RetiresOnSaturation();
212        // backfill0 should be run now, we return success.
213        EXPECT_CALL(*backfill0, run())
214                .WillOnce(Return(backfill_success))
215                .RetiresOnSaturation();
216
217        // Second call to backfill() will again attempt to add the pending item
218        // to activeQ. Again tracker denies it.
219        EXPECT_CALL(backfillTracker, canAddBackfillToActiveQ())
220                .WillOnce(Return(false))
221                .RetiresOnSaturation();
222        // backfill0 should run a second time, we return finished.
223        EXPECT_CALL(*backfill0, run())
224                .WillOnce(Return(backfill_finished))
225                .RetiresOnSaturation();
226        // Upon backfill_finished, backfill manager should decrement the number
227        // of actve/snoozing backfills.
228        EXPECT_CALL(backfillTracker, decrNumActiveSnoozingBackfills())
229                .WillOnce(Return())
230                .RetiresOnSaturation();
231
232        // Third call to backfill() - the pending backfill1 should now be
233        // allowed to be added to activeQ.
234        EXPECT_CALL(backfillTracker, canAddBackfillToActiveQ())
235                .WillOnce(Return(true))
236                .RetiresOnSaturation();
237        // backfill1 should be run now.
238        EXPECT_CALL(*backfill1, run())
239                .WillOnce(Return(backfill_success))
240                .RetiresOnSaturation();
241
242        // Fourth call to backfill() - no items in pendingQ so no call to
243        // backfilLTracker this fine; backfill1 should finish and decrment
244        // active/snoozing backfills
245        EXPECT_CALL(*backfill1, run())
246                .WillOnce(Return(backfill_finished))
247                .RetiresOnSaturation();
248        EXPECT_CALL(backfillTracker, decrNumActiveSnoozingBackfills())
249                .WillOnce(Return())
250                .RetiresOnSaturation();
251    }
252
253    // Test - schedule the two backfills - first should be added to active Q,
254    // second to pending Q.
255    ASSERT_EQ(BackfillManager::ScheduleResult::Active,
256              backfillMgr->schedule(std::move(backfill0)));
257    ASSERT_EQ(BackfillManager::ScheduleResult::Pending,
258              backfillMgr->schedule(std::move(backfill1)));
259
260    // Drive manager forward - see above expectations.
261    backfillMgr->backfill();
262    backfillMgr->backfill();
263    backfillMgr->backfill();
264    backfillMgr->backfill();
265}
266
267/**
268 * Check that if BackfillManager is destroyed with Backfills in the
269 * initializingQ, then it correctly notifies the backfill tracker when the
270 * BackfillManager is destroyed.
271 */
272TEST_F(BackfillManagerTest, InitializingQNotifiesTrackerOnDtor) {
273    auto backfill = std::make_unique<GMockDCPBackfill>();
274
275    // Configure expectations.
276    // (Note: must configure all GMockDCPBackfill expectations before
277    //  schedule() is called, as that takes ownership of the object).
278    {
279        InSequence s;
280        // Call to schedule() for the backfills. BackfillManager should
281        // check if it is allowed to add backfill to initializingQ. Backfill
282        // should be allowed, which increments tracker's count.
283        EXPECT_CALL(backfillTracker, canAddBackfillToActiveQ())
284                .WillOnce(Return(true))
285                .RetiresOnSaturation();
286
287        // When BackfillManager is destroyed, the incomplete backfill in
288        // initializingQ should be decremented in the tracker.
289        EXPECT_CALL(backfillTracker, decrNumActiveSnoozingBackfills())
290                .WillOnce(Return())
291                .RetiresOnSaturation();
292    }
293
294    // Setup: schedule a single backfill.
295    ASSERT_EQ(BackfillManager::ScheduleResult::Active,
296              backfillMgr->schedule(std::move(backfill)));
297
298    // Test: Destroy the backfill manager while backfill still in initializingQ.
299    backfillMgr.reset();
300}
301
302/**
303 * Check that if BackfillManager is destroyed with Backfills in the activeQ,
304 * then it correctly notifies the backfill tracker when the BackfillManager is
305 * destroyed.
306 */
307TEST_F(BackfillManagerTest, ActiveQNotifiesTrackerOnDtor) {
308    auto backfill = std::make_unique<GMockDCPBackfill>();
309
310    // Configure expectations.
311    // (Note: must configure all GMockDCPBackfill expectations before
312    //  schedule() is called, as that takes ownership of the object).
313    {
314        InSequence s;
315        // Call to schedule() for the backfills. BackfillManager should
316        // check if it is allowed to add backfill to initializingQ. Backfill
317        // should be allowed, which increments tracker's count.
318        EXPECT_CALL(backfillTracker, canAddBackfillToActiveQ())
319                .WillOnce(Return(true))
320                .RetiresOnSaturation();
321
322        // Call to backfill(). Want to move backfill into the activeQ.
323        EXPECT_CALL(*backfill, run())
324                .WillOnce(Return(backfill_success))
325                .RetiresOnSaturation();
326
327        // When BackfillManager is destroyed, the incomplete backfill in
328        // activeQ should be decremented in the tracker.
329        EXPECT_CALL(backfillTracker, decrNumActiveSnoozingBackfills())
330                .WillOnce(Return())
331                .RetiresOnSaturation();
332    }
333
334    // Setup: schedule a single backfill and run it once so it can be moved
335    // to activeQ.
336    ASSERT_EQ(BackfillManager::ScheduleResult::Active,
337              backfillMgr->schedule(std::move(backfill)));
338    backfillMgr->backfill();
339
340    // Test: Destroy the backfill manager while backfill still in activeQ.
341    backfillMgr.reset();
342}
343
344/**
345 * Check that if BackfillManager is destroyed with Backfills in the snoozingQ,
346 * then it correctly notifies the backfill tracker when the Backfill is
347 * destroyed.
348 */
349TEST_F(BackfillManagerTest, SnoozingQNotifiesTrackerOnDtor) {
350    auto backfill = std::make_unique<GMockDCPBackfill>();
351
352    // Configure expectations.
353    // (Note: must configure all GMockDCPBackfill expectations before
354    //  schedule() is called, as that takes ownership of the object).
355    {
356        InSequence s;
357        // Call to schedule() for the backfills. BackfillManager should
358        // check if it is allowed to add backfill to activeQ. Backfill should
359        // be allowed, which increments tracker's count.
360        EXPECT_CALL(backfillTracker, canAddBackfillToActiveQ())
361                .WillOnce(Return(true))
362                .RetiresOnSaturation();
363
364        // Call to backfill(). Want to move backfill into the snoozinqQ.
365        EXPECT_CALL(*backfill, run())
366                .WillOnce(Return(backfill_snooze))
367                .RetiresOnSaturation();
368
369        // When BackfillManager is destroyed, the incomplete backfill in
370        // snoozingQ should be decremented in the tracker.
371        EXPECT_CALL(backfillTracker, decrNumActiveSnoozingBackfills())
372                .WillOnce(Return())
373                .RetiresOnSaturation();
374    }
375
376    // Setup: schedule a single backfill and run it one so it can be moved
377    // to snoozingQ.
378    ASSERT_EQ(BackfillManager::ScheduleResult::Active,
379              backfillMgr->schedule(std::move(backfill)));
380    backfillMgr->backfill();
381
382    // Test: Destroy the backfill manager while backfill still in snoozingQ.
383    backfillMgr.reset();
384}
385