1 //getNumCPU method code taken from /ep-engine/src/executorpool.cc in ep-engine source code
2 #include "BaseTest.cc"
3 
4 extern struct stats_generic genericstats;
5 
6 static const size_t EP_MIN_NUM_IO_THREADS = 4;
7 class PriorityTest : public BaseTest
8 {
9 protected:
10 
createlcbinstance(lcb_t *instance, char* host, char* user, char* passwd, int& numargs)11 	void createlcbinstance(lcb_t *instance, char* host, char* user, char* passwd, int& numargs)
12 	{
13 
14 		memset(&create_options, 0, sizeof(create_options));
15 		if (numargs > 0)
16 		{
17 			create_options.v.v0.host = host;
18 		}
19 		if (numargs > 1)
20 		{
21 			create_options.v.v0.user = user;
22 			create_options.v.v0.bucket = user;
23 		}
24 		if (numargs > 2)
25 		{
26 			create_options.v.v0.passwd = passwd;
27 		}
28 		err = lcb_create(instance, &create_options);
29 		assert(err == LCB_SUCCESS);
30 		(void) lcb_set_error_callback(*instance, error_callback);
31 		fprintf(stderr, "\nconnect succeeded\n");
32 		(void) lcb_set_get_callback(*instance, get_callback);
33 		(void) lcb_set_store_callback(*instance, store_callback);
34 
35 	}
36 
SetUp()37 	virtual void SetUp()
38 	{
39 
40 		createBucket(true);
41 		createlcbinstance(&instance, testargv[1], testargv[2], testargv[3], testargc);
42 
43 	}
44 
deleteBucket(std::string bucketname)45 	void deleteBucket(std::string bucketname)
46 	{
47 
48 		std::string del = "/opt/couchbase/bin/couchbase-cli bucket-delete -c 127.0.0.1:8091 --bucket=" + bucketname
49 		    + " -u Administrator -p password";
50 		exec((char*) del.c_str());
51 		sleep(60);
52 	}
53 
createBucketwithPriority(bool highprio, std::string bucketname)54 	void createBucketwithPriority(bool highprio, std::string bucketname)
55 	{
56 		if (!highprio)
57 		{
58 			std::string create = "/opt/couchbase/bin/couchbase-cli bucket-create -c 127.0.0.1:8091 --bucket=" + bucketname
59 			    + " --bucket-type=couchbase --bucket-ramsize=200 --bucket-replica=1"
60 			    + " --bucket-priority=low -u Administrator -p password";
61 			exec((char*) create.c_str());
62 		}
63 		else
64 		{
65 			std::string create = "/opt/couchbase/bin/couchbase-cli bucket-create -c 127.0.0.1:8091 --bucket=" + bucketname
66 			    + " --bucket-type=couchbase --bucket-ramsize=200 --bucket-replica=1"
67 			    + " --bucket-priority=high -u Administrator -p password";
68 			exec((char*) create.c_str());
69 		}
70 		sleep(60);
71 	}
72 
verifywithintolerance(int num1, int num2)73 	void verifywithintolerance(int num1, int num2)
74 	{
75 		if (num1 > num2)
76 		{
77 			int percdiff = (int) (((num1 - num2) * 100) / num1);
78 			EXPECT_LT(percdiff, 10);
79 			std::cout << "percdiff: " << percdiff << "\n";
80 		}
81 		else
82 		{
83 			int percdiff = (int) (((num2 - num1) * 100) / num2);
84 			EXPECT_LT(percdiff, 10);
85 			std::cout << "percdiff: " << percdiff << "\n";
86 		}
87 	}
88 
getNumCPU(void)89 	size_t getNumCPU(void)
90 	{
91 
92 		size_t numCPU;
93 #ifdef WIN32
94 		SYSTEM_INFO sysinfo;
95 		GetSystemInfo(&sysinfo);
96 		numCPU = (size_t)sysinfo.dwNumberOfProcessors;
97 #else
98 		numCPU = (size_t) sysconf(_SC_NPROCESSORS_ONLN);
99 #endif
100 
101 		return (numCPU < 256) ? numCPU : 0;
102 	}
103 
getnumThreads()104 	size_t getnumThreads()
105 	{
106 
107 		size_t numCPU = getNumCPU();
108 		size_t numThreads = (size_t) ((numCPU * 3) / 4);
109 		numThreads = (numThreads < EP_MIN_NUM_IO_THREADS) ? EP_MIN_NUM_IO_THREADS : numThreads;
110 		return numThreads;
111 	}
112 
getNumNonIO()113 	size_t getNumNonIO()
114 	{
115 
116 		size_t numThreads = getnumThreads();
117 		size_t count = numThreads / 10;
118 		if (count == 0)
119 			return count + 1;
120 		else if (numThreads % 10 == 0)
121 			return count;
122 		else
123 			return count + 1;
124 	}
125 
getNumAuxIO()126 	size_t getNumAuxIO()
127 	{
128 
129 		size_t numThreads = getnumThreads();
130 		size_t count = numThreads / 10;
131 		if (count == 0)
132 			return count + 1;
133 		else if (numThreads % 10 == 0)
134 			return count;
135 		else
136 			return count + 1;
137 	}
138 
getNumWriters()139 	size_t getNumWriters()
140 	{
141 
142 		size_t numThreads = getnumThreads();
143 		size_t count = numThreads - getNumAuxIO() - getNumNonIO();
144 		count = count / 2;
145 		if (count == 0)
146 			return count + 1;
147 		else
148 			return count;
149 	}
150 
getNumReaders()151 	size_t getNumReaders()
152 	{
153 
154 		size_t numThreads = getnumThreads();
155 		size_t count = numThreads - getNumAuxIO() - getNumNonIO() - getNumWriters();
156 		return count;
157 	}
158 };
159 
TEST_F(PriorityTest, CreateLowBucketPriorityTest)160 TEST_F(PriorityTest, CreateLowBucketPriorityTest)
161 {
162 	(void) lcb_set_stat_callback(instance, stats_generic_callback);
163 	bool highprio = true;
164 	deleteBucket("default");
165 	createBucketwithPriority(!highprio, "default");
166 	err = lcb_connect(instance);
167 	assert(err == LCB_SUCCESS);
168 	std::string all = "";
169 	StatsVector defaultstats = getgenericstats(all, instance);
170 	std::cout << "vector length : " << defaultstats.size() << "\n";
171 	EXPECT_GT(defaultstats.size(), 0);
172 	for (StatsVector::iterator itr = defaultstats.begin(); itr != defaultstats.end(); ++itr)
173 	{
174 		if (itr->first.compare("ep_bucket_priority") == 0)
175 		{
176 			EXPECT_TRUE(!itr->second.compare("LOW"));
177 			std::cout << itr->second << "\n";
178 		}
179 	}
180 }
181 
TEST_F(PriorityTest, CreateHighBucketPriorityTest)182 TEST_F(PriorityTest, CreateHighBucketPriorityTest)
183 {
184 	(void) lcb_set_stat_callback(instance, stats_generic_callback);
185 	bool highprio = true;
186 	deleteBucket("default");
187 	createBucketwithPriority(highprio, "default");
188 	err = lcb_connect(instance);
189 	assert(err == LCB_SUCCESS);
190 	std::string all = "";
191 	StatsVector defaultstats = getgenericstats(all, instance);
192 	std::cout << "vector length : " << defaultstats.size() << "\n";
193 	EXPECT_GT(defaultstats.size(), 0);
194 	for (StatsVector::iterator itr = defaultstats.begin(); itr != defaultstats.end(); ++itr)
195 	{
196 		if (itr->first.compare("ep_bucket_priority") == 0)
197 		{
198 			EXPECT_TRUE(!itr->second.compare("HIGH"));
199 			std::cout << itr->second << "\n";
200 		}
201 	}
202 }
203 
TEST_F(PriorityTest, TotalSpawnedThreadsTest)204 TEST_F(PriorityTest, TotalSpawnedThreadsTest)
205 {
206 	(void) lcb_set_stat_callback(instance, stats_generic_callback);
207 	err = lcb_connect(instance);
208 	assert(err == LCB_SUCCESS);
209 	std::string workload("workload");
210 	StatsVector workloadstats = getgenericstats(workload, instance);
211 	std::cout << "vector length : " << workloadstats.size() << "\n";
212 	size_t numThreads = 0;
213 	for (StatsVector::iterator itr = workloadstats.begin(); itr != workloadstats.end(); ++itr)
214 	{
215 		if (!itr->first.compare("ep_workload:num_auxio") || !itr->first.compare("ep_workload:num_nonio")
216 		    || !itr->first.compare("ep_workload:num_readers") || !itr->first.compare("ep_workload:num_writers"))
217 		{
218 			numThreads = numThreads + atoi(itr->second.c_str());
219 		}
220 	}
221 	std::cout << "calculated total num of threads: " << numThreads << "\n";
222 	size_t expectednumThreads = getnumThreads();
223 	EXPECT_EQ(expectednumThreads, numThreads);
224 }
225 
TEST_F(PriorityTest, NumAuxIOThreadsTest)226 TEST_F(PriorityTest, NumAuxIOThreadsTest)
227 {
228 	(void) lcb_set_stat_callback(instance, stats_generic_callback);
229 	err = lcb_connect(instance);
230 	assert(err == LCB_SUCCESS);
231 	std::string workload("workload");
232 	StatsVector workloadstats = getgenericstats(workload, instance);
233 	std::cout << "vector length : " << workloadstats.size() << "\n";
234 	size_t numAuxIOThreads = 0;
235 	for (StatsVector::iterator itr = workloadstats.begin(); itr != workloadstats.end(); ++itr)
236 	{
237 		if (!itr->first.compare("ep_workload:num_auxio"))
238 		{
239 			numAuxIOThreads = atoi(itr->second.c_str());
240 		}
241 	}
242 	size_t expectednumAuxIOThreads = getNumAuxIO();
243 	std::cout << "calculated num of AuxIO threads: " << expectednumAuxIOThreads << "\n";
244 	EXPECT_EQ(expectednumAuxIOThreads, numAuxIOThreads);
245 }
246 
TEST_F(PriorityTest, NumnonIOThreadsTest)247 TEST_F(PriorityTest, NumnonIOThreadsTest)
248 {
249 	(void) lcb_set_stat_callback(instance, stats_generic_callback);
250 	err = lcb_connect(instance);
251 	assert(err == LCB_SUCCESS);
252 	std::string workload("workload");
253 	StatsVector workloadstats = getgenericstats(workload, instance);
254 	std::cout << "vector length : " << workloadstats.size() << "\n";
255 	size_t numNonIOThreads = 0;
256 	for (StatsVector::iterator itr = workloadstats.begin(); itr != workloadstats.end(); ++itr)
257 	{
258 		if (!itr->first.compare("ep_workload:num_nonio"))
259 		{
260 			numNonIOThreads = atoi(itr->second.c_str());
261 		}
262 	}
263 	size_t expectednumNonIOThreads = getNumNonIO();
264 	std::cout << "calculated total num of nonio threads: " << expectednumNonIOThreads << "\n";
265 	EXPECT_EQ(expectednumNonIOThreads, numNonIOThreads);
266 }
267 
TEST_F(PriorityTest, NumWriterThreadsTest)268 TEST_F(PriorityTest, NumWriterThreadsTest)
269 {
270 	(void) lcb_set_stat_callback(instance, stats_generic_callback);
271 	err = lcb_connect(instance);
272 	assert(err == LCB_SUCCESS);
273 	std::string workload("workload");
274 	StatsVector workloadstats = getgenericstats(workload, instance);
275 	std::cout << "vector length : " << workloadstats.size() << "\n";
276 	size_t numWriterThreads = 0;
277 	for (StatsVector::iterator itr = workloadstats.begin(); itr != workloadstats.end(); ++itr)
278 	{
279 		if (!itr->first.compare("ep_workload:num_writers"))
280 		{
281 			numWriterThreads = atoi(itr->second.c_str());
282 		}
283 	}
284 	size_t expectednumWriterThreads = getNumWriters();
285 	std::cout << "calculated num of writer threads: " << expectednumWriterThreads << "\n";
286 	EXPECT_EQ(expectednumWriterThreads, numWriterThreads);
287 }
288 
TEST_F(PriorityTest, NumReaderThreadsTest)289 TEST_F(PriorityTest, NumReaderThreadsTest)
290 {
291 	(void) lcb_set_stat_callback(instance, stats_generic_callback);
292 	err = lcb_connect(instance);
293 	assert(err == LCB_SUCCESS);
294 	std::string workload("workload");
295 	StatsVector workloadstats = getgenericstats(workload, instance);
296 	std::cout << "vector length : " << workloadstats.size() << "\n";
297 	size_t numReaderThreads = 0;
298 	for (StatsVector::iterator itr = workloadstats.begin(); itr != workloadstats.end(); ++itr)
299 	{
300 		if (!itr->first.compare("ep_workload:num_readers"))
301 		{
302 			numReaderThreads = atoi(itr->second.c_str());
303 		}
304 	}
305 	size_t expectednumReaderThreads = getNumReaders();
306 	std::cout << "calculated num of reader threads: " << expectednumReaderThreads << "\n";
307 	EXPECT_EQ(expectednumReaderThreads, numReaderThreads);
308 }
309 
TEST_F(PriorityTest, HighLowStorageTest)310 TEST_F(PriorityTest, HighLowStorageTest)
311 {
312 	(void) lcb_set_stat_callback(instance, stats_generic_callback);
313 	err = lcb_connect(instance);
314 	assert(err == LCB_SUCCESS);
315 	bool highprio = true;
316 	createBucketwithPriority(highprio, "highprio");
317 	lcb_t inst_second;       //second lcb instance to connect to second bucket
318 	int numargs = 2;
319 	const char* bucketname = "highprio";
320 	//create connection to bucket highprio
321 	createlcbinstance(&inst_second, testargv[1], (char*) bucketname, NULL, numargs);
322 	(void) lcb_set_stat_callback(inst_second, stats_generic_callback);
323 	err = lcb_connect(inst_second);
324 	assert(err == LCB_SUCCESS);
325 	std::vector<lcb_t*> instancevec;
326 	instancevec.push_back(&instance);
327 	instancevec.push_back(&inst_second);
328 	insert_items_instances(100000, instancevec);
329 	std::string timings("timings");
330 	StatsVector timingstats = getgenericstats(timings, inst_second);
331 	std::cout << "vector length : " << timingstats.size() << "\n";
332 	int highstoragebin1 = 0;
333 	int highstoragebin2 = 0;
334 	for (StatsVector::iterator itr = timingstats.begin(); itr != timingstats.end(); ++itr)
335 	{
336 		if (!itr->first.compare("storage_age_0,1000000"))
337 		{
338 			std::cout << "storage_age_0,1000000 : " << itr->second << "\n";
339 			highstoragebin1 = atoi(itr->second.c_str());
340 		}
341 		else if (!itr->first.compare("storage_age_1000000,2400000"))
342 		{
343 			std::cout << "storage_age_1000000,2400000 : " << itr->second << "\n";
344 			highstoragebin2 = atoi(itr->second.c_str());
345 		}
346 	}
347 	genericstats.refcount = 0;
348 	genericstats.statsvec.clear();
349 	int lowstoragebin1 = 0;
350 	int lowstoragebin2 = 0;
351 	StatsVector timingstatsbucket2 = getgenericstats(timings, instance);
352 	std::cout << "vector length : " << timingstatsbucket2.size() << "\n";
353 	for (StatsVector::iterator itr = timingstatsbucket2.begin(); itr != timingstatsbucket2.end(); ++itr)
354 	{
355 		if (!itr->first.compare("storage_age_0,1000000"))
356 		{
357 			std::cout << "storage_age_0,1000000 : " << itr->second << "\n";
358 			lowstoragebin1 = atoi(itr->second.c_str());
359 		}
360 		else if (!itr->first.compare("storage_age_1000000,2400000"))
361 		{
362 			lowstoragebin2 = atoi(itr->second.c_str());
363 			std::cout << "storage_age_1000000,2400000 : " << itr->second << "\n";
364 		}
365 	}
366 	EXPECT_GT(highstoragebin1, lowstoragebin1);
367 	EXPECT_GT(highstoragebin2, lowstoragebin2);
368 }
369 
TEST_F(PriorityTest, SingleBucketStorageAgeTest)370 TEST_F(PriorityTest, SingleBucketStorageAgeTest)
371 {
372 	(void) lcb_set_stat_callback(instance, stats_generic_callback);
373 	err = lcb_connect(instance);
374 	assert(err == LCB_SUCCESS);
375 	std::vector<lcb_t*> instancevec;
376 	instancevec.push_back(&instance);
377 	insert_items_instances(100000, instancevec);
378 	std::string timings("timings");
379 	StatsVector timingstats = getgenericstats(timings, instance);
380 	std::cout << "vector length : " << timingstats.size() << "\n";
381 	int lowstoragebin1 = 0;
382 	int lowstoragebin2 = 0;
383 	for (StatsVector::iterator itr = timingstats.begin(); itr != timingstats.end(); ++itr)
384 	{
385 		if (!itr->first.compare("storage_age_0,1000000"))
386 		{
387 			std::cout << "storage_age_0,1000000 : " << itr->second << "\n";
388 			lowstoragebin1 = atoi(itr->second.c_str());
389 		}
390 		else if (!itr->first.compare("storage_age_1000000,2400000"))
391 		{
392 			std::cout << "storage_age_1000000,2400000 : " << itr->second << "\n";
393 			lowstoragebin2 = atoi(itr->second.c_str());
394 		}
395 	}
396 	genericstats.refcount = 0;
397 	genericstats.statsvec.clear();
398 	deleteBucket("default");
399 	bool highprio = true;
400 	createBucketwithPriority(highprio, "highprio");
401 	lcb_t inst_second;       //second lcb instance to connect to second bucket
402 	int numargs = 2;
403 	const char* bucketname = "highprio";
404 	//create connection to bucket highprio
405 	createlcbinstance(&inst_second, testargv[1], (char*) bucketname, NULL, numargs);
406 	(void) lcb_set_stat_callback(inst_second, stats_generic_callback);
407 	err = lcb_connect(inst_second);
408 	assert(err == LCB_SUCCESS);
409 	instancevec.clear();
410 	instancevec.push_back(&inst_second);
411 	insert_items_instances(100000, instancevec);
412 	StatsVector timingstatsbucket2 = getgenericstats(timings, inst_second);
413 	std::cout << "vector length : " << timingstatsbucket2.size() << "\n";
414 	int highstoragebin1 = 0;
415 	int highstoragebin2 = 0;
416 	for (StatsVector::iterator itr = timingstatsbucket2.begin(); itr != timingstatsbucket2.end(); ++itr)
417 	{
418 		if (!itr->first.compare("storage_age_0,1000000"))
419 		{
420 			std::cout << "storage_age_0,1000000 : " << itr->second << "\n";
421 			highstoragebin1 = atoi(itr->second.c_str());
422 		}
423 		else if (!itr->first.compare("storage_age_1000000,2400000"))
424 		{
425 			highstoragebin2 = atoi(itr->second.c_str());
426 			std::cout << "storage_age_1000000,2400000 : " << itr->second << "\n";
427 		}
428 	}
429 	verifywithintolerance(highstoragebin1, lowstoragebin1);
430 	verifywithintolerance(highstoragebin2, lowstoragebin2);
431 }
432 
main(int argc, char *argv[])433 int main(int argc, char *argv[])
434 {
435 	::testing::InitGoogleTest(&argc, argv);
436 	testargv = argv;
437 	testargc = argc;
438 	return RUN_ALL_TESTS();
439 }
440