1 /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2 /*
3  *     Copyright 2013 Couchbase, Inc
4  *
5  *   Licensed under the Apache License, Version 2.0 (the "License");
6  *   you may not use this file except in compliance with the License.
7  *   You may obtain a copy of the License at
8  *
9  *       http://www.apache.org/licenses/LICENSE-2.0
10  *
11  *   Unless required by applicable law or agreed to in writing, software
12  *   distributed under the License is distributed on an "AS IS" BASIS,
13  *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14  *   See the License for the specific language governing permissions and
15  *   limitations under the License.
16  */
17 #include <platform/cb_malloc.h>
18 #include <platform/dirutils.h>
19 
20 #ifdef _MSC_VER
21 #include <direct.h>
22 #include <folly/portability/Windows.h>
23 #include <shlobj.h>
24 #define rmdir _rmdir
25 #include <io.h> // _setmode
26 #else
27 
28 #include <dirent.h>
29 #include <dlfcn.h>
30 #include <sys/resource.h>
31 #include <unistd.h>
32 
33 #endif
34 
35 #include <platform/memorymap.h>
36 #include <platform/strerror.h>
37 
38 #include <fcntl.h>
39 #include <stdio.h>
40 #include <string.h>
41 #include <sys/stat.h>
42 #include <chrono>
43 #include <limits>
44 #include <system_error>
45 
split(const std::string& input, bool directory)46 static std::string split(const std::string& input, bool directory) {
47     std::string::size_type path = input.find_last_of("\\/");
48     std::string file;
49     std::string dir;
50 
51     if (path == std::string::npos) {
52         dir = ".";
53         file = input;
54     } else {
55         dir = input.substr(0, path);
56         if (dir.length() == 0) {
57             dir = input.substr(0, 1);
58         }
59 
60         while (dir.length() > 1 &&
61                dir.find_last_of("\\/") == (dir.length() - 1)) {
62             if (dir.length() > 1) {
63                 dir.resize(dir.length() - 1);
64             }
65         }
66 
67         file = input.substr(path + 1);
68     }
69 
70     if (directory) {
71         return dir;
72     } else {
73         return file;
74     }
75 }
76 
77 DIRUTILS_PUBLIC_API
dirname(const std::string& dir)78 std::string cb::io::dirname(const std::string& dir) {
79     return split(dir, true);
80 }
81 
82 DIRUTILS_PUBLIC_API
basename(const std::string& name)83 std::string cb::io::basename(const std::string& name) {
84     return split(name, false);
85 }
86 
87 #ifdef _MSC_VER
88 DIRUTILS_PUBLIC_API
findFilesWithPrefix(const std::string &dir, const std::string &name)89 std::vector<std::string> cb::io::findFilesWithPrefix(const std::string &dir,
90                                                      const std::string &name)
91 {
92     std::vector<std::string> files;
93     std::string match = dir + "\\" + name + "*";
94     WIN32_FIND_DATA FindFileData;
95 
96     HANDLE hFind = FindFirstFileEx(match.c_str(), FindExInfoStandard,
97                                    &FindFileData, FindExSearchNameMatch,
98                                    NULL, 0);
99 
100     if (hFind != INVALID_HANDLE_VALUE) {
101         do {
102             std::string fnm(FindFileData.cFileName);
103             if (fnm != "." && fnm != "..") {
104                 std::string entry = dir;
105                 entry.append("\\");
106                 entry.append(FindFileData.cFileName);
107                 files.push_back(entry);
108             }
109         } while (FindNextFile(hFind, &FindFileData));
110 
111         FindClose(hFind);
112     }
113     return files;
114 }
115 #else
116 
117 DIRUTILS_PUBLIC_API
findFilesWithPrefix(const std::string& dir, const std::string& name)118 std::vector<std::string> cb::io::findFilesWithPrefix(const std::string& dir,
119                                                      const std::string& name) {
120     std::vector<std::string> files;
121     DIR* dp = opendir(dir.c_str());
122     if (dp != NULL) {
123         struct dirent* de;
124         while ((de = readdir(dp)) != NULL) {
125             std::string fnm(de->d_name);
126             if (fnm == "." || fnm == "..") {
127                 continue;
128             }
129             if (strncmp(de->d_name, name.c_str(), name.length()) == 0) {
130                 std::string entry = dir;
131                 entry.append("/");
132                 entry.append(de->d_name);
133                 files.push_back(entry);
134             }
135         }
136 
137         closedir(dp);
138     }
139     return files;
140 }
141 #endif
142 
143 DIRUTILS_PUBLIC_API
findFilesWithPrefix(const std::string& name)144 std::vector<std::string> cb::io::findFilesWithPrefix(const std::string& name) {
145     return findFilesWithPrefix(dirname(name), basename(name));
146 }
147 
148 #ifdef _MSC_VER
149 DIRUTILS_PUBLIC_API
findFilesContaining(const std::string &dir, const std::string &name)150 std::vector<std::string> cb::io::findFilesContaining(const std::string &dir,
151                                                             const std::string &name)
152 {
153     std::vector<std::string> files;
154     std::string match = dir + "\\*" + name + "*";
155     WIN32_FIND_DATA FindFileData;
156 
157     HANDLE hFind = FindFirstFileEx(match.c_str(), FindExInfoStandard,
158                                    &FindFileData, FindExSearchNameMatch,
159                                    NULL, 0);
160 
161     if (hFind != INVALID_HANDLE_VALUE) {
162         do {
163             std::string fnm(FindFileData.cFileName);
164             if (fnm != "." && fnm != "..") {
165                 std::string entry = dir;
166                 entry.append("\\");
167                 entry.append(FindFileData.cFileName);
168                 files.push_back(entry);
169             }
170         } while (FindNextFile(hFind, &FindFileData));
171 
172         FindClose(hFind);
173     }
174     return files;
175 }
176 #else
177 
178 DIRUTILS_PUBLIC_API
findFilesContaining(const std::string& dir, const std::string& name)179 std::vector<std::string> cb::io::findFilesContaining(const std::string& dir,
180                                                             const std::string& name) {
181     std::vector<std::string> files;
182     DIR* dp = opendir(dir.c_str());
183     if (dp != NULL) {
184         struct dirent* de;
185         while ((de = readdir(dp)) != NULL) {
186             if (name.empty() || strstr(de->d_name, name.c_str()) != NULL) {
187                 std::string fnm(de->d_name);
188                 if (fnm != "." && fnm != "..") {
189                     std::string entry = dir;
190                     entry.append("/");
191                     entry.append(de->d_name);
192                     files.push_back(entry);
193                 }
194             }
195         }
196 
197         closedir(dp);
198     }
199 
200     return files;
201 }
202 #endif
203 
204 DIRUTILS_PUBLIC_API
rmrf(const std::string& path)205 void cb::io::rmrf(const std::string& path) {
206     struct stat st;
207     if (stat(path.c_str(), &st) == -1) {
208         throw std::system_error(errno, std::system_category(),
209                                 "cb::io::rmrf: stat of " +
210                                 path + " failed");
211     }
212 
213     if ((st.st_mode & S_IFDIR) != S_IFDIR) {
214         if (remove(path.c_str()) != 0) {
215             throw std::system_error(errno, std::system_category(),
216                                     "cb::io::rmrf: remove of " +
217                                     path + " failed");
218         }
219         return;
220     }
221 
222     if (rmdir(path.c_str()) == 0) {
223         return;
224     }
225 
226     // Ok, this is a directory. Go ahead and delete it recursively
227     std::vector<std::string> directories;
228     std::vector<std::string> emptyDirectories;
229     directories.push_back(path);
230 
231     // Iterate all the files/directories found in path, when we encounter
232     // a sub-directory, that is added to the directories vector so we move
233     // deeper into the path.
234     do {
235         std::vector<std::string> vec =
236                 findFilesContaining(directories.back(), "");
237         emptyDirectories.push_back(directories.back());
238         directories.pop_back();
239         std::vector<std::string>::iterator ii;
240 
241         for (ii = vec.begin(); ii != vec.end(); ++ii) {
242             if (stat(ii->c_str(), &st) == -1) {
243                 throw std::system_error(errno, std::system_category(),
244                                         "cb::io::rmrf: stat of file/directory " +
245                                         *ii + " under " + path + " failed");
246             }
247 
248             if ((st.st_mode & S_IFDIR) == S_IFDIR) {
249                 if (rmdir(ii->c_str()) != 0) {
250                     directories.push_back(*ii);
251                 }
252             } else if (remove(ii->c_str()) != 0) {
253                 throw std::system_error(errno,
254                                         std::system_category(),
255                                         "cb::io::rmrf: remove of file " + *ii +
256                                                 " under " + path + " failed");
257             }
258         }
259     } while (!directories.empty());
260 
261     // Finally iterate our list of now empty directories, we reverse iterate so
262     // that we remove the deepest first
263     for (auto itr = emptyDirectories.rbegin(); itr != emptyDirectories.rend();
264          ++itr) {
265         if (rmdir(itr->c_str()) != 0) {
266             throw std::system_error(
267                     errno,
268                     std::system_category(),
269                     "cb::io::rmrf: remove of directory " + *itr + " failed");
270         }
271     }
272 }
273 
274 DIRUTILS_PUBLIC_API
isDirectory(const std::string& directory)275 bool cb::io::isDirectory(const std::string& directory) {
276 #ifdef WIN32
277     DWORD dwAttrib = GetFileAttributes(directory.c_str());
278     if (dwAttrib == INVALID_FILE_ATTRIBUTES) {
279         return false;
280     }
281     return (dwAttrib & FILE_ATTRIBUTE_DIRECTORY) == FILE_ATTRIBUTE_DIRECTORY;
282 #else
283     struct stat st;
284     if (stat(directory.c_str(), &st) == -1) {
285         return false;
286     }
287     return (S_ISDIR(st.st_mode));
288 #endif
289 }
290 
291 DIRUTILS_PUBLIC_API
isFile(const std::string& file)292 bool cb::io::isFile(const std::string& file) {
293 #ifdef WIN32
294     DWORD dwAttrib = GetFileAttributes(file.c_str());
295     if (dwAttrib == INVALID_FILE_ATTRIBUTES) {
296         return false;
297     }
298     return (dwAttrib & FILE_ATTRIBUTE_DIRECTORY) == 0;
299 #else
300     struct stat st;
301     if (stat(file.c_str(), &st) == -1) {
302         return false;
303     }
304     return (S_ISREG(st.st_mode) || S_ISLNK(st.st_mode));
305 #endif
306 }
307 
308 DIRUTILS_PUBLIC_API
mkdirp(const std::string& directory)309 void cb::io::mkdirp(const std::string& directory) {
310     // Bail out immediately if the directory already exists.
311     // note that both mkdir and CreateDirectory on windows returns
312     // EEXISTS if the directory already exists, BUT it could also
313     // return "permission denied" depending on the order the checks
314     // is run within "libc"
315     if (isDirectory(directory)) {
316         return;
317     }
318 
319 #ifdef WIN32
320     // ensure directory path is absolute
321     // Initial call without buffer to find size of buffer needed
322     auto ret = GetFullPathNameA(directory.c_str(), 0, nullptr, nullptr);
323     if (!ret) {
324         throw std::system_error(GetLastError(),
325                                 std::system_category(),
326                                 "cb::io::mkdirp(\"" + directory +
327                                         "\") failed - could not get buffer "
328                                         "size for absolute path.");
329     }
330 
331     // make string of right size
332     std::string absPath(ret, '\0');
333     // populate absPath with the path
334     ret = GetFullPathNameA(directory.c_str(), /* rel or abs path */
335                            (DWORD)absPath.size(), /* dest buffer size */
336                            &absPath[0], /* dest buffer */
337                            nullptr /* don't need ptr to filePart */
338     );
339 
340     if (!ret) {
341         throw std::system_error(
342                 GetLastError(),
343                 std::system_category(),
344                 "cb::io::mkdirp(\"" + directory +
345                         "\") failed - could not get absolute path.");
346     }
347 
348     auto err = SHCreateDirectoryEx(nullptr, /* no window handle */
349                                    absPath.data(),
350                                    nullptr /* no security attrs */
351     );
352     if (err != ERROR_SUCCESS) {
353         throw std::system_error(
354                 err,
355                 std::system_category(),
356                 "cb::io::mkdirp(\"" + directory +
357                         "\") failed, could not create directory.");
358     }
359 #else
360     do {
361         if (mkdir(directory.c_str(), S_IREAD | S_IWRITE | S_IEXEC) == 0) {
362             return;
363         }
364 
365         switch (errno) {
366         case EEXIST:
367             return;
368         case ENOENT:
369             break;
370         default:
371             throw std::system_error(errno, std::system_category(),
372                                     "cb::io::mkdirp(\"" + directory +
373                                     "\") failed");
374         }
375 
376         // Try to create the parent directory..
377         mkdirp(dirname(directory));
378     } while (true);
379 #endif
380 }
381 
mktemp(const std::string& prefix)382 std::string cb::io::mktemp(const std::string& prefix) {
383     static const std::string patternmask{"XXXXXX"};
384     std::string pattern = prefix;
385 
386     auto index = pattern.find(patternmask);
387     if (index == pattern.npos) {
388         index = pattern.size();
389         pattern.append(patternmask);
390     }
391 
392     auto* ptr = const_cast<char*>(pattern.data()) + index;
393     auto counter = std::chrono::steady_clock::now().time_since_epoch().count();
394 
395     do {
396         ++counter;
397         sprintf(ptr, "%06" PRIu64, static_cast<uint64_t>(counter) % 1000000);
398 
399 #ifdef WIN32
400         HANDLE handle = CreateFile(pattern.c_str(),
401                                    GENERIC_READ | GENERIC_WRITE,
402                                    0,
403                                    NULL,
404                                    CREATE_NEW,
405                                    FILE_ATTRIBUTE_NORMAL,
406                                    NULL);
407         if (handle != INVALID_HANDLE_VALUE) {
408             CloseHandle(handle);
409             return pattern;
410         }
411 #else
412         int fd = open(pattern.c_str(),
413                       O_WRONLY | O_EXCL | O_CREAT,
414                       S_IRUSR | S_IWUSR);
415         if (fd != -1) {
416             close(fd);
417             return pattern;
418         }
419 #endif
420 
421     } while (true);
422 }
423 
mkdtemp(const std::string& prefix)424 std::string cb::io::mkdtemp(const std::string& prefix) {
425     static const std::string patternmask{"XXXXXX"};
426     std::string pattern = prefix;
427 
428     auto index = pattern.find(patternmask);
429     char* ptr;
430     if (index == pattern.npos) {
431         index = pattern.size();
432         pattern.append(patternmask);
433     }
434 
435     ptr = const_cast<char*>(pattern.data()) + index;
436     int searching = 1;
437     auto counter = std::chrono::steady_clock::now().time_since_epoch().count();
438 
439     do {
440         ++counter;
441         sprintf(ptr, "%06" PRIu64, static_cast<uint64_t>(counter) % 1000000);
442 
443 #ifdef WIN32
444         if (CreateDirectory(pattern.c_str(), nullptr)) {
445             searching = 0;
446         }
447 #else
448         if (mkdir(pattern.c_str(), S_IREAD | S_IWRITE | S_IEXEC) == 0) {
449             searching = 0;
450         }
451 #endif
452 
453     } while (searching);
454 
455     return pattern;
456 }
457 
getcwd(void)458 std::string cb::io::getcwd(void) {
459     std::string result(4096, 0);
460 #ifdef WIN32
461     if (GetCurrentDirectory(result.size(), &result[0]) == 0) {
462         throw std::system_error(GetLastError(), std::system_category(),
463                                 "Failed to determine current working directory");
464     }
465 #else
466     if (::getcwd(&result[0], result.size()) == nullptr) {
467         throw std::system_error(errno, std::system_category(),
468                                 "Failed to determine current working directory");
469     }
470 #endif
471 
472     // Trim off any trailing \0 characters.
473     result.resize(strlen(result.c_str()));
474     return result;
475 }
476 
maximizeFileDescriptors(uint64_t limit)477 uint64_t cb::io::maximizeFileDescriptors(uint64_t limit) {
478 #ifdef WIN32
479     return limit;
480 #else
481     struct rlimit rlim;
482 
483     if (getrlimit(RLIMIT_NOFILE, &rlim) != 0) {
484         throw std::system_error(errno, std::system_category(),
485                                 "getrlimit(RLIMIT_NOFILE, &rlim) failed");
486     } else if (limit <= rlim.rlim_cur) {
487         return uint64_t(rlim.rlim_cur);
488     } else {
489         struct rlimit org = rlim;
490 
491         rlim.rlim_cur = limit;
492 
493         // we don't want to limit the current max if it is higher ;)
494         if (rlim.rlim_max < limit) {
495             rlim.rlim_max = limit;
496         }
497 
498         if (setrlimit(RLIMIT_NOFILE, &rlim) == 0) {
499             return limit;
500         }
501 
502         // Ok, we failed to get what we wanted.. Try a binary search
503         // to go as high as we can...
504         auto min = org.rlim_cur;
505         auto max = limit;
506         rlim_t last_good = 0;
507 
508         while (min <= max) {
509             rlim_t avg;
510 
511             // make sure we don't overflow
512             uint64_t high = std::numeric_limits<uint64_t>::max() - min;
513             if (high < max) {
514                 avg = max / 2;
515             } else {
516                 avg = (min + max) / 2;
517             }
518 
519             rlim.rlim_cur = rlim.rlim_max = avg;
520             if (setrlimit(RLIMIT_NOFILE, &rlim) == 0) {
521                 last_good = avg;
522                 min = avg + 1;
523             } else {
524                 max = avg - 1;
525             }
526         }
527 
528         if (last_good == 0) {
529             // all setrlimit's failed... lets go fetch it again
530             if (getrlimit(RLIMIT_NOFILE, &rlim) != 0) {
531                 throw std::system_error(errno, std::system_category(),
532                                         "getrlimit(RLIMIT_NOFILE, &rlim) failed");
533             }
534             return uint64_t(rlim.rlim_cur);
535         }
536 
537         return uint64_t(last_good);
538     }
539 #endif
540 }
541 
542 DIRUTILS_PUBLIC_API
loadFile(const std::string& name)543 std::string cb::io::loadFile(const std::string& name) {
544     MemoryMappedFile map(name.c_str(), MemoryMappedFile::Mode::RDONLY);
545     return to_string(map.content());
546 }
547 
setBinaryMode(FILE* fp)548 void cb::io::setBinaryMode(FILE* fp) {
549 #ifdef WIN32
550     if (_setmode(_fileno(fp), _O_BINARY) == -1) {
551         throw std::system_error(
552                 errno, std::system_category(), "cb::io::setBinaryMode");
553     }
554 #else
555     (void)fp;
556 #endif
557 }
558 
559 namespace cb {
560 namespace io {
561 
562 LibraryHandle::~LibraryHandle() = default;
563 
564 class LibraryHandleImpl : public LibraryHandle {
565 public:
LibraryHandleImpl(std::string soname)566     LibraryHandleImpl(std::string soname) : soname(std::move(soname)) {
567         if (LibraryHandleImpl::soname.empty()) {
568             throw std::invalid_argument(
569                     "LibraryHandleImpl: shared object name cannot be empty");
570         }
571         openDynamicLibrary();
572     }
573 
574     ~LibraryHandleImpl() override {
575 #ifdef WIN32
576         FreeLibrary(handle);
577 #else
578         dlclose(handle);
579 #endif
580     }
581 
582     void* findSymbol(const std::string& symbol) const override {
583 #ifdef WIN32
584         auto* ret = GetProcAddress(handle, symbol.c_str());
585         if (ret == nullptr) {
586             throw std::runtime_error(cb_strerror());
587         }
588 #else
589         // Clear the error status
590         dlerror();
591         void* ret = dlsym(handle, symbol.c_str());
592         const auto* error = dlerror();
593         if (ret == nullptr && error != nullptr) {
594             throw std::runtime_error(error);
595         }
596 #endif
597         return ret;
598     }
599 
600     std::string getName() const override {
601         return soname;
602     }
603 
604 protected:
openDynamicLibrary()605     void openDynamicLibrary() {
606 #ifdef WIN32
607         sanitizePath(soname);
608         handle = LoadLibrary(soname.c_str());
609         if (handle == nullptr) {
610             auto alternative = getAlternativeSoName();
611             if (alternative != soname) {
612                 soname = std::move(alternative);
613                 handle = LoadLibrary(soname.c_str());
614             }
615 
616             if (handle == nullptr) {
617                 throw std::runtime_error(cb_strerror());
618             }
619         }
620 #else
621         const int dlopen_flags = RTLD_NOW | RTLD_GLOBAL;
622         handle = dlopen(soname.c_str(), dlopen_flags);
623         if (handle == nullptr) {
624             auto alternative = getAlternativeSoName();
625             if (alternative != soname) {
626                 soname = std::move(alternative);
627                 handle = dlopen(soname.c_str(), dlopen_flags);
628             }
629 
630             if (handle == nullptr) {
631                 throw std::runtime_error(dlerror());
632             }
633         }
634 #endif
635     }
636 
637     // We might want to add .so or .dylib to the name if it isn't there
getAlternativeSoName()638     std::string getAlternativeSoName() {
639 #ifdef WIN32
640         const std::string ext = ".dll";
641 #else
642         const std::string ext = ".so";
643 #endif
644         if (soname.find(ext) != std::string::npos) {
645             // already fixed
646             return soname;
647         }
648 
649 #ifdef WIN32
650         auto index = soname.find(".so");
651         if (index != std::string::npos) {
652             auto ret = soname.substr(0, index) + ext;
653             return ret;
654         }
655 #endif
656 
657         return soname + ext;
658     }
659 
660     std::string soname;
661 #ifdef WIN32
662     HMODULE handle;
663 #else
664     void* handle = nullptr;
665 #endif
666 };
667 
668 DIRUTILS_PUBLIC_API
loadLibrary(const std::string& filename)669 std::unique_ptr<LibraryHandle> loadLibrary(const std::string& filename) {
670     return std::make_unique<LibraryHandleImpl>(filename);
671 }
672 
673 } // namespace io
674 } // namespace cb
675