1package common
2
3import "errors"
4import "expvar"
5import "fmt"
6import "io"
7import "io/ioutil"
8import "path/filepath"
9import "net"
10import "net/url"
11import "os"
12import "strconv"
13import "strings"
14import "net/http"
15import "net/http/pprof"
16import "runtime"
17import "hash/crc64"
18import "reflect"
19import "unsafe"
20import "regexp"
21import "time"
22import "math/big"
23
24import "github.com/couchbase/cbauth"
25import "github.com/couchbase/cbauth/cbauthimpl"
26import "github.com/couchbase/indexing/secondary/dcp"
27import "github.com/couchbase/indexing/secondary/dcp/transport/client"
28import "github.com/couchbase/indexing/secondary/logging"
29
30const IndexNamePattern = "^[A-Za-z0-9#_-]+$"
31
32const (
33	MAX_AUTH_RETRIES = 10
34)
35
36var ErrInvalidIndexName = fmt.Errorf("Invalid index name")
37
38var _isIpv6 bool
39
40// ExcludeStrings will exclude strings in `excludes` from `strs`. preserves the
41// order of `strs` in the result.
42func ExcludeStrings(strs []string, excludes []string) []string {
43	cache := make(map[string]bool)
44	for _, s := range excludes {
45		cache[s] = true
46	}
47	ss := make([]string, 0, len(strs))
48	for _, s := range strs {
49		if _, ok := cache[s]; ok == false {
50			ss = append(ss, s)
51		}
52	}
53	return ss
54}
55
56// CommonStrings returns intersection of two set of strings.
57func CommonStrings(xs []string, ys []string) []string {
58	ss := make([]string, 0, len(xs))
59	cache := make(map[string]bool)
60	for _, x := range xs {
61		cache[x] = true
62	}
63	for _, y := range ys {
64		if _, ok := cache[y]; ok {
65			ss = append(ss, y)
66		}
67	}
68	return ss
69}
70
71// HasString does membership check for a string.
72func HasString(str string, strs []string) bool {
73	for _, s := range strs {
74		if str == s {
75			return true
76		}
77	}
78	return false
79}
80
81// ExcludeUint32 remove items from list.
82func ExcludeUint32(xs []uint32, from []uint32) []uint32 {
83	fromSubXs := make([]uint32, 0, len(from))
84	for _, num := range from {
85		if HasUint32(num, xs) == false {
86			fromSubXs = append(fromSubXs, num)
87		}
88	}
89	return fromSubXs
90}
91
92// ExcludeUint64 remove items from list.
93func ExcludeUint64(xs []uint64, from []uint64) []uint64 {
94	fromSubXs := make([]uint64, 0, len(from))
95	for _, num := range from {
96		if HasUint64(num, xs) == false {
97			fromSubXs = append(fromSubXs, num)
98		}
99	}
100	return fromSubXs
101}
102
103// RemoveUint32 delete `item` from list `xs`.
104func RemoveUint32(item uint32, xs []uint32) []uint32 {
105	ys := make([]uint32, 0, len(xs))
106	for _, x := range xs {
107		if x == item {
108			continue
109		}
110		ys = append(ys, x)
111	}
112	return ys
113}
114
115// RemoveUint16 delete `item` from list `xs`.
116func RemoveUint16(item uint16, xs []uint16) []uint16 {
117	ys := make([]uint16, 0, len(xs))
118	for _, x := range xs {
119		if x == item {
120			continue
121		}
122		ys = append(ys, x)
123	}
124	return ys
125}
126
127// RemoveString delete `item` from list `xs`.
128func RemoveString(item string, xs []string) []string {
129	ys := make([]string, 0, len(xs))
130	for _, x := range xs {
131		if x == item {
132			continue
133		}
134		ys = append(ys, x)
135	}
136	return ys
137}
138
139// HasUint32 does membership check for a uint32 integer.
140func HasUint32(item uint32, xs []uint32) bool {
141	for _, x := range xs {
142		if x == item {
143			return true
144		}
145	}
146	return false
147}
148
149// HasUint64 does membership check for a uint32 integer.
150func HasUint64(item uint64, xs []uint64) bool {
151	for _, x := range xs {
152		if x == item {
153			return true
154		}
155	}
156	return false
157}
158
159// FailsafeOp can be used by gen-server implementors to avoid infinitely
160// blocked API calls.
161func FailsafeOp(
162	reqch, respch chan []interface{},
163	cmd []interface{},
164	finch chan bool) ([]interface{}, error) {
165
166	select {
167	case reqch <- cmd:
168		if respch != nil {
169			select {
170			case resp := <-respch:
171				return resp, nil
172			case <-finch:
173				return nil, ErrorClosed
174			}
175		}
176	case <-finch:
177		return nil, ErrorClosed
178	}
179	return nil, nil
180}
181
182// FailsafeOpAsync is same as FailsafeOp that can be used for
183// asynchronous operation, that is, caller does not wait for response.
184func FailsafeOpAsync(
185	reqch chan []interface{}, cmd []interface{}, finch chan bool) error {
186
187	select {
188	case reqch <- cmd:
189	case <-finch:
190		return ErrorClosed
191	}
192	return nil
193}
194
195// FailsafeOpNoblock is same as FailsafeOpAsync that can be used for
196// non-blocking operation, that is, if `reqch` is full caller does not block.
197func FailsafeOpNoblock(
198	reqch chan []interface{}, cmd []interface{}, finch chan bool) error {
199
200	select {
201	case reqch <- cmd:
202	case <-finch:
203		return ErrorClosed
204	default:
205		return ErrorChannelFull
206	}
207	return nil
208}
209
210// OpError suppliments FailsafeOp used by gen-servers.
211func OpError(err error, vals []interface{}, idx int) error {
212	if err != nil {
213		return err
214	} else if vals != nil {
215		if vals[idx] != nil {
216			return vals[idx].(error)
217		} else {
218			return nil
219		}
220	}
221	return nil
222}
223
224// cbauth admin authentication helper
225// Uses default cbauth env variables internally to provide auth creds
226type CbAuthHandler struct {
227	Hostport string
228	Bucket   string
229}
230
231func (ah *CbAuthHandler) GetCredentials() (string, string) {
232
233	var u, p string
234
235	fn := func(r int, err error) error {
236		if r > 0 {
237			logging.Warnf("CbAuthHandler::GetCredentials error=%v Retrying (%d)", err, r)
238		}
239
240		u, p, err = cbauth.GetHTTPServiceAuth(ah.Hostport)
241		return err
242	}
243
244	rh := NewRetryHelper(MAX_AUTH_RETRIES, time.Second, 2, fn)
245	err := rh.Run()
246	if err != nil {
247		panic(err)
248	}
249
250	return u, p
251}
252
253func (ah *CbAuthHandler) AuthenticateMemcachedConn(host string, conn *memcached.Client) error {
254
255	var u, p string
256
257	fn := func(r int, err error) error {
258		if r > 0 {
259			logging.Warnf("CbAuthHandler::AuthenticateMemcachedConn error=%v Retrying (%d)", err, r)
260		}
261
262		u, p, err = cbauth.GetMemcachedServiceAuth(host)
263		return err
264	}
265
266	rh := NewRetryHelper(MAX_AUTH_RETRIES, time.Second*3, 1, fn)
267	err := rh.Run()
268	if err != nil {
269		return err
270	}
271
272	_, err = conn.Auth(u, p)
273	_, err = conn.SelectBucket(ah.Bucket)
274	return err
275}
276
277// GetKVAddrs gather the list of kvnode-address based on the latest vbmap.
278func GetKVAddrs(cluster, pooln, bucketn string) ([]string, error) {
279	b, err := ConnectBucket(cluster, pooln, bucketn)
280	if err != nil {
281		return nil, err
282	}
283	defer b.Close()
284
285	b.Refresh()
286	m, err := b.GetVBmap(nil)
287	if err != nil {
288		return nil, err
289	}
290
291	kvaddrs := make([]string, 0, len(m))
292	for kvaddr := range m {
293		kvaddrs = append(kvaddrs, kvaddr)
294	}
295	return kvaddrs, nil
296}
297
298// IsIPLocal return whether `ip` address is loopback address or
299// compares equal with local-IP-address.
300func IsIPLocal(ip string) bool {
301	netIP := net.ParseIP(ip)
302
303	// if loopback address, return true
304	if netIP.IsLoopback() {
305		return true
306	}
307
308	// compare with the local ip
309	if localIP, err := GetLocalIP(); err == nil {
310		if localIP.Equal(netIP) {
311			return true
312		}
313	}
314	return false
315}
316
317// GetLocalIP return the first external-IP4 configured for the first
318// interface connected to this node.
319func GetLocalIP() (net.IP, error) {
320	interfaces, err := net.Interfaces()
321	if err != nil {
322		return nil, err
323	}
324	for _, iface := range interfaces {
325		if (iface.Flags & net.FlagUp) == 0 {
326			continue // interface down
327		}
328		if (iface.Flags & net.FlagLoopback) != 0 {
329			continue // loopback interface
330		}
331		addrs, err := iface.Addrs()
332		if err != nil {
333			return nil, err
334		}
335		for _, addr := range addrs {
336			var ip net.IP
337			switch v := addr.(type) {
338			case *net.IPNet:
339				ip = v.IP
340			case *net.IPAddr:
341				ip = v.IP
342			}
343			if ip != nil && !ip.IsLoopback() {
344				if ip = ip.To4(); ip != nil {
345					return ip, nil
346				}
347			}
348		}
349	}
350	return nil, errors.New("cannot find local IP address")
351}
352
353// ExitOnStdinClose is exit handler to be used with ns-server.
354func ExitOnStdinClose() {
355	buf := make([]byte, 4)
356	for {
357		_, err := os.Stdin.Read(buf)
358		if err != nil {
359			if err == io.EOF {
360				time.Sleep(1 * time.Second)
361				os.Exit(0)
362			}
363
364			panic(fmt.Sprintf("Stdin: Unexpected error occured %v", err))
365		}
366	}
367}
368
369// GetColocatedHost find the server addr for localhost and return the same.
370func GetColocatedHost(cluster string) (string, error) {
371	// get vbmap from bucket connection.
372	bucket, err := ConnectBucket(cluster, "default", "default")
373	if err != nil {
374		return "", err
375	}
376	defer bucket.Close()
377
378	hostports := bucket.NodeAddresses()
379	serversM := make(map[string]string)
380	servers := make([]string, 0)
381	for _, hostport := range hostports {
382		host, _, err := net.SplitHostPort(hostport)
383		if err != nil {
384			return "", err
385		}
386		serversM[host] = hostport
387		servers = append(servers, host)
388	}
389
390	for _, server := range servers {
391		addrs, err := net.LookupIP(server)
392		if err != nil {
393			return "", err
394		}
395		for _, addr := range addrs {
396			if IsIPLocal(addr.String()) {
397				return serversM[server], nil
398			}
399		}
400	}
401	return "", errors.New("unknown host")
402}
403
404func CrashOnError(err error) {
405	if err != nil {
406		panic(err)
407	}
408}
409
410func ClusterAuthUrl(cluster string) (string, error) {
411
412	if strings.HasPrefix(cluster, "http") {
413		u, err := url.Parse(cluster)
414		if err != nil {
415			return "", err
416		}
417		cluster = u.Host
418	}
419
420	adminUser, adminPasswd, err := cbauth.GetHTTPServiceAuth(cluster)
421	if err != nil {
422		return "", err
423	}
424
425	clusterUrl := url.URL{
426		Scheme: "http",
427		Host:   cluster,
428		User:   url.UserPassword(adminUser, adminPasswd),
429	}
430
431	return clusterUrl.String(), nil
432}
433
434func ClusterUrl(cluster string) string {
435	host := cluster
436	if strings.HasPrefix(cluster, "http") {
437		u, err := url.Parse(cluster)
438		if err != nil {
439			panic(err) // TODO: should we panic ?
440		}
441		host = u.Host
442	}
443	clusterUrl := url.URL{
444		Scheme: "http",
445		Host:   host,
446	}
447
448	return clusterUrl.String()
449}
450
451func MaybeSetEnv(key, value string) string {
452	if s := os.Getenv(key); s != "" {
453		return s
454	}
455	os.Setenv(key, value)
456	return value
457}
458
459func EquivalentIP(
460	raddr string,
461	raddrs []string) (this string, other string, err error) {
462
463	host, port, err := net.SplitHostPort(raddr)
464	if err != nil {
465		return "", "", err
466	}
467
468	if host == "localhost" {
469		host = GetLocalIpAddr(IsIpv6())
470	}
471
472	netIP := net.ParseIP(host)
473
474	for _, raddr1 := range raddrs {
475		host1, port1, err := net.SplitHostPort(raddr1)
476		if err != nil {
477			return "", "", err
478		}
479
480		if host1 == "localhost" {
481			host1 = GetLocalIpAddr(IsIpv6())
482		}
483		netIP1 := net.ParseIP(host1)
484		// check whether ports are same.
485		if port != port1 {
486			continue
487		}
488		// check whether both are local-ip.
489		if IsIPLocal(host) && IsIPLocal(host1) {
490			return net.JoinHostPort(host, port),
491				net.JoinHostPort(host1, port), nil // raddr => raddr1
492		}
493		// check whether they are coming from the same remote.
494		if netIP.Equal(netIP1) {
495			return net.JoinHostPort(host, port),
496				net.JoinHostPort(host1, port1), nil // raddr == raddr1
497		}
498	}
499	return net.JoinHostPort(host, port),
500		net.JoinHostPort(host, port), nil
501}
502
503//---------------------
504// SDK bucket operation
505//---------------------
506
507// ConnectBucket will instantiate a couchbase-bucket instance with cluster.
508// caller's responsibility to close the bucket.
509func ConnectBucket(cluster, pooln, bucketn string) (*couchbase.Bucket, error) {
510	if strings.HasPrefix(cluster, "http") {
511		u, err := url.Parse(cluster)
512		if err != nil {
513			return nil, err
514		}
515		cluster = u.Host
516	}
517
518	ah := &CbAuthHandler{
519		Hostport: cluster,
520		Bucket:   bucketn,
521	}
522
523	couch, err := couchbase.ConnectWithAuth("http://"+cluster, ah)
524	if err != nil {
525		return nil, err
526	}
527	pool, err := couch.GetPool(pooln)
528	if err != nil {
529		return nil, err
530	}
531	bucket, err := pool.GetBucket(bucketn)
532	if err != nil {
533		return nil, err
534	}
535	return bucket, err
536}
537
538// MaxVbuckets return the number of vbuckets in bucket.
539func MaxVbuckets(bucket *couchbase.Bucket) (int, error) {
540	count := 0
541	m, err := bucket.GetVBmap(nil)
542	if err == nil {
543		for _, vbnos := range m {
544			count += len(vbnos)
545		}
546	}
547	return count, err
548}
549
550// BucketTs return bucket timestamp for all vbucket.
551func BucketTs(bucket *couchbase.Bucket, maxvb int) (seqnos, vbuuids []uint64, err error) {
552	seqnos = make([]uint64, maxvb)
553	vbuuids = make([]uint64, maxvb)
554	stats, err := bucket.GetStats("vbucket-details")
555	// for all nodes in cluster
556	for _, nodestat := range stats {
557		// for all vbuckets
558		for i := 0; i < maxvb; i++ {
559			vbno_str := strconv.Itoa(i)
560			vbstatekey := "vb_" + vbno_str
561			vbhseqkey := "vb_" + vbno_str + ":high_seqno"
562			vbuuidkey := "vb_" + vbno_str + ":uuid"
563			vbstate, ok := nodestat[vbstatekey]
564			highseqno_s, hseq_ok := nodestat[vbhseqkey]
565			vbuuid_s, uuid_ok := nodestat[vbuuidkey]
566			if ok && hseq_ok && uuid_ok && vbstate == "active" {
567				if uuid, err := strconv.ParseUint(vbuuid_s, 10, 64); err == nil {
568					vbuuids[i] = uuid
569				}
570				if s, err := strconv.ParseUint(highseqno_s, 10, 64); err == nil {
571					if s > seqnos[i] {
572						seqnos[i] = s
573					}
574				}
575			}
576		}
577	}
578	return seqnos, vbuuids, err
579}
580
581func IsAuthValid(r *http.Request) (cbauth.Creds, bool, error) {
582
583	creds, err := cbauth.AuthWebCreds(r)
584	if err != nil {
585		if strings.Contains(err.Error(), cbauthimpl.ErrNoAuth.Error()) {
586			return nil, false, nil
587		}
588		return nil, false, err
589	}
590
591	return creds, true, nil
592}
593
594func SetNumCPUs(percent int) int {
595	ncpu := percent / 100
596	if ncpu == 0 {
597		ncpu = runtime.NumCPU()
598	}
599	runtime.GOMAXPROCS(ncpu)
600	return ncpu
601}
602
603func IndexStatement(def IndexDefn, numPartitions int, printNodes bool) string {
604	var stmt string
605	primCreate := "CREATE PRIMARY INDEX `%s` ON `%s`"
606	secCreate := "CREATE INDEX `%s` ON `%s`(%s)"
607	where := " WHERE %s"
608	partition := " PARTITION BY hash(%s)"
609
610	if def.IsPrimary {
611		stmt = fmt.Sprintf(primCreate, def.Name, def.Bucket)
612	} else {
613		exprs := ""
614		for i, exp := range def.SecExprs {
615			if exprs != "" {
616				exprs += ","
617			}
618			exprs += exp
619			if def.Desc != nil && def.Desc[i] {
620				exprs += " DESC"
621			}
622		}
623		stmt = fmt.Sprintf(secCreate, def.Name, def.Bucket, exprs)
624
625		if len(def.PartitionKeys) != 0 {
626			exprs := ""
627			for _, exp := range def.PartitionKeys {
628				if exprs != "" {
629					exprs += ","
630				}
631				exprs += exp
632			}
633			stmt += fmt.Sprintf(partition, exprs)
634		}
635
636		if def.WhereExpr != "" {
637			stmt += fmt.Sprintf(where, def.WhereExpr)
638		}
639	}
640
641	withExpr := ""
642	/*
643		if def.Immutable {
644			withExpr += "\"immutable\":true"
645		}
646	*/
647
648	if def.Deferred {
649		if len(withExpr) != 0 {
650			withExpr += ","
651		}
652
653		withExpr += " \"defer_build\":true"
654	}
655
656	if def.RetainDeletedXATTR {
657		if len(withExpr) != 0 {
658			withExpr += ","
659		}
660
661		withExpr += " \"retain_deleted_xattr\":true"
662	}
663
664	if printNodes && len(def.Nodes) != 0 {
665		if len(withExpr) != 0 {
666			withExpr += ","
667		}
668		withExpr += " \"nodes\":[ "
669
670		for i, node := range def.Nodes {
671			withExpr += "\"" + node + "\""
672			if i < len(def.Nodes)-1 {
673				withExpr += ","
674			}
675		}
676
677		withExpr += " ]"
678	}
679
680	if def.NumReplica != 0 {
681		if len(withExpr) != 0 {
682			withExpr += ","
683		}
684
685		withExpr += fmt.Sprintf(" \"num_replica\":%v", def.NumReplica)
686	}
687
688	if IsPartitioned(def.PartitionScheme) {
689		if len(withExpr) != 0 {
690			withExpr += ","
691		}
692
693		withExpr += fmt.Sprintf(" \"num_partition\":%v", numPartitions)
694	}
695
696	if len(withExpr) != 0 {
697		stmt += fmt.Sprintf(" WITH { %s }", withExpr)
698	}
699
700	return stmt
701}
702
703func LogRuntime() string {
704	n := runtime.NumCPU()
705	v := runtime.Version()
706	m := runtime.GOMAXPROCS(-1)
707	fmsg := "%v %v; cpus: %v; GOMAXPROCS: %v; version: %v"
708	return fmt.Sprintf(fmsg, runtime.GOARCH, runtime.GOOS, n, m, v)
709}
710
711func LogOs() string {
712	gid := os.Getgid()
713	uid := os.Getuid()
714	hostname, _ := os.Hostname()
715	return fmt.Sprintf("uid: %v; gid: %v; hostname: %v", uid, gid, hostname)
716}
717
718//
719// This method fetch the bucket UUID.  If this method return an error,
720// then it means that the node is not able to connect in order to fetch
721// bucket UUID.
722//
723func GetBucketUUID(cluster, bucket string) (string, error) {
724
725	url, err := ClusterAuthUrl(cluster)
726	if err != nil {
727		return BUCKET_UUID_NIL, err
728	}
729
730	cinfo, err := NewClusterInfoCache(url, "default")
731	if err != nil {
732		return BUCKET_UUID_NIL, err
733	}
734
735	cinfo.Lock()
736	defer cinfo.Unlock()
737
738	if err := cinfo.Fetch(); err != nil {
739		return BUCKET_UUID_NIL, err
740	}
741
742	return cinfo.GetBucketUUID(bucket), nil
743}
744
745func FileSize(name string) (int64, error) {
746	f, err := os.Open(name)
747	if err != nil {
748		return 0, err
749	}
750	defer f.Close()
751
752	fi, err := f.Stat()
753	if err != nil {
754		return 0, err
755	}
756
757	return fi.Size(), nil
758}
759
760// HashVbuuid return crc64 value of list of 64-bit vbuuids.
761func HashVbuuid(vbuuids []uint64) uint64 {
762	var bytes []byte
763	vbuuids_sl := (*reflect.SliceHeader)(unsafe.Pointer(&vbuuids))
764	bytes_sl := (*reflect.SliceHeader)(unsafe.Pointer(&bytes))
765	bytes_sl.Data = vbuuids_sl.Data
766	bytes_sl.Len = vbuuids_sl.Len * 8
767	bytes_sl.Cap = vbuuids_sl.Cap * 8
768	return crc64.Checksum(bytes, crc64.MakeTable(crc64.ECMA))
769}
770
771func IsValidIndexName(n string) error {
772	valid, _ := regexp.MatchString(IndexNamePattern, n)
773	if !valid {
774		return ErrInvalidIndexName
775	}
776
777	return nil
778}
779
780func ComputeAvg(lastAvg, lastValue, currValue int64) int64 {
781	if lastValue == 0 {
782		return 0
783	}
784
785	diff := currValue - lastValue
786	// Compute avg for first time
787	if lastAvg == 0 {
788		return diff
789	}
790
791	return (diff + lastAvg) / 2
792}
793
794// Write to the admin console
795func Console(clusterAddr string, format string, v ...interface{}) error {
796	msg := fmt.Sprintf(format, v...)
797	values := url.Values{"message": {msg}, "logLevel": {"info"}, "component": {"indexing"}}
798	reader := strings.NewReader(values.Encode())
799
800	if !strings.HasPrefix(clusterAddr, "http://") {
801		clusterAddr = "http://" + clusterAddr
802	}
803	clusterAddr += "/_log"
804
805	req, err := http.NewRequest("POST", clusterAddr, reader)
806	if err != nil {
807		return err
808	}
809	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
810	cbauth.SetRequestAuthVia(req, nil)
811
812	client := http.Client{Timeout: time.Duration(10 * time.Second)}
813	_, err = client.Do(req)
814
815	return err
816}
817
818func CopyFile(dest, source string) (err error) {
819	var sf, df *os.File
820
821	defer func() {
822		if sf != nil {
823			sf.Close()
824		}
825		if df != nil {
826			df.Close()
827		}
828	}()
829
830	if sf, err = os.Open(source); err != nil {
831		return err
832	} else if IsPathExist(dest) {
833		return nil
834	} else if df, err = os.Create(dest); err != nil {
835		return err
836	} else if _, err = io.Copy(df, sf); err != nil {
837		return err
838	}
839
840	var info os.FileInfo
841	if info, err = os.Stat(source); err != nil {
842		return err
843	} else if err = os.Chmod(dest, info.Mode()); err != nil {
844		return err
845	}
846	return
847}
848
849// CopyDir compose destination path based on source and,
850//   - if dest is file, and path is reachable, it is a no-op.
851//   - if dest is file, and path is not reachable, create and copy.
852//   - if dest is dir, and path is reachable, recurse into the dir.
853//   - if dest is dir, and path is not reachable, create and recurse into the dir.
854func CopyDir(dest, source string) error {
855	var created bool
856
857	if fi, err := os.Stat(source); err != nil {
858		return err
859	} else if !fi.IsDir() {
860		return fmt.Errorf("source not a directory")
861	} else if IsPathExist(dest) == false {
862		created = true
863		if err := os.MkdirAll(dest, fi.Mode()); err != nil {
864			return err
865		}
866	}
867
868	var err error
869	defer func() {
870		// if copy failed in the middle and directory was created by us, clean.
871		if err != nil && created {
872			os.RemoveAll(dest)
873		}
874	}()
875
876	var entries []os.FileInfo
877	if entries, err = ioutil.ReadDir(source); err != nil {
878		return err
879	} else {
880		for _, entry := range entries {
881			s := filepath.Join(source, entry.Name())
882			d := filepath.Join(dest, entry.Name())
883			if entry.IsDir() {
884				if err = CopyDir(d, s); err != nil {
885					return err
886				}
887			} else if err = CopyFile(d, s); err != nil {
888				return err
889			}
890		}
891	}
892	return nil
893}
894
895func IsPathExist(path string) bool {
896	if _, err := os.Stat(path); err != nil {
897		return !os.IsNotExist(err)
898	}
899	return true
900}
901
902func DiskUsage(dir string) (int64, error) {
903	var sz int64
904	err := filepath.Walk(dir, func(_ string, fi os.FileInfo, err error) error {
905		if err != nil {
906			return err
907		}
908
909		if !fi.IsDir() {
910			sz += fi.Size()
911		}
912		return nil
913	})
914
915	if err != nil {
916		return 0, err
917	}
918
919	return sz, nil
920}
921
922func GenNextBiggerKey(b []byte, isPrimary bool) []byte {
923	var x big.Int
924	if !isPrimary {
925		// Remove last 1 byte terminator encoding
926		x.SetBytes(b[:len(b)-1])
927	} else {
928		x.SetBytes(b[:len(b)])
929	}
930	x.Add(&x, big.NewInt(1))
931	return x.Bytes()
932}
933
934func IsAllowed(creds cbauth.Creds, permissions []string, w http.ResponseWriter) bool {
935
936	allow := false
937	err := error(nil)
938
939	for _, permission := range permissions {
940		allow, err = creds.IsAllowed(permission)
941		if allow && err == nil {
942			break
943		}
944	}
945
946	if err != nil {
947		w.WriteHeader(http.StatusInternalServerError)
948		w.Write([]byte(err.Error()))
949		return false
950	}
951
952	if !allow {
953		w.WriteHeader(http.StatusUnauthorized)
954		w.Write([]byte(http.StatusText(http.StatusUnauthorized)))
955		return false
956	}
957
958	return true
959}
960
961func IsAllAllowed(creds cbauth.Creds, permissions []string, w http.ResponseWriter) bool {
962
963	allow := true
964	err := error(nil)
965
966	for _, permission := range permissions {
967		allow, err = creds.IsAllowed(permission)
968		if !allow || err != nil {
969			break
970		}
971	}
972
973	if err != nil {
974		w.WriteHeader(http.StatusInternalServerError)
975		w.Write([]byte(err.Error()))
976		return false
977	}
978
979	if !allow {
980		w.WriteHeader(http.StatusUnauthorized)
981		w.Write([]byte(http.StatusText(http.StatusUnauthorized)))
982		return false
983	}
984
985	return true
986}
987
988func ComputePercent(a, b int64) int64 {
989	if a+b > 0 {
990		return a * 100 / (a + b)
991	}
992
993	return 0
994}
995
996func SetIpv6(isIpv6 bool) {
997	_isIpv6 = isIpv6
998}
999
1000func IsIpv6() bool {
1001	return _isIpv6
1002}
1003
1004func validateAuth(w http.ResponseWriter, r *http.Request) bool {
1005	_, valid, err := IsAuthValid(r)
1006	if err != nil {
1007		w.WriteHeader(http.StatusBadRequest)
1008		w.Write([]byte(err.Error() + "\n"))
1009	} else if valid == false {
1010		w.WriteHeader(401)
1011		w.Write([]byte("401 Unauthorized\n"))
1012	}
1013	return valid
1014}
1015
1016func GrHandler(rw http.ResponseWriter, r *http.Request) {
1017
1018	valid := validateAuth(rw, r)
1019	if !valid {
1020		return
1021	}
1022
1023	hndlr := pprof.Handler("goroutine")
1024	hndlr.ServeHTTP(rw, r)
1025}
1026
1027func BlockHandler(rw http.ResponseWriter, r *http.Request) {
1028
1029	valid := validateAuth(rw, r)
1030	if !valid {
1031		return
1032	}
1033
1034	hndlr := pprof.Handler("block")
1035	hndlr.ServeHTTP(rw, r)
1036}
1037
1038func HeapHandler(rw http.ResponseWriter, r *http.Request) {
1039
1040	valid := validateAuth(rw, r)
1041	if !valid {
1042		return
1043	}
1044
1045	hndlr := pprof.Handler("heap")
1046	hndlr.ServeHTTP(rw, r)
1047}
1048
1049func TCHandler(rw http.ResponseWriter, r *http.Request) {
1050
1051	valid := validateAuth(rw, r)
1052	if !valid {
1053		return
1054	}
1055
1056	hndlr := pprof.Handler("threadcreate")
1057	hndlr.ServeHTTP(rw, r)
1058}
1059
1060func PProfHandler(rw http.ResponseWriter, r *http.Request) {
1061
1062	valid := validateAuth(rw, r)
1063	if !valid {
1064		return
1065	}
1066
1067	pprof.Index(rw, r)
1068}
1069
1070func ProfileHandler(rw http.ResponseWriter, r *http.Request) {
1071
1072	valid := validateAuth(rw, r)
1073	if !valid {
1074		return
1075	}
1076
1077	pprof.Profile(rw, r)
1078}
1079
1080func CmdlineHandler(rw http.ResponseWriter, r *http.Request) {
1081
1082	valid := validateAuth(rw, r)
1083	if !valid {
1084		return
1085	}
1086
1087	pprof.Cmdline(rw, r)
1088}
1089
1090func SymbolHandler(rw http.ResponseWriter, r *http.Request) {
1091
1092	valid := validateAuth(rw, r)
1093	if !valid {
1094		return
1095	}
1096
1097	pprof.Symbol(rw, r)
1098}
1099
1100func TraceHandler(rw http.ResponseWriter, r *http.Request) {
1101
1102	valid := validateAuth(rw, r)
1103	if !valid {
1104		return
1105	}
1106
1107	pprof.Trace(rw, r)
1108}
1109
1110func ExpvarHandler(rw http.ResponseWriter, r *http.Request) {
1111
1112	valid := validateAuth(rw, r)
1113	if !valid {
1114		return
1115	}
1116
1117	rw.Header().Set("Content-Type", "application/json; charset=utf-8")
1118	fmt.Fprintf(rw, "{\n")
1119	first := true
1120	expvar.Do(func(kv expvar.KeyValue) {
1121		if !first {
1122			fmt.Fprintf(rw, ",\n")
1123		}
1124		first = false
1125		fmt.Fprintf(rw, "%q: %s", kv.Key, kv.Value)
1126	})
1127	fmt.Fprintf(rw, "\n}\n")
1128}
1129