1//  Copyright (c) 2019 Couchbase, Inc.
2//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
3//  except in compliance with the License. You may obtain a copy of the License at
4//    http://www.apache.org/licenses/LICENSE-2.0
5//  Unless required by applicable law or agreed to in writing, software distributed under the
6//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
7//  either express or implied. See the License for the specific language governing permissions
8//  and limitations under the License.
9
10package flex
11
12import (
13	"encoding/json"
14	"fmt"
15	"sort"
16	"strings"
17
18	"github.com/blevesearch/bleve/mapping"
19	"github.com/couchbase/query/expression"
20	"github.com/couchbase/query/value"
21)
22
23var DefaultTypeFieldPath = []string{"type"}
24
25// BleveToCondFlexIndexes translates a bleve index into CondFlexIndexes.
26// NOTE: checking for DocConfig.Mode should be done beforehand.
27func BleveToCondFlexIndexes(im *mapping.IndexMappingImpl,
28	typeFieldPath []string) (rv CondFlexIndexes, err error) {
29	// Map of FieldTrack => fieldType => count.
30	fieldTrackTypeCounts := map[FieldTrack]map[string]int{}
31	// Map of FieldTrack => array of type mappings within which they occur
32	fieldTrackTypes := map[FieldTrack]map[string]struct{}{}
33	for t, dm := range im.TypeMapping {
34		countFieldTrackTypes(nil, t, dm, im.DefaultAnalyzer,
35			fieldTrackTypeCounts, fieldTrackTypes)
36	}
37	countFieldTrackTypes(nil, "", im.DefaultMapping, im.DefaultAnalyzer,
38		fieldTrackTypeCounts, fieldTrackTypes)
39
40	types := make([]string, 0, len(im.TypeMapping))
41	for t := range im.TypeMapping {
42		types = append(types, t)
43	}
44
45	sort.Strings(types) // For output stability.
46
47	if len(typeFieldPath) == 0 {
48		typeFieldPath = DefaultTypeFieldPath
49	}
50
51	fi := &FlexIndex{
52		IndexedFields: FieldInfos{
53			&FieldInfo{FieldPath: typeFieldPath, FieldType: "text"},
54		},
55		SupportedExprs: []SupportedExpr{},
56	}
57
58	if len(fieldTrackTypes) > 0 {
59		fi.FieldTrackTypes = fieldTrackTypes
60	}
61
62	var values value.Values
63	for _, t := range types {
64		typeEqEffect := "FlexBuild:n" // Strips `type = "BEER"` from expressions.
65		if !im.TypeMapping[t].Enabled {
66			typeEqEffect = "not-sargable"
67		}
68
69		// Strips `type = "BEER"` from expressions.
70		fi.SupportedExprs = append(fi.SupportedExprs, &SupportedExprCmpFieldConstant{
71			Cmp:       "eq",
72			FieldPath: typeFieldPath,
73			ValueType: "text",
74			ValueMust: value.NewValue(t),
75			Effect:    typeEqEffect,
76		})
77
78		// To treat `type > "BEER"` as not-sargable.
79		fi.SupportedExprs = append(fi.SupportedExprs, &SupportedExprCmpFieldConstant{
80			Cmp:       "lt gt le ge",
81			FieldPath: typeFieldPath,
82			ValueType: "", // Treated as not-sargable.
83			Effect:    "not-sargable",
84		})
85
86		fi, err = BleveToFlexIndex(fi, nil, im.TypeMapping[t], im.DefaultAnalyzer,
87			fieldTrackTypeCounts)
88		if err != nil {
89			return nil, err
90		}
91
92		values = append(values, value.NewValue(t))
93	}
94
95	if len(fi.SupportedExprs) > 0 {
96		// Add CondFlexIndex over all the types, iff at least one type mapping
97		// is enabled
98		rv = append(rv, &CondFlexIndex{
99			Cond:      MakeCondFuncEqVals(typeFieldPath, values),
100			FlexIndex: fi,
101		})
102	}
103
104	if im.DefaultMapping != nil {
105		fi, err := BleveToFlexIndex(&FlexIndex{
106			IndexedFields: FieldInfos{},
107		}, nil, im.DefaultMapping, im.DefaultAnalyzer, fieldTrackTypeCounts)
108		if err != nil {
109			return nil, err
110		}
111
112		if len(fieldTrackTypes) > 0 {
113			fi.FieldTrackTypes = fieldTrackTypes
114		}
115
116		rv = append(rv, &CondFlexIndex{
117			Cond:      MakeCondFuncNeqVals(typeFieldPath, types),
118			FlexIndex: fi,
119		})
120	}
121
122	return rv, nil
123}
124
125// ------------------------------------------------------------------------
126
127// Populates into fieldTrackTypeCounts the counts of field types, and into
128// fieldTrackTypes all the type mappings within which the field tracks occur.
129func countFieldTrackTypes(path []string, typeMappingName string,
130	dm *mapping.DocumentMapping, defaultAnalyzer string,
131	fieldTrackTypeCounts map[FieldTrack]map[string]int,
132	fieldTrackTypes map[FieldTrack]map[string]struct{}) {
133	if dm == nil || !dm.Enabled {
134		return
135	}
136
137	if dm.DefaultAnalyzer != "" {
138		defaultAnalyzer = dm.DefaultAnalyzer
139	}
140
141	for _, f := range dm.Fields {
142		// For now, only consider fields whose propName == f.Name.
143		if f.Index && len(path) > 0 && path[len(path)-1] == f.Name {
144			if _, ok := BleveSupportedTypes[f.Type]; !ok {
145				continue
146			}
147
148			analyzer := defaultAnalyzer
149			if f.Analyzer != "" {
150				analyzer = f.Analyzer
151			}
152
153			if f.Type == "text" && analyzer != "keyword" {
154				continue
155			}
156
157			fieldTrack := FieldTrack(strings.Join(path, "."))
158
159			m := fieldTrackTypeCounts[fieldTrack]
160			if m == nil {
161				m = map[string]int{}
162				fieldTrackTypeCounts[fieldTrack] = m
163			}
164			m[f.Type] = m[f.Type] + 1
165
166			if len(typeMappingName) > 0 {
167				if _, exists := fieldTrackTypes[fieldTrack]; !exists {
168					fieldTrackTypes[fieldTrack] = map[string]struct{}{}
169				}
170				fieldTrackTypes[fieldTrack][typeMappingName] = struct{}{}
171			}
172		}
173	}
174
175	for propName, propDM := range dm.Properties {
176		countFieldTrackTypes(append(path, propName), typeMappingName, propDM,
177			defaultAnalyzer, fieldTrackTypeCounts, fieldTrackTypes)
178	}
179}
180
181// ------------------------------------------------------------------------
182
183// This map contains types that Bleve supports for N1QL queries.
184var BleveSupportedTypes = map[string]bool{
185	"text":     true,
186	"number":   true,
187	"boolean":  true,
188	"datetime": true,
189}
190
191// This map translates Bleve's supported types to types as identified
192// by N1QL.
193var BleveTypeConv = map[string]string{
194	"text":     "string",
195	"number":   "number",
196	"boolean":  "boolean",
197	"datetime": "string",
198	"string":   "string",
199}
200
201// ------------------------------------------------------------------------
202
203// Recursively initializes a FlexIndex from a given bleve document
204// mapping.  Note: the backing array for path is mutated as the
205// recursion proceeds.
206func BleveToFlexIndex(fi *FlexIndex, path []string, dm *mapping.DocumentMapping,
207	defaultAnalyzer string, fieldTrackTypeCounts map[FieldTrack]map[string]int) (
208	rv *FlexIndex, err error) {
209	if dm == nil || !dm.Enabled {
210		return fi, nil
211	}
212
213	if dm.DefaultAnalyzer != "" {
214		defaultAnalyzer = dm.DefaultAnalyzer
215	}
216
217	for _, f := range dm.Fields {
218		if !f.Index || len(path) <= 0 || path[len(path)-1] != f.Name {
219			continue
220		}
221
222		if _, ok := BleveSupportedTypes[f.Type]; !ok {
223			continue
224		}
225
226		analyzer := defaultAnalyzer
227		if f.Analyzer != "" {
228			analyzer = f.Analyzer
229		}
230
231		// For now, only keyword text fields are supported.
232		if f.Type == "text" && analyzer != "keyword" {
233			continue
234		}
235
236		// Fields that are indexed using different types are not supported.
237		if len(fieldTrackTypeCounts[FieldTrack(strings.Join(path, "."))]) != 1 {
238			continue
239		}
240
241		fieldPath := append([]string(nil), path...) // Copy.
242
243		fi.IndexedFields = append(fi.IndexedFields, &FieldInfo{
244			FieldPath: fieldPath,
245			FieldType: f.Type,
246		})
247
248		fi.SupportedExprs = append(fi.SupportedExprs, &SupportedExprCmpFieldConstant{
249			Cmp:       "eq",
250			FieldPath: fieldPath,
251			ValueType: f.Type,
252		})
253
254		fi.SupportedExprs = append(fi.SupportedExprs, &SupportedExprCmpFieldConstant{
255			Cmp:            "lt gt le ge",
256			FieldPath:      fieldPath,
257			ValueType:      f.Type,
258			FieldTypeCheck: true,
259		})
260
261		// TODO: Currently supports only keyword fields.
262		// TODO: Need to support geopoint field types?
263		// TODO: Need to support non-keyword analyzers?
264		// TODO: f.Store IncludeTermVectors, IncludeInAll, DateFormat, DocValues
265	}
266
267	ns := make([]string, 0, len(dm.Properties))
268	for n := range dm.Properties {
269		if n != "" {
270			ns = append(ns, n)
271		}
272	}
273
274	sort.Strings(ns)
275
276	for _, n := range ns {
277		fi, err = BleveToFlexIndex(fi, append(path, n), dm.Properties[n],
278			defaultAnalyzer, fieldTrackTypeCounts)
279		if err != nil {
280			return nil, err
281		}
282	}
283
284	// Support dynamic indexing only when defaultAnalyzer is set to "keyword"
285	if dm.Dynamic {
286		if defaultAnalyzer == "keyword" {
287			dynamicPath := append([]string(nil), path...) // Copy.
288
289			// Register the dynamic path prefix into the indexed
290			// fields so complex expressions will be not-sargable.
291			fi.IndexedFields = append(fi.IndexedFields, &FieldInfo{
292				FieldPath: dynamicPath,
293				FieldType: "text",
294			})
295			fi.IndexedFields = append(fi.IndexedFields, &FieldInfo{
296				FieldPath: dynamicPath,
297				FieldType: "number",
298			})
299			fi.IndexedFields = append(fi.IndexedFields, &FieldInfo{
300				FieldPath: dynamicPath,
301				FieldType: "boolean",
302			})
303			fi.IndexedFields = append(fi.IndexedFields, &FieldInfo{
304				FieldPath: dynamicPath,
305				FieldType: "datetime",
306			})
307
308			fi.SupportedExprs = append(fi.SupportedExprs, &SupportedExprCmpFieldConstant{
309				Cmp:              "eq",
310				FieldPath:        dynamicPath,
311				ValueType:        "text",
312				FieldPathPartial: true,
313			})
314
315			fi.SupportedExprs = append(fi.SupportedExprs, &SupportedExprCmpFieldConstant{
316				Cmp:              "eq",
317				FieldPath:        dynamicPath,
318				ValueType:        "number",
319				FieldPathPartial: true,
320			})
321
322			fi.SupportedExprs = append(fi.SupportedExprs, &SupportedExprCmpFieldConstant{
323				Cmp:              "eq",
324				FieldPath:        dynamicPath,
325				ValueType:        "boolean",
326				FieldPathPartial: true,
327			})
328
329			fi.SupportedExprs = append(fi.SupportedExprs, &SupportedExprCmpFieldConstant{
330				Cmp:              "eq",
331				FieldPath:        dynamicPath,
332				ValueType:        "datetime",
333				FieldPathPartial: true,
334			})
335
336			fi.SupportedExprs = append(fi.SupportedExprs, &SupportedExprCmpFieldConstant{
337				Cmp:              "lt gt le ge",
338				FieldPath:        dynamicPath,
339				ValueType:        "text",
340				FieldTypeCheck:   true,
341				FieldPathPartial: true,
342			})
343
344			fi.SupportedExprs = append(fi.SupportedExprs, &SupportedExprCmpFieldConstant{
345				Cmp:              "lt gt le ge",
346				FieldPath:        dynamicPath,
347				ValueType:        "number",
348				FieldTypeCheck:   true,
349				FieldPathPartial: true,
350			})
351
352			fi.SupportedExprs = append(fi.SupportedExprs, &SupportedExprCmpFieldConstant{
353				Cmp:              "lt gt le ge",
354				FieldPath:        dynamicPath,
355				ValueType:        "datetime",
356				FieldTypeCheck:   true,
357				FieldPathPartial: true,
358			})
359
360			fi.Dynamic = true
361		}
362	}
363
364	return fi, nil
365}
366
367// --------------------------------------
368
369// FlexBuildToBleveQuery translates a flex build tree into a bleve
370// query tree in map[string]interface{} representation.
371func FlexBuildToBleveQuery(fb *FlexBuild, prevSibling map[string]interface{}) (
372	q map[string]interface{}, err error) {
373	if fb == nil {
374		return nil, nil
375	}
376
377	isConjunct := fb.Kind == "conjunct"
378	if isConjunct || fb.Kind == "disjunct" {
379		qs := make([]interface{}, 0, len(fb.Children))
380
381		var prev map[string]interface{}
382
383		for _, c := range fb.Children {
384			q, err = FlexBuildToBleveQuery(c, prev)
385			if err != nil {
386				return nil, err
387			}
388
389			if q != nil {
390				qs = append(qs, q)
391
392				if isConjunct {
393					prev = q // Prev sibling optimization only when conjunct.
394				}
395			}
396		}
397
398		if len(qs) <= 0 {
399			return nil, nil // Optimize case of con/disjuncts empty.
400		}
401
402		if m, ok := qs[0].(map[string]interface{}); ok && len(qs) == 1 {
403			return m, nil // Optimize case of con/disjuncts of 1.
404		}
405
406		if isConjunct {
407			return map[string]interface{}{"conjuncts": qs}, nil
408		}
409
410		return map[string]interface{}{"disjuncts": qs}, nil
411	}
412
413	if fb.Kind == "cmpFieldConstant" {
414		// Ex: fb.Data: {"eq", "city", "text", `"nyc"`}.
415		if args, ok := fb.Data.([]string); ok && len(args) == 4 {
416			if args[2] == "text" {
417				var v string
418				if err = json.Unmarshal([]byte(args[3]), &v); err != nil {
419					return nil, err
420				}
421
422				switch args[0] {
423				case "eq":
424					return map[string]interface{}{"term": v, "field": args[1]}, nil
425
426				case "lt":
427					return MaxTermRangeQuery(args[1], v, false, prevSibling)
428				case "le":
429					return MaxTermRangeQuery(args[1], v, true, prevSibling)
430				case "gt":
431					return MinTermRangeQuery(args[1], v, false, prevSibling)
432				case "ge":
433					return MinTermRangeQuery(args[1], v, true, prevSibling)
434				default:
435					return nil, fmt.Errorf("incorrect expression: %v", args)
436				}
437			}
438
439			if args[2] == "number" {
440				// Negative numbers will be enclosed within parantheses, so
441				// drop any parantheses from the string.
442				// For eg. (-10) -> -10
443				numStr := strings.Replace(strings.Replace(args[3], "(", "", 1), ")", "", 1)
444				var v float64
445				if err := json.Unmarshal([]byte(numStr), &v); err != nil {
446					return nil, err
447				}
448
449				switch args[0] {
450				case "eq":
451					return map[string]interface{}{
452						"min": v, "max": v,
453						"inclusive_min": true, "inclusive_max": true,
454						"field": args[1],
455					}, nil
456
457				case "lt":
458					return MaxNumericRangeQuery(args[1], v, false, prevSibling)
459				case "le":
460					return MaxNumericRangeQuery(args[1], v, true, prevSibling)
461				case "gt":
462					return MinNumericRangeQuery(args[1], v, false, prevSibling)
463				case "ge":
464					return MinNumericRangeQuery(args[1], v, true, prevSibling)
465				default:
466					return nil, fmt.Errorf("incorrect expression: %v", args)
467				}
468			}
469
470			if args[2] == "boolean" {
471				var v bool
472				if err := json.Unmarshal([]byte(args[3]), &v); err != nil {
473					return nil, err
474				}
475
476				if args[0] != "eq" {
477					return nil, fmt.Errorf("incorrect expression: %v", args)
478				}
479
480				return map[string]interface{}{"bool": v, "field": args[1]}, nil
481			}
482
483			if args[2] == "datetime" {
484				var v string
485				if err = json.Unmarshal([]byte(args[3]), &v); err != nil {
486					return nil, err
487				}
488
489				// datetime needs to comply with ISO-8601 standard
490				if _, _, err := expression.StrToTimeFormat(v); err != nil {
491					return nil, err
492				}
493
494				switch args[0] {
495				case "eq":
496					return map[string]interface{}{
497						"start": v, "end": v,
498						"inclusive_start": true, "inclusive_end": true,
499						"field": args[1],
500					}, nil
501
502				case "lt":
503					return MaxDatetimeRangeQuery(args[1], v, false, prevSibling)
504				case "le":
505					return MaxDatetimeRangeQuery(args[1], v, true, prevSibling)
506				case "gt":
507					return MinDatetimeRangeQuery(args[1], v, false, prevSibling)
508				case "ge":
509					return MinDatetimeRangeQuery(args[1], v, true, prevSibling)
510				default:
511					return nil, fmt.Errorf("incorrect expression: %v", args)
512				}
513			}
514		}
515	}
516
517	return nil, fmt.Errorf("FlexBuildToBleveQuery: could not convert: %+v", fb)
518}
519
520func MinTermRangeQuery(f string, v string, inclusive bool,
521	prev map[string]interface{}) (map[string]interface{}, error) {
522	if prev != nil && prev["field"] == f {
523		_, prevMinOk := prev["min"].(string)
524		prevMax, prevMaxOk := prev["max"].(string)
525		if !prevMinOk && prevMaxOk && v <= prevMax {
526			prev["min"] = v
527			prev["inclusive_min"] = inclusive
528			return nil, nil
529		}
530	}
531
532	return map[string]interface{}{
533		"min": v, "inclusive_min": inclusive, "field": f,
534	}, nil
535}
536
537func MaxTermRangeQuery(f string, v string, inclusive bool,
538	prev map[string]interface{}) (map[string]interface{}, error) {
539	if prev != nil && prev["field"] == f {
540		_, prevMaxOk := prev["max"].(string)
541		prevMin, prevMinOk := prev["min"].(string)
542		if !prevMaxOk && prevMinOk && prevMin <= v {
543			prev["max"] = v
544			prev["inclusive_max"] = inclusive
545			return nil, nil
546		}
547	}
548
549	return map[string]interface{}{
550		"max": v, "inclusive_max": inclusive, "field": f,
551	}, nil
552}
553
554func MinNumericRangeQuery(f string, v float64, inclusive bool,
555	prev map[string]interface{}) (map[string]interface{}, error) {
556	if prev != nil && prev["field"] == f {
557		_, prevMinOk := prev["min"].(float64)
558		prevMax, prevMaxOk := prev["max"].(float64)
559		if !prevMinOk && prevMaxOk && v <= prevMax {
560			prev["min"] = v
561			prev["inclusive_min"] = inclusive
562			return nil, nil
563		}
564	}
565
566	return map[string]interface{}{
567		"min": v, "inclusive_min": inclusive, "field": f,
568	}, nil
569}
570
571func MaxNumericRangeQuery(f string, v float64, inclusive bool,
572	prev map[string]interface{}) (map[string]interface{}, error) {
573	if prev != nil && prev["field"] == f {
574		_, prevMaxOk := prev["max"].(float64)
575		prevMin, prevMinOk := prev["min"].(float64)
576		if !prevMaxOk && prevMinOk && prevMin <= v {
577			prev["max"] = v
578			prev["inclusive_max"] = inclusive
579			return nil, nil
580		}
581	}
582
583	return map[string]interface{}{
584		"max": v, "inclusive_max": inclusive, "field": f,
585	}, nil
586}
587
588func MinDatetimeRangeQuery(f string, v string, inclusive bool,
589	prev map[string]interface{}) (map[string]interface{}, error) {
590	if prev != nil && prev["field"] == f {
591		vDT, _, err := expression.StrToTimeFormat(v)
592		if err != nil {
593			return nil, err
594		}
595		_, prevStartOk := prev["start"].(string)
596		prevEnd, prevEndOk := prev["end"].(string)
597		if !prevStartOk && prevEndOk {
598			prevEndDT, _, err := expression.StrToTimeFormat(prevEnd)
599			if err == nil && (vDT.Before(prevEndDT) || vDT.Equal(prevEndDT)) {
600				prev["start"] = v
601				prev["inclusive_start"] = inclusive
602				return nil, nil
603			}
604		}
605	}
606
607	return map[string]interface{}{
608		"start": v, "inclusive_start": inclusive, "field": f,
609	}, nil
610}
611
612func MaxDatetimeRangeQuery(f string, v string, inclusive bool,
613	prev map[string]interface{}) (map[string]interface{}, error) {
614	if prev != nil && prev["field"] == f {
615		vDT, _, err := expression.StrToTimeFormat(v)
616		if err != nil {
617			return nil, err
618		}
619		_, prevEndOk := prev["end"].(string)
620		prevStart, prevStartOk := prev["start"].(string)
621		if !prevEndOk && prevStartOk {
622			prevStartDT, _, err := expression.StrToTimeFormat(prevStart)
623			if err == nil && (vDT.After(prevStartDT) || vDT.Equal(prevStartDT)) {
624				prev["end"] = v
625				prev["inclusive_end"] = inclusive
626				return nil, nil
627			}
628		}
629	}
630
631	return map[string]interface{}{
632		"end": v, "inclusive_end": inclusive, "field": f,
633	}, nil
634}
635