1//  Copyright (c) 2013 Couchbase, Inc.
2
3// Package collatejson supplies Encoding and Decoding function to transform
4// JSON text into binary representation without loosing information. That is,
5//
6// * binary representation should preserve the sort order such that, sorting
7//   binary encoded json documents much match sorting by functions that parse
8//   and compare JSON documents.
9// * it must be possible to get back the original document, in semantically
10//   correct form, from its binary representation.
11//
12// Notes:
13//
14// * items in a property object are sorted by its property name before they
15//   are compared with property's value.
16package collatejson
17
18import "bytes"
19import json "github.com/couchbase/indexing/secondary/common/json"
20import "errors"
21import "strings"
22import "sort"
23import "fmt"
24import "strconv"
25import "sync"
26import n1ql "github.com/couchbase/query/value"
27
28var bufPool *sync.Pool
29
30const bufSize = 1024
31
32func init() {
33	bufPool = &sync.Pool{
34		New: func() interface{} {
35			b := make([]byte, bufSize, bufSize)
36			return &b
37		},
38	}
39}
40
41var _ = fmt.Sprintf("dummy print")
42
43// ErrorNumberType means configured number type is not supported by codec.
44var ErrorNumberType = errors.New("collatejson.numberType")
45
46// ErrorOutputLen means output buffer has insufficient length.
47var ErrorOutputLen = errors.New("collatejson.outputLen")
48
49// ErrInvalidKeyTypeInObject means key of the object is not string
50var ErrInvalidKeyTypeInObject = errors.New("collatejson.invalidKeyTypeInObject")
51
52// Length is an internal type used for prefixing length
53// of arrays and properties.
54type Length int64
55
56// Missing denotes a special type for an item that evaluates
57// to _nothing_.
58type Missing string
59
60// MissingLiteral is special string to denote missing item.
61// IMPORTANT: we are assuming that MissingLiteral will not
62// occur in the keyspace.
63const MissingLiteral = Missing("~[]{}falsenilNA~")
64
65// MinBufferSize for target buffer to encode or decode.
66const MinBufferSize = 16
67
68// While encoding JSON data-element, both basic and composite, encoded string
69// is prefixed with a type-byte. `Terminator` terminates encoded datum.
70const (
71	Terminator byte = iota
72	TypeMissing
73	TypeNull
74	TypeFalse
75	TypeTrue
76	TypeNumber
77	TypeString
78	TypeLength
79	TypeArray
80	TypeObj
81)
82
83// Codec structure
84type Codec struct {
85	arrayLenPrefix    bool        // if true, first sort arrays based on its length
86	propertyLenPrefix bool        // if true, first sort properties based on length
87	doMissing         bool        // if true, handle missing values (for N1QL)
88	numberType        interface{} // "float64" | "int64" | "decimal"
89	//-- unicode
90	//backwards        bool
91	//hiraganaQ        bool
92	//caseLevel        bool
93	//numeric          bool
94	//nfkd              bool
95	//utf8              bool
96	//strength          colltab.Level
97	//alternate         collate.AlternateHandling
98	//language          language.Tag
99}
100
101// NewCodec creates a new codec object and returns a reference to it.
102func NewCodec(propSize int) *Codec {
103	return &Codec{
104		arrayLenPrefix:    false,
105		propertyLenPrefix: true,
106		doMissing:         true,
107		numberType:        float64(0.0),
108	}
109}
110
111// SortbyArrayLen sorts array by length before sorting by array
112// elements. Use `false` to sort only by array elements.
113// Default is `true`.
114func (codec *Codec) SortbyArrayLen(what bool) {
115	codec.arrayLenPrefix = what
116}
117
118// SortbyPropertyLen sorts property by length before sorting by
119// property items. Use `false` to sort only by proprety items.
120// Default is `true`.
121func (codec *Codec) SortbyPropertyLen(what bool) {
122	codec.propertyLenPrefix = what
123}
124
125// UseMissing will interpret special string MissingLiteral and
126// encode them as TypeMissing.
127// Default is `true`.
128func (codec *Codec) UseMissing(what bool) {
129	codec.doMissing = what
130}
131
132// NumberType chooses type of encoding / decoding for JSON
133// numbers. Can be "float64", "int64", "decimal".
134// Default is "float64"
135func (codec *Codec) NumberType(what string) {
136	switch what {
137	case "float64":
138		codec.numberType = float64(0.0)
139	case "int64":
140		codec.numberType = int64(0)
141	case "decimal":
142		codec.numberType = "0"
143	}
144}
145
146// Encode json documents to order preserving binary representation.
147// `code` is the output buffer for encoding and expected to have
148// enough capacity, atleast 3x of input `text` and > MinBufferSize.
149func (codec *Codec) Encode(text, code []byte) ([]byte, error) {
150	code = code[:0]
151	if cap(code) < (3*len(text)) || cap(code) < MinBufferSize {
152		return nil, ErrorOutputLen
153	} else if len(text) == 0 {
154		return code, nil
155	}
156	var m interface{}
157	if err := json.Unmarshal(text, &m); err != nil {
158		return nil, err
159	}
160	return codec.json2code(m, code)
161}
162
163// Decode a slice of byte into json string and return them as
164// slice of byte. `text` is the output buffer for decoding and
165// expected to have enough capacity, atleast 3x of input `code`
166// and > MinBufferSize.
167func (codec *Codec) Decode(code, text []byte) ([]byte, error) {
168	text = text[:0]
169	if cap(text) < len(code) || cap(text) < MinBufferSize {
170		return nil, ErrorOutputLen
171	}
172	text, _, err := codec.code2json(code, text)
173	return text, err
174}
175
176// local function that encodes basic json types to binary representation.
177// composite types recursively call this function.
178func (codec *Codec) json2code(val interface{}, code []byte) ([]byte, error) {
179	if val == nil {
180		code = append(code, TypeNull, Terminator)
181	}
182
183	var cs []byte
184	var err error
185
186	switch value := val.(type) {
187	case bool:
188		if value {
189			code = append(code, TypeTrue, Terminator)
190		} else {
191			code = append(code, TypeFalse, Terminator)
192		}
193
194	case float64:
195		code = append(code, TypeNumber)
196		cs, err = codec.normalizeFloat(value, code[1:])
197		if err == nil {
198			code = code[:len(code)+len(cs)]
199			code = append(code, Terminator)
200		}
201
202	case int64:
203		code = append(code, TypeNumber)
204		var intStr string
205		var number Integer
206		intStr, err = number.ConvertToScientificNotation(value)
207		cs = EncodeFloat([]byte(intStr), code[1:])
208		if err == nil {
209			code = code[:len(code)+len(cs)]
210			code = append(code, Terminator)
211		}
212
213	case int:
214		code = append(code, TypeNumber)
215		cs = EncodeInt([]byte(strconv.Itoa(value)), code[1:])
216		code = code[:len(code)+len(cs)]
217		code = append(code, Terminator)
218
219	case Length:
220		code = append(code, TypeLength)
221		cs = EncodeInt([]byte(strconv.Itoa(int(value))), code[1:])
222		code = code[:len(code)+len(cs)]
223		code = append(code, Terminator)
224
225	case string:
226		if codec.doMissing && MissingLiteral.Equal(value) {
227			code = append(code, TypeMissing)
228			code = append(code, Terminator)
229		} else {
230			code = append(code, TypeString)
231			cs = suffixEncodeString([]byte(value), code[1:])
232			code = code[:len(code)+len(cs)]
233			code = append(code, Terminator)
234		}
235
236	case []interface{}:
237		code = append(code, TypeArray)
238		if codec.arrayLenPrefix {
239			arrlen := Length(len(value))
240			if cs, err = codec.json2code(arrlen, code[1:]); err == nil {
241				code = code[:len(code)+len(cs)]
242			}
243		}
244		if err == nil {
245			for _, val := range value {
246				l := len(code)
247				cs, err = codec.json2code(val, code[l:])
248				if err == nil {
249					code = code[:l+len(cs)]
250					continue
251				}
252				break
253			}
254			code = append(code, Terminator)
255		}
256
257	case map[string]interface{}:
258		code = append(code, TypeObj)
259		if codec.propertyLenPrefix {
260			proplen := Length(len(value))
261			if cs, err = codec.json2code(proplen, code[1:]); err == nil {
262				code = code[:len(code)+len(cs)]
263			}
264		}
265
266		if err == nil {
267			keys := codec.sortProps(value)
268			for _, key := range keys {
269				l := len(code)
270				// encode key
271				if cs, err = codec.json2code(key, code[l:]); err != nil {
272					break
273				}
274				code = code[:l+len(cs)]
275				l = len(code)
276				// encode value
277				if cs, err = codec.json2code(value[key], code[l:]); err != nil {
278					break
279				}
280				code = code[:l+len(cs)]
281			}
282			code = append(code, Terminator)
283		}
284	}
285	return code, err
286}
287
288var null = []byte("null")
289var boolTrue = []byte("true")
290var boolFalse = []byte("false")
291
292func (codec *Codec) code2json(code, text []byte) ([]byte, []byte, error) {
293	if len(code) == 0 {
294		return text, code, nil
295	}
296
297	var ts, remaining, datum []byte
298	var err error
299
300	switch code[0] {
301	case Terminator:
302		remaining = code
303
304	case TypeMissing:
305		datum, remaining = getDatum(code)
306		text = append(text, '"')
307		text = append(text, MissingLiteral...)
308		text = append(text, '"')
309
310	case TypeNull:
311		datum, remaining = getDatum(code)
312		text = append(text, null...)
313
314	case TypeTrue:
315		datum, remaining = getDatum(code)
316		text = append(text, boolTrue...)
317
318	case TypeFalse:
319		datum, remaining = getDatum(code)
320		text = append(text, boolFalse...)
321
322	case TypeLength:
323		var scratch [128]byte
324		datum, remaining = getDatum(code)
325		_, ts = DecodeInt(datum[1:], scratch[:0])
326		text = append(text, ts...)
327
328	case TypeNumber:
329		var scratch [128]byte
330		datum, remaining = getDatum(code)
331		ts = DecodeFloat(datum[1:], scratch[:0])
332		ts, err = codec.denormalizeFloat(ts)
333		ts = bytes.TrimLeft(ts, "+")
334		var number Integer
335		ts, _ = number.TryConvertFromScientificNotation(ts)
336		text = append(text, ts...)
337
338	case TypeString:
339		var strb []byte
340		tmp := bufPool.Get().(*[]byte)
341		strb, remaining, err = suffixDecodeString(code[1:], (*tmp)[:0])
342		if err == nil {
343			text, err = encodeString(strb, text)
344			bufPool.Put(tmp)
345		}
346
347	case TypeArray:
348		var l int
349		var scratch [128]byte
350		text = append(text, '[')
351		if codec.arrayLenPrefix {
352			datum, code = getDatum(code[1:])
353			_, ts := DecodeInt(datum[1:], scratch[:0])
354			l, err = strconv.Atoi(string(ts))
355			if err == nil {
356				for ; l > 0; l-- {
357					text, code, err = codec.code2json(code, text)
358					if err != nil {
359						break
360					}
361					if l > 1 {
362						text = append(text, ',')
363					}
364				}
365			}
366		} else {
367			comma := false
368			code = code[1:]
369			for code[0] != Terminator {
370				if comma {
371					text = append(text, ',')
372				}
373				text, code, err = codec.code2json(code, text)
374				if err != nil {
375					break
376				}
377				comma = true
378			}
379		}
380		remaining = code[1:] // remove Terminator
381		text = append(text, ']')
382
383	case TypeObj:
384		var scratch [128]byte
385		var l int
386		text = append(text, '{')
387		if codec.propertyLenPrefix {
388			datum, code = getDatum(code[1:])
389			_, ts := DecodeInt(datum[1:], scratch[:0])
390			l, err = strconv.Atoi(string(ts))
391			if err == nil {
392				for ; l > 0; l-- {
393					// decode key
394					text, code, err = codec.code2json(code, text)
395					if err != nil {
396						break
397					}
398					text = append(text, ':')
399					// decode value
400					text, code, err = codec.code2json(code, text)
401					if err != nil {
402						break
403					}
404					if l > 1 {
405						text = append(text, ',')
406					}
407				}
408			}
409		} else {
410			comma := false
411			code = code[1:]
412			for code[0] != Terminator {
413				if comma {
414					text = append(text, ',')
415				}
416				// decode key
417				text, code, err = codec.code2json(code, text)
418				if err != nil {
419					break
420				}
421				text = append(text, ':')
422				// decode value
423				text, code, err = codec.code2json(code, text)
424				if err != nil {
425					break
426				}
427				comma = true
428			}
429		}
430		remaining = code[1:] // remove Terminator
431		text = append(text, '}')
432	}
433	return text, remaining, err
434}
435
436// local function that sorts JSON property objects based on property names.
437func (codec *Codec) sortProps(props map[string]interface{}) []string {
438	keys := make([]string, 0, len(props))
439	for k := range props {
440		keys = append(keys, k)
441	}
442	ss := sort.StringSlice(keys)
443	ss.Sort()
444	return keys
445}
446
447// get the encoded datum (basic JSON datatype) based on Terminator and return a
448// tuple of, `encoded-datum`, `remaining-code`, where remaining-code starts
449// after the Terminator
450func getDatum(code []byte) (datum []byte, remaining []byte) {
451	var i int
452	var b byte
453	for i, b = range code {
454		if b == Terminator {
455			break
456		}
457	}
458	return code[:i], code[i+1:]
459}
460
461func (codec *Codec) normalizeFloat(value float64, code []byte) ([]byte, error) {
462	switch codec.numberType.(type) {
463	case float64:
464		cs := EncodeFloat([]byte(strconv.FormatFloat(value, 'e', -1, 64)), code)
465		return cs, nil
466
467	case int64:
468		return EncodeInt([]byte(strconv.Itoa(int(value))), code), nil
469
470	case string:
471		cs := EncodeFloat([]byte(strconv.FormatFloat(value, 'e', -1, 64)), code)
472		return cs, nil
473	}
474	return nil, ErrorNumberType
475}
476
477func (codec *Codec) denormalizeFloat(text []byte) ([]byte, error) {
478	var err error
479	var f float64
480
481	switch codec.numberType.(type) {
482	case float64:
483		return text, nil
484
485	case int64:
486		f, err = strconv.ParseFloat(string(text), 64)
487		return []byte(strconv.Itoa(int(f))), nil
488
489	case string:
490		f, err = strconv.ParseFloat(string(text), 64)
491		if err == nil {
492			return []byte(strconv.FormatFloat(f, 'f', -1, 64)), nil
493		}
494
495	default:
496		return text, nil
497	}
498	return nil, ErrorNumberType
499}
500
501// Equal checks wether n is MissingLiteral
502func (m Missing) Equal(n string) bool {
503	s := string(m)
504	if len(n) == len(s) && n[0] == '~' && n[1] == '[' {
505		return s == n
506	}
507	return false
508}
509
510func (codec *Codec) n1ql2code(val n1ql.Value, code []byte) ([]byte, error) {
511	var cs []byte
512	var err error
513
514	switch val.Type() {
515	case n1ql.NULL:
516		code = append(code, TypeNull, Terminator)
517	case n1ql.BOOLEAN:
518		act := val.ActualForIndex().(bool)
519		if act {
520			code = append(code, TypeTrue, Terminator)
521		} else {
522			code = append(code, TypeFalse, Terminator)
523		}
524	case n1ql.NUMBER:
525		act := val.ActualForIndex()
526		code = append(code, TypeNumber)
527		var cs []byte
528		switch act.(type) {
529		case float64:
530			cs, err = codec.normalizeFloat(act.(float64), code[1:])
531		case int64:
532			var intStr string
533			var number Integer
534			intStr, err = number.ConvertToScientificNotation(act.(int64))
535			cs = EncodeFloat([]byte(intStr), code[1:])
536		}
537		if err == nil {
538			code = code[:len(code)+len(cs)]
539			code = append(code, Terminator)
540		}
541	case n1ql.STRING:
542		code = append(code, TypeString)
543		act := val.ActualForIndex().(string)
544		cs = suffixEncodeString([]byte(act), code[1:])
545		code = code[:len(code)+len(cs)]
546		code = append(code, Terminator)
547	case n1ql.MISSING:
548		code = append(code, TypeMissing)
549		code = append(code, Terminator)
550	case n1ql.ARRAY:
551		act := val.ActualForIndex().([]interface{})
552		code = append(code, TypeArray)
553		if codec.arrayLenPrefix {
554			arrlen := Length(len(act))
555			if cs, err = codec.json2code(arrlen, code[1:]); err == nil {
556				code = code[:len(code)+len(cs)]
557			}
558		}
559		if err == nil {
560			for _, val := range act {
561				l := len(code)
562				cs, err = codec.n1ql2code(n1ql.NewValue(val), code[l:])
563				if err == nil {
564					code = code[:l+len(cs)]
565					continue
566				}
567				break
568			}
569			code = append(code, Terminator)
570		}
571	case n1ql.OBJECT:
572		act := val.ActualForIndex().(map[string]interface{})
573		code = append(code, TypeObj)
574		if codec.propertyLenPrefix {
575			proplen := Length(len(act))
576			if cs, err = codec.json2code(proplen, code[1:]); err == nil {
577				code = code[:len(code)+len(cs)]
578			}
579		}
580
581		if err == nil {
582			keys := codec.sortProps(val.ActualForIndex().(map[string]interface{}))
583			for _, key := range keys {
584				l := len(code)
585				// encode key
586				if cs, err = codec.n1ql2code(n1ql.NewValue(key), code[l:]); err != nil {
587					break
588				}
589				code = code[:l+len(cs)]
590				l = len(code)
591				// encode value
592				if cs, err = codec.n1ql2code(n1ql.NewValue(act[key]), code[l:]); err != nil {
593					break
594				}
595				code = code[:l+len(cs)]
596			}
597			code = append(code, Terminator)
598		}
599	}
600
601	return code, err
602}
603
604// Caller is responsible for providing sufficiently sized buffer
605// Otherwise it may panic
606func (codec *Codec) EncodeN1QLValue(val n1ql.Value, buf []byte) (bs []byte, err error) {
607	defer func() {
608		if r := recover(); r != nil {
609			if strings.Contains(fmt.Sprint(r), "slice bounds out of range") {
610				err = ErrorOutputLen
611			} else {
612				err = fmt.Errorf("%v", r)
613			}
614		}
615	}()
616	return codec.n1ql2code(val, buf)
617}
618
619type Integer struct{}
620
621// Formats an int64 to scientic notation. Example:
622// 75284 converts to 7.5284e+04
623// 1200000 converts to 1.2e+06
624// -612988654 converts to -6.12988654e+08
625// This is used in encode path
626func (i *Integer) ConvertToScientificNotation(val int64) (string, error) {
627
628	// For integers that can be represented precisely with
629	// float64, return using FormatFloat
630	if val < 9007199254740991 && val > -9007199254740991 {
631		return strconv.FormatFloat(float64(val), 'e', -1, 64), nil
632	}
633
634	intStr := strconv.FormatInt(val, 10)
635	if len(intStr) == 0 {
636		return "", nil
637	}
638
639	format := func(str string) string {
640		var first, rem string
641		first = str[0:1]
642		if len(str) >= 2 {
643			rem = str[1:]
644		}
645		if len(rem) == 0 { // The integer is a single digit number
646			return first + ".e+00"
647		} else {
648			return first + "." + strings.TrimRight(rem, "0") + "e+" + strconv.Itoa(len(rem))
649		}
650	}
651
652	var enotation string
653	sign := ""
654	if intStr[0:1] == "-" || intStr[0:1] == "+" { // The integer has a sign
655		sign = intStr[0:1]
656		enotation = format(intStr[1:])
657	} else {
658		enotation = format(intStr)
659	}
660
661	return sign + enotation, nil
662}
663
664// This function has been retained to support unit tests.
665// Regular code path should use ConvertToScientificNotation.
666func (i *Integer) ConvertToScientificNotation_TestOnly(val int64) (string, error) {
667	intStr := strconv.FormatInt(val, 10)
668	if len(intStr) == 0 {
669		return "", nil
670	}
671
672	format := func(str string) string {
673		var first, rem string
674		first = str[0:1]
675		if len(str) >= 2 {
676			rem = str[1:]
677		}
678		if len(rem) == 0 { // The integer is a single digit number
679			return first + ".e+00"
680		} else {
681			return first + "." + rem + "e+" + strconv.Itoa(len(rem))
682		}
683	}
684
685	var enotation string
686	sign := ""
687	if intStr[0:1] == "-" || intStr[0:1] == "+" { // The integer has a sign
688		sign = intStr[0:1]
689		enotation = format(intStr[1:])
690	} else {
691		enotation = format(intStr)
692	}
693
694	return sign + enotation, nil
695}
696
697// If float, return e notation
698// If integer, convert from e notation to standard notation
699// This is used in decode path
700func (i *Integer) TryConvertFromScientificNotation(val []byte) (ret []byte, isInt64 bool) {
701	defer func() {
702		if r := recover(); r != nil {
703			ret = val
704			isInt64 = false
705		}
706	}()
707
708	number := string(val)
709	sign := ""
710	if number[0:1] == "-" || number[0:1] == "+" {
711		sign = number[0:1]
712		number = number[1:]
713	}
714
715	decimalPos := strings.Index(number, ".")
716	ePos := strings.Index(number, "e")
717	characteristic := number[0:decimalPos]
718	mantissa := number[decimalPos+1 : ePos]
719
720	exp, err := strconv.ParseInt(number[ePos+1:], 10, 64)
721	if err != nil {
722		return val, false // error condition, return input format
723	}
724
725	if exp > 0 && (int(exp) >= len(mantissa)) { // It is an integer
726		if int(exp) > len(mantissa) {
727			mantissa = mantissa + strings.Repeat("0", int(exp)-len(mantissa))
728		}
729		if characteristic == "0" {
730			return []byte(sign + mantissa), true
731		} else {
732			return []byte(sign + characteristic + mantissa), true
733		}
734	}
735
736	return val, false
737}
738
739// Caller is responsible for providing sufficiently sized buffer
740// Otherwise it may panic
741func (codec *Codec) DecodeN1QLValue(code, buf []byte) (val n1ql.Value, err error) {
742
743	defer func() {
744		if r := recover(); r != nil {
745			if strings.Contains(fmt.Sprint(r), "slice bounds out of range") {
746				err = ErrorOutputLen
747			} else {
748				err = fmt.Errorf("%v", r)
749			}
750		}
751	}()
752
753	val, _, err = codec.code2n1ql(code, buf, true)
754	return val, err
755}
756
757func (codec *Codec) code2n1ql(code, text []byte, decode bool) (n1ql.Value, []byte, error) {
758	if len(code) == 0 {
759		return nil, nil, nil
760	}
761
762	var ts, remaining, datum []byte
763	var err error
764	var n1qlVal n1ql.Value
765
766	switch code[0] {
767	case Terminator:
768		remaining = code
769
770	case TypeMissing:
771		datum, remaining = getDatum(code)
772		if decode {
773			n1qlVal = n1ql.NewMissingValue()
774		}
775
776	case TypeNull:
777		datum, remaining = getDatum(code)
778		if decode {
779			n1qlVal = n1ql.NewNullValue()
780		}
781
782	case TypeTrue:
783		datum, remaining = getDatum(code)
784		if decode {
785			n1qlVal = n1ql.TRUE_VALUE
786		}
787
788	case TypeFalse:
789		datum, remaining = getDatum(code)
790		if decode {
791			n1qlVal = n1ql.FALSE_VALUE
792		}
793
794	case TypeLength:
795		datum, remaining = getDatum(code)
796		if decode {
797			var val int64
798			_, ts = DecodeInt(datum[1:], text)
799			val, err = strconv.ParseInt(string(ts), 10, 64)
800			if err == nil {
801				n1qlVal = n1ql.NewValue(val)
802			}
803		}
804
805	case TypeNumber:
806		datum, remaining = getDatum(code)
807		if decode {
808			ts = DecodeFloat(datum[1:], text)
809			ts, err = codec.denormalizeFloat(ts)
810			ts = bytes.TrimLeft(ts, "+")
811			var number Integer
812			var isInt64 bool
813			ts, isInt64 = number.TryConvertFromScientificNotation(ts)
814
815			if isInt64 {
816				var val int64
817				val, err = strconv.ParseInt(string(ts), 10, 64)
818				if err == nil {
819					n1qlVal = n1ql.NewValue(val)
820				}
821			} else {
822				var val float64
823				val, err = strconv.ParseFloat(string(ts), 64)
824				if err == nil {
825					n1qlVal = n1ql.NewValue(val)
826				}
827			}
828		}
829
830	case TypeString:
831		var strb []byte
832		strb, remaining, err = suffixDecodeString(code[1:], text)
833		if decode && err == nil {
834			n1qlVal = n1ql.NewValue(string(strb))
835		}
836
837	case TypeArray:
838		var l int
839		var tv n1ql.Value
840		arrval := make([]interface{}, 0)
841		if codec.arrayLenPrefix {
842			datum, code = getDatum(code[1:])
843			_, ts := DecodeInt(datum[1:], text)
844			l, err = strconv.Atoi(string(ts))
845			if err == nil {
846				for ; l > 0; l-- {
847					ln := len(text)
848					lnc := len(code)
849					tv, code, err = codec.code2n1ql(code, text[ln:], decode)
850					if err != nil {
851						break
852					}
853					text = text[:ln+lnc-len(code)]
854					if decode {
855						arrval = append(arrval, tv.ActualForIndex())
856					}
857				}
858				if decode {
859					n1qlVal = n1ql.NewValue(arrval)
860				}
861			}
862		} else {
863			code = code[1:]
864			for code[0] != Terminator {
865				ln := len(text)
866				lnc := len(code)
867				tv, code, err = codec.code2n1ql(code, text[ln:], decode)
868				if err != nil {
869					break
870				}
871				text = text[:ln+lnc-len(code)]
872				if decode {
873					arrval = append(arrval, tv.ActualForIndex())
874				}
875			}
876			if decode {
877				n1qlVal = n1ql.NewValue(arrval)
878			}
879		}
880		remaining = code[1:] // remove Terminator
881
882	case TypeObj:
883		var l int
884		var key, value n1ql.Value
885		objval := make(map[string]interface{})
886		if codec.propertyLenPrefix {
887			datum, code = getDatum(code[1:])
888			_, ts := DecodeInt(datum[1:], text)
889			l, err = strconv.Atoi(string(ts))
890			if err == nil {
891				for ; l > 0; l-- {
892					// decode key
893					ln := len(text)
894					lnc := len(code)
895					key, code, err = codec.code2n1ql(code, text[ln:], decode)
896					if err != nil {
897						break
898					}
899					text = text[:ln+lnc-len(code)]
900					// decode value
901					ln = len(text)
902					lnc = len(code)
903					value, code, err = codec.code2n1ql(code, text[ln:], decode)
904					if err != nil {
905						break
906					}
907					text = text[:ln+lnc-len(code)]
908					if decode {
909						if keystr, ok := key.ActualForIndex().(string); ok {
910							objval[keystr] = value.ActualForIndex()
911						} else {
912							err = ErrInvalidKeyTypeInObject
913							break
914						}
915					}
916				}
917				if decode {
918					n1qlVal = n1ql.NewValue(objval)
919				}
920			}
921		} else {
922			code = code[1:]
923			for code[0] != Terminator {
924				// decode key
925				ln := len(text)
926				lnc := len(code)
927				key, code, err = codec.code2n1ql(code, text[ln:], decode)
928				if err != nil {
929					break
930				}
931				text = text[:ln+lnc-len(code)]
932				// decode value
933				ln = len(text)
934				lnc = len(code)
935				value, code, err = codec.code2n1ql(code, text[ln:], decode)
936				if err != nil {
937					break
938				}
939				text = text[:ln+lnc-len(code)]
940				if decode {
941					if keystr, ok := key.ActualForIndex().(string); ok {
942						objval[keystr] = value.ActualForIndex()
943					} else {
944						err = ErrInvalidKeyTypeInObject
945						break
946					}
947				}
948			}
949			if decode {
950				n1qlVal = n1ql.NewValue(objval)
951			}
952		}
953		remaining = code[1:] // remove Terminator
954	}
955	return n1qlVal, remaining, err
956}
957
958// FixEncodedInt is a special purpose method only to address MB-28956.
959// Do not use otherwise.
960func (codec *Codec) FixEncodedInt(code, buf []byte) ([]byte, error) {
961
962	var fixed []byte
963	var err error
964
965	defer func() {
966		if r := recover(); r != nil {
967			if strings.Contains(fmt.Sprint(r), "slice bounds out of range") {
968				err = ErrorOutputLen
969			} else {
970				err = fmt.Errorf("%v", r)
971			}
972		}
973	}()
974
975	fixed, _, err = codec.fixEncodedInt(code, buf)
976	return fixed, err
977}
978
979// fixEncodedInt is a special purpose method only to address MB-28956.
980// Do not use otherwise.
981func (codec *Codec) fixEncodedInt(code, text []byte) ([]byte, []byte, error) {
982	if len(code) == 0 {
983		return text, code, nil
984	}
985
986	var ts, remaining, datum []byte
987	var err error
988
989	switch code[0] {
990	case Terminator:
991		remaining = code
992
993	case TypeMissing, TypeNull, TypeTrue, TypeFalse, TypeLength:
994		datum, remaining = getDatum(code)
995		text = append(text, datum...)
996		text = append(text, Terminator)
997
998	case TypeNumber:
999		text = append(text, TypeNumber)
1000
1001		datum, remaining = getDatum(code)
1002		ts = DecodeFloat(datum[1:], text[1:])
1003		ts, err = codec.denormalizeFloat(ts)
1004		ts = bytes.TrimLeft(ts, "+")
1005
1006		var number Integer
1007		var isInt64 bool
1008		ts, isInt64 = number.TryConvertFromScientificNotation(ts)
1009
1010		if isInt64 {
1011			var intStr string
1012			var intval int64
1013			intval, err = strconv.ParseInt(string(ts), 10, 64)
1014			if err != nil {
1015				break
1016			}
1017			intStr, err = number.ConvertToScientificNotation(intval)
1018			ts = EncodeFloat([]byte(intStr), text[1:])
1019		} else {
1020			var val float64
1021			val, err = strconv.ParseFloat(string(ts), 64)
1022			ts, err = codec.normalizeFloat(val, text[1:])
1023		}
1024
1025		text = append(text, ts...)
1026		text = append(text, Terminator)
1027
1028	case TypeString:
1029		datum, remaining, err = getStringDatum(code)
1030		text = append(text, datum...)
1031		text = append(text, Terminator)
1032
1033	case TypeArray:
1034		var l int
1035		text = append(text, TypeArray)
1036		if codec.arrayLenPrefix {
1037			datum, code = getDatum(code[1:])
1038			_, ts := DecodeInt(datum[1:], text[1:])
1039			l, err = strconv.Atoi(string(ts))
1040
1041			text = append(text, datum...)
1042			text = append(text, Terminator)
1043			if err == nil {
1044				for ; l > 0; l-- {
1045					ln := len(text)
1046					ts, code, err = codec.fixEncodedInt(code, text[ln:])
1047					if err != nil {
1048						break
1049					}
1050					text = append(text, ts...)
1051				}
1052			}
1053		} else {
1054			code = code[1:]
1055			for code[0] != Terminator {
1056				ln := len(text)
1057				ts, code, err = codec.fixEncodedInt(code, text[ln:])
1058				if err != nil {
1059					break
1060				}
1061				text = append(text, ts...)
1062			}
1063		}
1064		remaining = code[1:] // remove Terminator
1065		text = append(text, Terminator)
1066
1067	case TypeObj:
1068		var l int
1069		var key, value []byte
1070		text = append(text, TypeObj)
1071		if codec.propertyLenPrefix {
1072			datum, code = getDatum(code[1:])
1073			_, ts := DecodeInt(datum[1:], text[1:])
1074			l, err = strconv.Atoi(string(ts))
1075
1076			text = append(text, datum...)
1077			text = append(text, Terminator)
1078			if err == nil {
1079				for ; l > 0; l-- {
1080					// decode key
1081					ln := len(text)
1082					key, code, err = codec.fixEncodedInt(code, text[ln:])
1083					if err != nil {
1084						break
1085					}
1086					text = append(text, key...)
1087					// decode value
1088					ln = len(text)
1089					value, code, err = codec.fixEncodedInt(code, text[ln:])
1090					if err != nil {
1091						break
1092					}
1093					text = append(text, value...)
1094				}
1095			}
1096		} else {
1097			code = code[1:]
1098			for code[0] != Terminator {
1099				// decode key
1100				ln := len(text)
1101				key, code, err = codec.fixEncodedInt(code, text[ln:])
1102				if err != nil {
1103					break
1104				}
1105				text = append(text, key...)
1106				// decode value
1107				ln = len(text)
1108				value, code, err = codec.fixEncodedInt(code, text[ln:])
1109				if err != nil {
1110					break
1111				}
1112				text = append(text, value...)
1113			}
1114		}
1115		remaining = code[1:] // remove Terminator
1116		text = append(text, Terminator)
1117	}
1118	return text, remaining, err
1119}
1120
1121func getStringDatum(code []byte) ([]byte, []byte, error) {
1122	for i := 0; i < len(code); i++ {
1123		x := code[i]
1124		if x == Terminator {
1125			i++
1126			switch x = code[i]; x {
1127			case Terminator:
1128				if i == (len(code)) {
1129					return nil, nil, nil
1130				}
1131				return code[:i], code[i+1:], nil
1132			default:
1133				return nil, nil, ErrorSuffixDecoding
1134			}
1135			continue
1136		}
1137	}
1138	return nil, nil, ErrorSuffixDecoding
1139}
1140