1//  Copyright (c) 2014 Couchbase, Inc.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// 		http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package bleve
16
17import (
18	"context"
19	"fmt"
20	"io/ioutil"
21	"log"
22	"math"
23	"os"
24	"reflect"
25	"sort"
26	"strconv"
27	"strings"
28	"sync"
29	"testing"
30	"time"
31
32	"github.com/blevesearch/bleve/analysis/analyzer/keyword"
33	"github.com/blevesearch/bleve/document"
34	"github.com/blevesearch/bleve/index"
35	"github.com/blevesearch/bleve/index/store/null"
36	"github.com/blevesearch/bleve/mapping"
37	"github.com/blevesearch/bleve/search"
38	"github.com/blevesearch/bleve/search/query"
39
40	"github.com/blevesearch/bleve/index/scorch"
41	"github.com/blevesearch/bleve/index/upsidedown"
42)
43
44func TestCrud(t *testing.T) {
45	defer func() {
46		err := os.RemoveAll("testidx")
47		if err != nil {
48			t.Fatal(err)
49		}
50	}()
51
52	index, err := New("testidx", NewIndexMapping())
53	if err != nil {
54		t.Fatal(err)
55	}
56
57	doca := map[string]interface{}{
58		"name": "marty",
59		"desc": "gophercon india",
60	}
61	err = index.Index("a", doca)
62	if err != nil {
63		t.Error(err)
64	}
65
66	docy := map[string]interface{}{
67		"name": "jasper",
68		"desc": "clojure",
69	}
70	err = index.Index("y", docy)
71	if err != nil {
72		t.Error(err)
73	}
74
75	err = index.Delete("y")
76	if err != nil {
77		t.Error(err)
78	}
79
80	docx := map[string]interface{}{
81		"name": "rose",
82		"desc": "googler",
83	}
84	err = index.Index("x", docx)
85	if err != nil {
86		t.Error(err)
87	}
88
89	err = index.SetInternal([]byte("status"), []byte("pending"))
90	if err != nil {
91		t.Error(err)
92	}
93
94	docb := map[string]interface{}{
95		"name": "steve",
96		"desc": "cbft master",
97	}
98	batch := index.NewBatch()
99	err = batch.Index("b", docb)
100	if err != nil {
101		t.Error(err)
102	}
103	batch.Delete("x")
104	batch.SetInternal([]byte("batchi"), []byte("batchv"))
105	batch.DeleteInternal([]byte("status"))
106	err = index.Batch(batch)
107	if err != nil {
108		t.Error(err)
109	}
110	val, err := index.GetInternal([]byte("batchi"))
111	if err != nil {
112		t.Error(err)
113	}
114	if string(val) != "batchv" {
115		t.Errorf("expected 'batchv', got '%s'", val)
116	}
117	val, err = index.GetInternal([]byte("status"))
118	if err != nil {
119		t.Error(err)
120	}
121	if val != nil {
122		t.Errorf("expected nil, got '%s'", val)
123	}
124
125	err = index.SetInternal([]byte("seqno"), []byte("7"))
126	if err != nil {
127		t.Error(err)
128	}
129	err = index.SetInternal([]byte("status"), []byte("ready"))
130	if err != nil {
131		t.Error(err)
132	}
133	err = index.DeleteInternal([]byte("status"))
134	if err != nil {
135		t.Error(err)
136	}
137	val, err = index.GetInternal([]byte("status"))
138	if err != nil {
139		t.Error(err)
140	}
141	if val != nil {
142		t.Errorf("expected nil, got '%s'", val)
143	}
144
145	val, err = index.GetInternal([]byte("seqno"))
146	if err != nil {
147		t.Error(err)
148	}
149	if string(val) != "7" {
150		t.Errorf("expected '7', got '%s'", val)
151	}
152
153	// close the index, open it again, and try some more things
154	err = index.Close()
155	if err != nil {
156		t.Fatal(err)
157	}
158
159	index, err = Open("testidx")
160	if err != nil {
161		t.Fatal(err)
162	}
163	defer func() {
164		err := index.Close()
165		if err != nil {
166			t.Fatal(err)
167		}
168	}()
169
170	count, err := index.DocCount()
171	if err != nil {
172		t.Fatal(err)
173	}
174	if count != 2 {
175		t.Errorf("expected doc count 2, got %d", count)
176	}
177
178	doc, err := index.Document("a")
179	if err != nil {
180		t.Fatal(err)
181	}
182	if doc == nil {
183		t.Errorf("expected doc not nil, got nil")
184	}
185	foundNameField := false
186	for _, field := range doc.Fields {
187		if field.Name() == "name" && string(field.Value()) == "marty" {
188			foundNameField = true
189		}
190	}
191	if !foundNameField {
192		t.Errorf("expected to find field named 'name' with value 'marty'")
193	}
194
195	fields, err := index.Fields()
196	if err != nil {
197		t.Fatal(err)
198	}
199	expectedFields := map[string]bool{
200		"_all": false,
201		"name": false,
202		"desc": false,
203	}
204	if len(fields) < len(expectedFields) {
205		t.Fatalf("expected %d fields got %d", len(expectedFields), len(fields))
206	}
207	for _, f := range fields {
208		expectedFields[f] = true
209	}
210	for ef, efp := range expectedFields {
211		if !efp {
212			t.Errorf("field %s is missing", ef)
213		}
214	}
215}
216
217func TestIndexCreateNewOverExisting(t *testing.T) {
218	defer func() {
219		err := os.RemoveAll("testidx")
220		if err != nil {
221			t.Fatal(err)
222		}
223	}()
224
225	index, err := New("testidx", NewIndexMapping())
226	if err != nil {
227		t.Fatal(err)
228	}
229	err = index.Close()
230	if err != nil {
231		t.Fatal(err)
232	}
233	index, err = New("testidx", NewIndexMapping())
234	if err != ErrorIndexPathExists {
235		t.Fatalf("expected error index path exists, got %v", err)
236	}
237}
238
239func TestIndexOpenNonExisting(t *testing.T) {
240	_, err := Open("doesnotexist")
241	if err != ErrorIndexPathDoesNotExist {
242		t.Fatalf("expected error index path does not exist, got %v", err)
243	}
244}
245
246func TestIndexOpenMetaMissingOrCorrupt(t *testing.T) {
247	defer func() {
248		err := os.RemoveAll("testidx")
249		if err != nil {
250			t.Fatal(err)
251		}
252	}()
253
254	index, err := New("testidx", NewIndexMapping())
255	if err != nil {
256		t.Fatal(err)
257	}
258	err = index.Close()
259	if err != nil {
260		t.Fatal(err)
261	}
262
263	// now intentionally change the storage type
264	err = ioutil.WriteFile("testidx/index_meta.json", []byte(`{"storage":"mystery"}`), 0666)
265	if err != nil {
266		t.Fatal(err)
267	}
268
269	index, err = Open("testidx")
270	if err != ErrorUnknownStorageType {
271		t.Fatalf("expected error unknown storage type, got %v", err)
272	}
273
274	// now intentionally corrupt the metadata
275	err = ioutil.WriteFile("testidx/index_meta.json", []byte("corrupted"), 0666)
276	if err != nil {
277		t.Fatal(err)
278	}
279
280	index, err = Open("testidx")
281	if err != ErrorIndexMetaCorrupt {
282		t.Fatalf("expected error index metadata corrupted, got %v", err)
283	}
284
285	// now intentionally remove the metadata
286	err = os.Remove("testidx/index_meta.json")
287	if err != nil {
288		t.Fatal(err)
289	}
290
291	index, err = Open("testidx")
292	if err != ErrorIndexMetaMissing {
293		t.Fatalf("expected error index metadata missing, got %v", err)
294	}
295}
296
297func TestInMemIndex(t *testing.T) {
298
299	index, err := NewMemOnly(NewIndexMapping())
300	if err != nil {
301		t.Fatal(err)
302	}
303	err = index.Close()
304	if err != nil {
305		t.Fatal(err)
306	}
307}
308
309func TestClosedIndex(t *testing.T) {
310	index, err := NewMemOnly(NewIndexMapping())
311	if err != nil {
312		t.Fatal(err)
313	}
314	err = index.Close()
315	if err != nil {
316		t.Fatal(err)
317	}
318
319	err = index.Index("test", "test")
320	if err != ErrorIndexClosed {
321		t.Errorf("expected error index closed, got %v", err)
322	}
323
324	err = index.Delete("test")
325	if err != ErrorIndexClosed {
326		t.Errorf("expected error index closed, got %v", err)
327	}
328
329	b := index.NewBatch()
330	err = index.Batch(b)
331	if err != ErrorIndexClosed {
332		t.Errorf("expected error index closed, got %v", err)
333	}
334
335	_, err = index.Document("test")
336	if err != ErrorIndexClosed {
337		t.Errorf("expected error index closed, got %v", err)
338	}
339
340	_, err = index.DocCount()
341	if err != ErrorIndexClosed {
342		t.Errorf("expected error index closed, got %v", err)
343	}
344
345	_, err = index.Search(NewSearchRequest(NewTermQuery("test")))
346	if err != ErrorIndexClosed {
347		t.Errorf("expected error index closed, got %v", err)
348	}
349
350	_, err = index.Fields()
351	if err != ErrorIndexClosed {
352		t.Errorf("expected error index closed, got %v", err)
353	}
354}
355
356type slowQuery struct {
357	actual query.Query
358	delay  time.Duration
359}
360
361func (s *slowQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) {
362	time.Sleep(s.delay)
363	return s.actual.Searcher(i, m, options)
364}
365
366func TestSlowSearch(t *testing.T) {
367	defer func() {
368		err := os.RemoveAll("testidx")
369		if err != nil {
370			t.Fatal(err)
371		}
372	}()
373
374	defer func() {
375		// reset logger back to normal
376		SetLog(log.New(ioutil.Discard, "bleve", log.LstdFlags))
377	}()
378	// set custom logger
379	var sdw sawDataWriter
380	SetLog(log.New(&sdw, "bleve", log.LstdFlags))
381
382	index, err := New("testidx", NewIndexMapping())
383	if err != nil {
384		t.Fatal(err)
385	}
386	defer func() {
387		err := index.Close()
388		if err != nil {
389			t.Fatal(err)
390		}
391	}()
392
393	Config.SlowSearchLogThreshold = 1 * time.Minute
394
395	query := NewTermQuery("water")
396	req := NewSearchRequest(query)
397	_, err = index.Search(req)
398	if err != nil {
399		t.Fatal(err)
400	}
401
402	if sdw.sawData {
403		t.Errorf("expected to not see slow query logged, but did")
404	}
405
406	sq := &slowQuery{
407		actual: query,
408		delay:  50 * time.Millisecond, // on Windows timer resolution is 15ms
409	}
410	req.Query = sq
411	Config.SlowSearchLogThreshold = 1 * time.Microsecond
412	_, err = index.Search(req)
413	if err != nil {
414		t.Fatal(err)
415	}
416
417	if !sdw.sawData {
418		t.Errorf("expected to see slow query logged, but didn't")
419	}
420}
421
422type sawDataWriter struct {
423	sawData bool
424}
425
426func (s *sawDataWriter) Write(p []byte) (n int, err error) {
427	s.sawData = true
428	return len(p), nil
429}
430
431func TestStoredFieldPreserved(t *testing.T) {
432	defer func() {
433		err := os.RemoveAll("testidx")
434		if err != nil {
435			t.Fatal(err)
436		}
437	}()
438
439	index, err := New("testidx", NewIndexMapping())
440	if err != nil {
441		t.Fatal(err)
442	}
443	defer func() {
444		err := index.Close()
445		if err != nil {
446			t.Fatal(err)
447		}
448	}()
449
450	doca := map[string]interface{}{
451		"name": "Marty",
452		"desc": "GopherCON India",
453		"bool": true,
454		"num":  float64(1),
455	}
456	err = index.Index("a", doca)
457	if err != nil {
458		t.Error(err)
459	}
460
461	q := NewTermQuery("marty")
462	req := NewSearchRequest(q)
463	req.Fields = []string{"name", "desc", "bool", "num"}
464	res, err := index.Search(req)
465	if err != nil {
466		t.Error(err)
467	}
468
469	if len(res.Hits) != 1 {
470		t.Fatalf("expected 1 hit, got %d", len(res.Hits))
471	}
472	if res.Hits[0].Fields["name"] != "Marty" {
473		t.Errorf("expected 'Marty' got '%s'", res.Hits[0].Fields["name"])
474	}
475	if res.Hits[0].Fields["desc"] != "GopherCON India" {
476		t.Errorf("expected 'GopherCON India' got '%s'", res.Hits[0].Fields["desc"])
477	}
478	if res.Hits[0].Fields["num"] != float64(1) {
479		t.Errorf("expected '1' got '%v'", res.Hits[0].Fields["num"])
480	}
481	if res.Hits[0].Fields["bool"] != true {
482		t.Errorf("expected 'true' got '%v'", res.Hits[0].Fields["bool"])
483	}
484}
485
486func TestDict(t *testing.T) {
487	defer func() {
488		err := os.RemoveAll("testidx")
489		if err != nil {
490			t.Fatal(err)
491		}
492	}()
493
494	index, err := New("testidx", NewIndexMapping())
495	if err != nil {
496		t.Fatal(err)
497	}
498
499	doca := map[string]interface{}{
500		"name": "marty",
501		"desc": "gophercon india",
502	}
503	err = index.Index("a", doca)
504	if err != nil {
505		t.Error(err)
506	}
507
508	docy := map[string]interface{}{
509		"name": "jasper",
510		"desc": "clojure",
511	}
512	err = index.Index("y", docy)
513	if err != nil {
514		t.Error(err)
515	}
516
517	docx := map[string]interface{}{
518		"name": "rose",
519		"desc": "googler",
520	}
521	err = index.Index("x", docx)
522	if err != nil {
523		t.Error(err)
524	}
525
526	dict, err := index.FieldDict("name")
527	if err != nil {
528		t.Error(err)
529	}
530
531	terms := []string{}
532	de, err := dict.Next()
533	for err == nil && de != nil {
534		terms = append(terms, string(de.Term))
535		de, err = dict.Next()
536	}
537
538	expectedTerms := []string{"jasper", "marty", "rose"}
539	if !reflect.DeepEqual(terms, expectedTerms) {
540		t.Errorf("expected %v, got %v", expectedTerms, terms)
541	}
542
543	err = dict.Close()
544	if err != nil {
545		t.Fatal(err)
546	}
547
548	// test start and end range
549	dict, err = index.FieldDictRange("name", []byte("marty"), []byte("rose"))
550	if err != nil {
551		t.Error(err)
552	}
553
554	terms = []string{}
555	de, err = dict.Next()
556	for err == nil && de != nil {
557		terms = append(terms, string(de.Term))
558		de, err = dict.Next()
559	}
560
561	expectedTerms = []string{"marty", "rose"}
562	if !reflect.DeepEqual(terms, expectedTerms) {
563		t.Errorf("expected %v, got %v", expectedTerms, terms)
564	}
565
566	err = dict.Close()
567	if err != nil {
568		t.Fatal(err)
569	}
570
571	docz := map[string]interface{}{
572		"name": "prefix",
573		"desc": "bob cat cats catting dog doggy zoo",
574	}
575	err = index.Index("z", docz)
576	if err != nil {
577		t.Error(err)
578	}
579
580	dict, err = index.FieldDictPrefix("desc", []byte("cat"))
581	if err != nil {
582		t.Error(err)
583	}
584
585	terms = []string{}
586	de, err = dict.Next()
587	for err == nil && de != nil {
588		terms = append(terms, string(de.Term))
589		de, err = dict.Next()
590	}
591
592	expectedTerms = []string{"cat", "cats", "catting"}
593	if !reflect.DeepEqual(terms, expectedTerms) {
594		t.Errorf("expected %v, got %v", expectedTerms, terms)
595	}
596
597	stats := index.Stats()
598	if stats == nil {
599		t.Errorf("expected IndexStat, got nil")
600	}
601
602	err = dict.Close()
603	if err != nil {
604		t.Fatal(err)
605	}
606
607	err = index.Close()
608	if err != nil {
609		t.Fatal(err)
610	}
611}
612
613func TestBatchString(t *testing.T) {
614	defer func() {
615		err := os.RemoveAll("testidx")
616		if err != nil {
617			t.Fatal(err)
618		}
619	}()
620
621	index, err := New("testidx", NewIndexMapping())
622	if err != nil {
623		t.Fatal(err)
624	}
625	defer func() {
626		err := index.Close()
627		if err != nil {
628			t.Fatal(err)
629		}
630	}()
631
632	batch := index.NewBatch()
633	err = batch.Index("a", []byte("{}"))
634	if err != nil {
635		t.Fatal(err)
636	}
637	batch.Delete("b")
638	batch.SetInternal([]byte("c"), []byte{})
639	batch.DeleteInternal([]byte("d"))
640
641	batchStr := batch.String()
642	if !strings.HasPrefix(batchStr, "Batch (2 ops, 2 internal ops)") {
643		t.Errorf("expected to start with Batch (2 ops, 2 internal ops), did not")
644	}
645	if !strings.Contains(batchStr, "INDEX - 'a'") {
646		t.Errorf("expected to contain INDEX - 'a', did not")
647	}
648	if !strings.Contains(batchStr, "DELETE - 'b'") {
649		t.Errorf("expected to contain DELETE - 'b', did not")
650	}
651	if !strings.Contains(batchStr, "SET INTERNAL - 'c'") {
652		t.Errorf("expected to contain SET INTERNAL - 'c', did not")
653	}
654	if !strings.Contains(batchStr, "DELETE INTERNAL - 'd'") {
655		t.Errorf("expected to contain DELETE INTERNAL - 'd', did not")
656	}
657
658}
659
660func TestIndexMetadataRaceBug198(t *testing.T) {
661	defer func() {
662		err := os.RemoveAll("testidx")
663		if err != nil {
664			t.Fatal(err)
665		}
666	}()
667
668	index, err := New("testidx", NewIndexMapping())
669	if err != nil {
670		t.Fatal(err)
671	}
672	defer func() {
673		err := index.Close()
674		if err != nil {
675			t.Fatal(err)
676		}
677	}()
678
679	wg := sync.WaitGroup{}
680	wg.Add(1)
681	done := make(chan struct{})
682	go func() {
683		for {
684			select {
685			case <-done:
686				wg.Done()
687				return
688			default:
689				_, err2 := index.DocCount()
690				if err2 != nil {
691					t.Fatal(err2)
692				}
693			}
694		}
695	}()
696
697	for i := 0; i < 100; i++ {
698		batch := index.NewBatch()
699		err = batch.Index("a", []byte("{}"))
700		if err != nil {
701			t.Fatal(err)
702		}
703		err = index.Batch(batch)
704		if err != nil {
705			t.Fatal(err)
706		}
707	}
708	close(done)
709	wg.Wait()
710}
711
712func TestSortMatchSearch(t *testing.T) {
713	defer func() {
714		err := os.RemoveAll("testidx")
715		if err != nil {
716			t.Fatal(err)
717		}
718	}()
719
720	index, err := New("testidx", NewIndexMapping())
721	if err != nil {
722		t.Fatal(err)
723	}
724
725	names := []string{"Noam", "Uri", "David", "Yosef", "Eitan", "Itay", "Ariel", "Daniel", "Omer", "Yogev", "Yehonatan", "Moshe", "Mohammed", "Yusuf", "Omar"}
726	days := []string{"Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday"}
727	numbers := []string{"One", "Two", "Three", "Four", "Five", "Six", "Seven", "Eight", "Nine", "Ten", "Eleven", "Twelve"}
728	for i := 0; i < 200; i++ {
729		doc := make(map[string]interface{})
730		doc["Name"] = names[i%len(names)]
731		doc["Day"] = days[i%len(days)]
732		doc["Number"] = numbers[i%len(numbers)]
733		err = index.Index(fmt.Sprintf("%d", i), doc)
734		if err != nil {
735			t.Fatal(err)
736		}
737	}
738
739	req := NewSearchRequest(NewMatchQuery("One"))
740	req.SortBy([]string{"Day", "Name"})
741	req.Fields = []string{"*"}
742	sr, err := index.Search(req)
743	if err != nil {
744		t.Fatal(err)
745	}
746	prev := ""
747	for _, hit := range sr.Hits {
748		val := hit.Fields["Day"].(string)
749		if prev > val {
750			t.Errorf("Hits must be sorted by 'Day'. Found '%s' before '%s'", prev, val)
751		}
752		prev = val
753	}
754	err = index.Close()
755	if err != nil {
756		t.Fatal(err)
757	}
758}
759
760func TestIndexCountMatchSearch(t *testing.T) {
761	defer func() {
762		err := os.RemoveAll("testidx")
763		if err != nil {
764			t.Fatal(err)
765		}
766	}()
767
768	index, err := New("testidx", NewIndexMapping())
769	if err != nil {
770		t.Fatal(err)
771	}
772
773	var wg sync.WaitGroup
774	for i := 0; i < 10; i++ {
775		wg.Add(1)
776		go func(i int) {
777			b := index.NewBatch()
778			for j := 0; j < 200; j++ {
779				id := fmt.Sprintf("%d", (i*200)+j)
780				doc := struct {
781					Body string
782				}{
783					Body: "match",
784				}
785				err := b.Index(id, doc)
786				if err != nil {
787					t.Fatal(err)
788				}
789			}
790			err := index.Batch(b)
791			if err != nil {
792				t.Fatal(err)
793			}
794			wg.Done()
795		}(i)
796	}
797	wg.Wait()
798
799	// search for something that should match all documents
800	sr, err := index.Search(NewSearchRequest(NewMatchQuery("match")))
801	if err != nil {
802		t.Fatal(err)
803	}
804
805	// get the index document count
806	dc, err := index.DocCount()
807	if err != nil {
808		t.Fatal(err)
809	}
810
811	// make sure test is working correctly, doc count should 2000
812	if dc != 2000 {
813		t.Errorf("expected doc count 2000, got %d", dc)
814	}
815
816	// make sure our search found all the documents
817	if dc != sr.Total {
818		t.Errorf("expected search result total %d to match doc count %d", sr.Total, dc)
819	}
820
821	err = index.Close()
822	if err != nil {
823		t.Fatal(err)
824	}
825}
826
827func TestBatchReset(t *testing.T) {
828	defer func() {
829		err := os.RemoveAll("testidx")
830		if err != nil {
831			t.Fatal(err)
832		}
833	}()
834
835	index, err := New("testidx", NewIndexMapping())
836	if err != nil {
837		t.Fatal(err)
838	}
839
840	batch := index.NewBatch()
841	err = batch.Index("k1", struct {
842		Body string
843	}{
844		Body: "v1",
845	})
846	if err != nil {
847		t.Error(err)
848	}
849	batch.Delete("k2")
850	batch.SetInternal([]byte("k3"), []byte("v3"))
851	batch.DeleteInternal([]byte("k4"))
852
853	if batch.Size() != 4 {
854		t.Logf("%v", batch)
855		t.Errorf("expected batch size 4, got %d", batch.Size())
856	}
857
858	batch.Reset()
859
860	if batch.Size() != 0 {
861		t.Errorf("expected batch size 0 after reset, got %d", batch.Size())
862	}
863
864	err = index.Close()
865	if err != nil {
866		t.Fatal(err)
867	}
868}
869
870func TestDocumentFieldArrayPositions(t *testing.T) {
871	defer func() {
872		err := os.RemoveAll("testidx")
873		if err != nil {
874			t.Fatal(err)
875		}
876	}()
877
878	index, err := New("testidx", NewIndexMapping())
879	if err != nil {
880		t.Fatal(err)
881	}
882
883	// index a document with an array of strings
884	err = index.Index("k", struct {
885		Messages []string
886	}{
887		Messages: []string{
888			"first",
889			"second",
890			"third",
891			"last",
892		},
893	})
894	if err != nil {
895		t.Fatal(err)
896	}
897
898	// load the document
899	doc, err := index.Document("k")
900	if err != nil {
901		t.Fatal(err)
902	}
903
904	for _, f := range doc.Fields {
905		if reflect.DeepEqual(f.Value(), []byte("first")) {
906			ap := f.ArrayPositions()
907			if len(ap) < 1 {
908				t.Errorf("expected an array position, got none")
909				continue
910			}
911			if ap[0] != 0 {
912				t.Errorf("expected 'first' in array position 0, got %d", ap[0])
913			}
914		}
915		if reflect.DeepEqual(f.Value(), []byte("second")) {
916			ap := f.ArrayPositions()
917			if len(ap) < 1 {
918				t.Errorf("expected an array position, got none")
919				continue
920			}
921			if ap[0] != 1 {
922				t.Errorf("expected 'second' in array position 1, got %d", ap[0])
923			}
924		}
925		if reflect.DeepEqual(f.Value(), []byte("third")) {
926			ap := f.ArrayPositions()
927			if len(ap) < 1 {
928				t.Errorf("expected an array position, got none")
929				continue
930			}
931			if ap[0] != 2 {
932				t.Errorf("expected 'third' in array position 2, got %d", ap[0])
933			}
934		}
935		if reflect.DeepEqual(f.Value(), []byte("last")) {
936			ap := f.ArrayPositions()
937			if len(ap) < 1 {
938				t.Errorf("expected an array position, got none")
939				continue
940			}
941			if ap[0] != 3 {
942				t.Errorf("expected 'last' in array position 3, got %d", ap[0])
943			}
944		}
945	}
946
947	// now index a document in the same field with a single string
948	err = index.Index("k2", struct {
949		Messages string
950	}{
951		Messages: "only",
952	})
953	if err != nil {
954		t.Fatal(err)
955	}
956
957	// load the document
958	doc, err = index.Document("k2")
959	if err != nil {
960		t.Fatal(err)
961	}
962
963	for _, f := range doc.Fields {
964		if reflect.DeepEqual(f.Value(), []byte("only")) {
965			ap := f.ArrayPositions()
966			if len(ap) != 0 {
967				t.Errorf("expected no array positions, got %d", len(ap))
968				continue
969			}
970		}
971	}
972
973	err = index.Close()
974	if err != nil {
975		t.Fatal(err)
976	}
977}
978
979func TestKeywordSearchBug207(t *testing.T) {
980	defer func() {
981		err := os.RemoveAll("testidx")
982		if err != nil {
983			t.Fatal(err)
984		}
985	}()
986
987	f := NewTextFieldMapping()
988	f.Analyzer = keyword.Name
989
990	m := NewIndexMapping()
991	m.DefaultMapping = NewDocumentMapping()
992	m.DefaultMapping.AddFieldMappingsAt("Body", f)
993
994	index, err := New("testidx", m)
995	if err != nil {
996		t.Fatal(err)
997	}
998
999	doc1 := struct {
1000		Body string
1001	}{
1002		Body: "a555c3bb06f7a127cda000005",
1003	}
1004
1005	err = index.Index("a", doc1)
1006	if err != nil {
1007		t.Fatal(err)
1008	}
1009
1010	doc2 := struct {
1011		Body string
1012	}{
1013		Body: "555c3bb06f7a127cda000005",
1014	}
1015
1016	err = index.Index("b", doc2)
1017	if err != nil {
1018		t.Fatal(err)
1019	}
1020
1021	// now search for these terms
1022	sreq := NewSearchRequest(NewTermQuery("a555c3bb06f7a127cda000005"))
1023	sres, err := index.Search(sreq)
1024	if err != nil {
1025		t.Fatal(err)
1026	}
1027	if sres.Total != 1 {
1028		t.Errorf("expected 1 result, got %d", sres.Total)
1029	}
1030	if sres.Hits[0].ID != "a" {
1031		t.Errorf("expecated id 'a', got '%s'", sres.Hits[0].ID)
1032	}
1033
1034	sreq = NewSearchRequest(NewTermQuery("555c3bb06f7a127cda000005"))
1035	sres, err = index.Search(sreq)
1036	if err != nil {
1037		t.Fatal(err)
1038	}
1039	if sres.Total != 1 {
1040		t.Errorf("expected 1 result, got %d", sres.Total)
1041	}
1042	if sres.Hits[0].ID != "b" {
1043		t.Errorf("expecated id 'b', got '%s'", sres.Hits[0].ID)
1044	}
1045
1046	// now do the same searches using query strings
1047	sreq = NewSearchRequest(NewQueryStringQuery("Body:a555c3bb06f7a127cda000005"))
1048	sres, err = index.Search(sreq)
1049	if err != nil {
1050		t.Fatal(err)
1051	}
1052	if sres.Total != 1 {
1053		t.Errorf("expected 1 result, got %d", sres.Total)
1054	}
1055	if sres.Hits[0].ID != "a" {
1056		t.Errorf("expecated id 'a', got '%s'", sres.Hits[0].ID)
1057	}
1058
1059	sreq = NewSearchRequest(NewQueryStringQuery(`Body:555c3bb06f7a127cda000005`))
1060	sres, err = index.Search(sreq)
1061	if err != nil {
1062		t.Fatal(err)
1063	}
1064	if sres.Total != 1 {
1065		t.Errorf("expected 1 result, got %d", sres.Total)
1066	}
1067	if sres.Hits[0].ID != "b" {
1068		t.Errorf("expecated id 'b', got '%s'", sres.Hits[0].ID)
1069	}
1070
1071	err = index.Close()
1072	if err != nil {
1073		t.Fatal(err)
1074	}
1075}
1076
1077func TestTermVectorArrayPositions(t *testing.T) {
1078	defer func() {
1079		err := os.RemoveAll("testidx")
1080		if err != nil {
1081			t.Fatal(err)
1082		}
1083	}()
1084
1085	index, err := New("testidx", NewIndexMapping())
1086	if err != nil {
1087		t.Fatal(err)
1088	}
1089
1090	// index a document with an array of strings
1091	err = index.Index("k", struct {
1092		Messages []string
1093	}{
1094		Messages: []string{
1095			"first",
1096			"second",
1097			"third",
1098			"last",
1099		},
1100	})
1101	if err != nil {
1102		t.Fatal(err)
1103	}
1104
1105	// search for this document in all field
1106	tq := NewTermQuery("second")
1107	tsr := NewSearchRequest(tq)
1108	tsr.IncludeLocations = true
1109	results, err := index.Search(tsr)
1110	if err != nil {
1111		t.Fatal(err)
1112	}
1113	if results.Total != 1 {
1114		t.Fatalf("expected 1 result, got %d", results.Total)
1115	}
1116	if len(results.Hits[0].Locations["Messages"]["second"]) < 1 {
1117		t.Fatalf("expected at least one location")
1118	}
1119	if len(results.Hits[0].Locations["Messages"]["second"][0].ArrayPositions) < 1 {
1120		t.Fatalf("expected at least one location array position")
1121	}
1122	if results.Hits[0].Locations["Messages"]["second"][0].ArrayPositions[0] != 1 {
1123		t.Fatalf("expected array position 1, got %d", results.Hits[0].Locations["Messages"]["second"][0].ArrayPositions[0])
1124	}
1125
1126	// repeat search for this document in Messages field
1127	tq2 := NewTermQuery("third")
1128	tq2.SetField("Messages")
1129	tsr = NewSearchRequest(tq2)
1130	tsr.IncludeLocations = true
1131	results, err = index.Search(tsr)
1132	if err != nil {
1133		t.Fatal(err)
1134	}
1135	if results.Total != 1 {
1136		t.Fatalf("expected 1 result, got %d", results.Total)
1137	}
1138	if len(results.Hits[0].Locations["Messages"]["third"]) < 1 {
1139		t.Fatalf("expected at least one location")
1140	}
1141	if len(results.Hits[0].Locations["Messages"]["third"][0].ArrayPositions) < 1 {
1142		t.Fatalf("expected at least one location array position")
1143	}
1144	if results.Hits[0].Locations["Messages"]["third"][0].ArrayPositions[0] != 2 {
1145		t.Fatalf("expected array position 2, got %d", results.Hits[0].Locations["Messages"]["third"][0].ArrayPositions[0])
1146	}
1147
1148	err = index.Close()
1149	if err != nil {
1150		t.Fatal(err)
1151	}
1152}
1153
1154func TestDocumentStaticMapping(t *testing.T) {
1155	defer func() {
1156		err := os.RemoveAll("testidx")
1157		if err != nil {
1158			t.Fatal(err)
1159		}
1160	}()
1161
1162	m := NewIndexMapping()
1163	m.DefaultMapping = NewDocumentStaticMapping()
1164	m.DefaultMapping.AddFieldMappingsAt("Text", NewTextFieldMapping())
1165	m.DefaultMapping.AddFieldMappingsAt("Date", NewDateTimeFieldMapping())
1166	m.DefaultMapping.AddFieldMappingsAt("Numeric", NewNumericFieldMapping())
1167
1168	index, err := New("testidx", m)
1169	if err != nil {
1170		t.Fatal(err)
1171	}
1172
1173	doc1 := struct {
1174		Text           string
1175		IgnoredText    string
1176		Numeric        float64
1177		IgnoredNumeric float64
1178		Date           time.Time
1179		IgnoredDate    time.Time
1180	}{
1181		Text:           "valid text",
1182		IgnoredText:    "ignored text",
1183		Numeric:        10,
1184		IgnoredNumeric: 20,
1185		Date:           time.Unix(1, 0),
1186		IgnoredDate:    time.Unix(2, 0),
1187	}
1188
1189	err = index.Index("a", doc1)
1190	if err != nil {
1191		t.Fatal(err)
1192	}
1193
1194	fields, err := index.Fields()
1195	if err != nil {
1196		t.Fatal(err)
1197	}
1198	sort.Strings(fields)
1199	expectedFields := []string{"Date", "Numeric", "Text", "_all"}
1200	if len(fields) < len(expectedFields) {
1201		t.Fatalf("invalid field count: %d", len(fields))
1202	}
1203	for i, expected := range expectedFields {
1204		if expected != fields[i] {
1205			t.Fatalf("unexpected field[%d]: %s", i, fields[i])
1206		}
1207	}
1208
1209	err = index.Close()
1210	if err != nil {
1211		t.Fatal(err)
1212	}
1213}
1214
1215func TestIndexEmptyDocId(t *testing.T) {
1216	defer func() {
1217		err := os.RemoveAll("testidx")
1218		if err != nil {
1219			t.Fatal(err)
1220		}
1221	}()
1222
1223	index, err := New("testidx", NewIndexMapping())
1224	if err != nil {
1225		t.Fatal(err)
1226	}
1227	defer func() {
1228		err := index.Close()
1229		if err != nil {
1230			t.Fatal(err)
1231		}
1232	}()
1233
1234	doc := map[string]interface{}{
1235		"body": "nodocid",
1236	}
1237
1238	err = index.Index("", doc)
1239	if err != ErrorEmptyID {
1240		t.Errorf("expect index empty doc id to fail")
1241	}
1242
1243	err = index.Delete("")
1244	if err != ErrorEmptyID {
1245		t.Errorf("expect delete empty doc id to fail")
1246	}
1247
1248	batch := index.NewBatch()
1249	err = batch.Index("", doc)
1250	if err != ErrorEmptyID {
1251		t.Errorf("expect index empty doc id in batch to fail")
1252	}
1253
1254	batch.Delete("")
1255	if batch.Size() > 0 {
1256		t.Errorf("expect delete empty doc id in batch to be ignored")
1257	}
1258}
1259
1260func TestDateTimeFieldMappingIssue287(t *testing.T) {
1261	defer func() {
1262		err := os.RemoveAll("testidx")
1263		if err != nil {
1264			t.Fatal(err)
1265		}
1266	}()
1267
1268	f := NewDateTimeFieldMapping()
1269
1270	m := NewIndexMapping()
1271	m.DefaultMapping = NewDocumentMapping()
1272	m.DefaultMapping.AddFieldMappingsAt("Date", f)
1273
1274	index, err := New("testidx", m)
1275	if err != nil {
1276		t.Fatal(err)
1277	}
1278
1279	type doc struct {
1280		Date time.Time
1281	}
1282
1283	now := time.Now()
1284
1285	// 3hr ago to 1hr ago
1286	for i := 0; i < 3; i++ {
1287		d := doc{now.Add(time.Duration((i - 3)) * time.Hour)}
1288
1289		err = index.Index(strconv.FormatInt(int64(i), 10), d)
1290		if err != nil {
1291			t.Fatal(err)
1292		}
1293	}
1294
1295	// search range across all docs
1296	start := now.Add(-4 * time.Hour)
1297	end := now
1298	sreq := NewSearchRequest(NewDateRangeQuery(start, end))
1299	sres, err := index.Search(sreq)
1300	if err != nil {
1301		t.Fatal(err)
1302	}
1303	if sres.Total != 3 {
1304		t.Errorf("expected 3 results, got %d", sres.Total)
1305	}
1306
1307	// search range includes only oldest
1308	start = now.Add(-4 * time.Hour)
1309	end = now.Add(-121 * time.Minute)
1310	sreq = NewSearchRequest(NewDateRangeQuery(start, end))
1311	sres, err = index.Search(sreq)
1312	if err != nil {
1313		t.Fatal(err)
1314	}
1315	if sres.Total != 1 {
1316		t.Errorf("expected 1 results, got %d", sres.Total)
1317	}
1318	if sres.Total > 0 && sres.Hits[0].ID != "0" {
1319		t.Errorf("expecated id '0', got '%s'", sres.Hits[0].ID)
1320	}
1321
1322	// search range includes only newest
1323	start = now.Add(-61 * time.Minute)
1324	end = now
1325	sreq = NewSearchRequest(NewDateRangeQuery(start, end))
1326	sres, err = index.Search(sreq)
1327	if err != nil {
1328		t.Fatal(err)
1329	}
1330	if sres.Total != 1 {
1331		t.Errorf("expected 1 results, got %d", sres.Total)
1332	}
1333	if sres.Total > 0 && sres.Hits[0].ID != "2" {
1334		t.Errorf("expecated id '2', got '%s'", sres.Hits[0].ID)
1335	}
1336
1337	err = index.Close()
1338	if err != nil {
1339		t.Fatal(err)
1340	}
1341}
1342
1343func TestDocumentFieldArrayPositionsBug295(t *testing.T) {
1344	defer func() {
1345		err := os.RemoveAll("testidx")
1346		if err != nil {
1347			t.Fatal(err)
1348		}
1349	}()
1350
1351	index, err := New("testidx", NewIndexMapping())
1352	if err != nil {
1353		t.Fatal(err)
1354	}
1355
1356	// index a document with an array of strings
1357	err = index.Index("k", struct {
1358		Messages []string
1359		Another  string
1360		MoreData []string
1361	}{
1362		Messages: []string{
1363			"bleve",
1364			"bleve",
1365		},
1366		Another: "text",
1367		MoreData: []string{
1368			"a",
1369			"b",
1370			"c",
1371			"bleve",
1372		},
1373	})
1374	if err != nil {
1375		t.Fatal(err)
1376	}
1377
1378	// search for it in the messages field
1379	tq := NewTermQuery("bleve")
1380	tq.SetField("Messages")
1381	tsr := NewSearchRequest(tq)
1382	tsr.IncludeLocations = true
1383	results, err := index.Search(tsr)
1384	if err != nil {
1385		t.Fatal(err)
1386	}
1387	if results.Total != 1 {
1388		t.Fatalf("expected 1 result, got %d", results.Total)
1389	}
1390	if len(results.Hits[0].Locations["Messages"]["bleve"]) != 2 {
1391		t.Fatalf("expected 2 locations of 'bleve', got %d", len(results.Hits[0].Locations["Messages"]["bleve"]))
1392	}
1393	if results.Hits[0].Locations["Messages"]["bleve"][0].ArrayPositions[0] != 0 {
1394		t.Errorf("expected array position to be 0")
1395	}
1396	if results.Hits[0].Locations["Messages"]["bleve"][1].ArrayPositions[0] != 1 {
1397		t.Errorf("expected array position to be 1")
1398	}
1399
1400	// search for it in all
1401	tq = NewTermQuery("bleve")
1402	tsr = NewSearchRequest(tq)
1403	tsr.IncludeLocations = true
1404	results, err = index.Search(tsr)
1405	if err != nil {
1406		t.Fatal(err)
1407	}
1408	if results.Total != 1 {
1409		t.Fatalf("expected 1 result, got %d", results.Total)
1410	}
1411	if len(results.Hits[0].Locations["Messages"]["bleve"]) != 2 {
1412		t.Fatalf("expected 2 locations of 'bleve', got %d", len(results.Hits[0].Locations["Messages"]["bleve"]))
1413	}
1414	if results.Hits[0].Locations["Messages"]["bleve"][0].ArrayPositions[0] != 0 {
1415		t.Errorf("expected array position to be 0")
1416	}
1417	if results.Hits[0].Locations["Messages"]["bleve"][1].ArrayPositions[0] != 1 {
1418		t.Errorf("expected array position to be 1")
1419	}
1420
1421	err = index.Close()
1422	if err != nil {
1423		t.Fatal(err)
1424	}
1425}
1426
1427func TestBooleanFieldMappingIssue109(t *testing.T) {
1428	defer func() {
1429		err := os.RemoveAll("testidx")
1430		if err != nil {
1431			t.Fatal(err)
1432		}
1433	}()
1434
1435	m := NewIndexMapping()
1436	m.DefaultMapping = NewDocumentMapping()
1437	m.DefaultMapping.AddFieldMappingsAt("Bool", NewBooleanFieldMapping())
1438
1439	index, err := New("testidx", m)
1440	if err != nil {
1441		t.Fatal(err)
1442	}
1443
1444	type doc struct {
1445		Bool bool
1446	}
1447	err = index.Index("true", &doc{Bool: true})
1448	if err != nil {
1449		t.Fatal(err)
1450	}
1451	err = index.Index("false", &doc{Bool: false})
1452	if err != nil {
1453		t.Fatal(err)
1454	}
1455
1456	q := NewBoolFieldQuery(true)
1457	q.SetField("Bool")
1458	sreq := NewSearchRequest(q)
1459	sres, err := index.Search(sreq)
1460	if err != nil {
1461		t.Fatal(err)
1462	}
1463	if sres.Total != 1 {
1464		t.Errorf("expected 1 results, got %d", sres.Total)
1465	}
1466
1467	q = NewBoolFieldQuery(false)
1468	q.SetField("Bool")
1469	sreq = NewSearchRequest(q)
1470	sres, err = index.Search(sreq)
1471	if err != nil {
1472		t.Fatal(err)
1473	}
1474	if sres.Total != 1 {
1475		t.Errorf("expected 1 results, got %d", sres.Total)
1476	}
1477
1478	sreq = NewSearchRequest(NewBoolFieldQuery(true))
1479	sres, err = index.Search(sreq)
1480	if err != nil {
1481		t.Fatal(err)
1482	}
1483	if sres.Total != 1 {
1484		t.Errorf("expected 1 results, got %d", sres.Total)
1485	}
1486
1487	err = index.Close()
1488	if err != nil {
1489		t.Fatal(err)
1490	}
1491}
1492
1493func TestSearchTimeout(t *testing.T) {
1494	defer func() {
1495		err := os.RemoveAll("testidx")
1496		if err != nil {
1497			t.Fatal(err)
1498		}
1499	}()
1500
1501	index, err := New("testidx", NewIndexMapping())
1502	if err != nil {
1503		t.Fatal(err)
1504	}
1505	defer func() {
1506		err := index.Close()
1507		if err != nil {
1508			t.Fatal(err)
1509		}
1510	}()
1511
1512	// first run a search with an absurdly long timeout (should succeed)
1513	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
1514	defer cancel()
1515	query := NewTermQuery("water")
1516	req := NewSearchRequest(query)
1517	_, err = index.SearchInContext(ctx, req)
1518	if err != nil {
1519		t.Fatal(err)
1520	}
1521
1522	// now run a search again with an absurdly low timeout (should timeout)
1523	ctx, cancel = context.WithTimeout(context.Background(), 1*time.Microsecond)
1524	defer cancel()
1525	sq := &slowQuery{
1526		actual: query,
1527		delay:  50 * time.Millisecond, // on Windows timer resolution is 15ms
1528	}
1529	req.Query = sq
1530	_, err = index.SearchInContext(ctx, req)
1531	if err != context.DeadlineExceeded {
1532		t.Fatalf("exected %v, got: %v", context.DeadlineExceeded, err)
1533	}
1534
1535	// now run a search with a long timeout, but with a long query, and cancel it
1536	ctx, cancel = context.WithTimeout(context.Background(), 10*time.Second)
1537	sq = &slowQuery{
1538		actual: query,
1539		delay:  100 * time.Millisecond, // on Windows timer resolution is 15ms
1540	}
1541	req = NewSearchRequest(sq)
1542	cancel()
1543	_, err = index.SearchInContext(ctx, req)
1544	if err != context.Canceled {
1545		t.Fatalf("exected %v, got: %v", context.Canceled, err)
1546	}
1547}
1548
1549// TestConfigCache exposes a concurrent map write with go 1.6
1550func TestConfigCache(t *testing.T) {
1551	for i := 0; i < 100; i++ {
1552		go func() {
1553			_, err := Config.Cache.HighlighterNamed(Config.DefaultHighlighter)
1554			if err != nil {
1555				t.Error(err)
1556			}
1557		}()
1558	}
1559}
1560
1561func TestBatchRaceBug260(t *testing.T) {
1562	defer func() {
1563		err := os.RemoveAll("testidx")
1564		if err != nil {
1565			t.Fatal(err)
1566		}
1567	}()
1568	i, err := New("testidx", NewIndexMapping())
1569	if err != nil {
1570		t.Fatal(err)
1571	}
1572	defer func() {
1573		err := i.Close()
1574		if err != nil {
1575			t.Fatal(err)
1576		}
1577	}()
1578	b := i.NewBatch()
1579	err = b.Index("1", 1)
1580	if err != nil {
1581		t.Fatal(err)
1582	}
1583	err = i.Batch(b)
1584	if err != nil {
1585		t.Fatal(err)
1586	}
1587	b.Reset()
1588	err = b.Index("2", 2)
1589	if err != nil {
1590		t.Fatal(err)
1591	}
1592	err = i.Batch(b)
1593	if err != nil {
1594		t.Fatal(err)
1595	}
1596	b.Reset()
1597}
1598
1599func BenchmarkBatchOverhead(b *testing.B) {
1600	defer func() {
1601		err := os.RemoveAll("testidx")
1602		if err != nil {
1603			b.Fatal(err)
1604		}
1605	}()
1606	m := NewIndexMapping()
1607	i, err := NewUsing("testidx", m, Config.DefaultIndexType, null.Name, nil)
1608	if err != nil {
1609		b.Fatal(err)
1610	}
1611	for n := 0; n < b.N; n++ {
1612		// put 1000 items in a batch
1613		batch := i.NewBatch()
1614		for i := 0; i < 1000; i++ {
1615			err = batch.Index(fmt.Sprintf("%d", i), map[string]interface{}{"name": "bleve"})
1616			if err != nil {
1617				b.Fatal(err)
1618			}
1619		}
1620		err = i.Batch(batch)
1621		if err != nil {
1622			b.Fatal(err)
1623		}
1624		batch.Reset()
1625	}
1626}
1627
1628func TestOpenReadonlyMultiple(t *testing.T) {
1629	defer func() {
1630		err := os.RemoveAll("testidx")
1631		if err != nil {
1632			t.Fatal(err)
1633		}
1634	}()
1635
1636	// build an index and close it
1637	index, err := New("testidx", NewIndexMapping())
1638	if err != nil {
1639		t.Fatal(err)
1640	}
1641
1642	doca := map[string]interface{}{
1643		"name": "marty",
1644		"desc": "gophercon india",
1645	}
1646	err = index.Index("a", doca)
1647	if err != nil {
1648		t.Fatal(err)
1649	}
1650
1651	err = index.Close()
1652	if err != nil {
1653		t.Fatal(err)
1654	}
1655
1656	// now open it read-only
1657	index, err = OpenUsing("testidx", map[string]interface{}{
1658		"read_only": true,
1659	})
1660
1661	if err != nil {
1662		t.Fatal(err)
1663	}
1664
1665	// now open it again
1666	index2, err := OpenUsing("testidx", map[string]interface{}{
1667		"read_only": true,
1668	})
1669
1670	if err != nil {
1671		t.Fatal(err)
1672	}
1673
1674	err = index.Close()
1675	if err != nil {
1676		t.Fatal(err)
1677	}
1678	err = index2.Close()
1679	if err != nil {
1680		t.Fatal(err)
1681	}
1682}
1683
1684// TestBug408 tests for VERY large values of size, even though actual result
1685// set may be reasonable size
1686func TestBug408(t *testing.T) {
1687	type TestStruct struct {
1688		ID     string  `json:"id"`
1689		UserID *string `json:"user_id"`
1690	}
1691
1692	docMapping := NewDocumentMapping()
1693	docMapping.AddFieldMappingsAt("id", NewTextFieldMapping())
1694	docMapping.AddFieldMappingsAt("user_id", NewTextFieldMapping())
1695
1696	indexMapping := NewIndexMapping()
1697	indexMapping.DefaultMapping = docMapping
1698
1699	index, err := NewMemOnly(indexMapping)
1700	if err != nil {
1701		t.Fatal(err)
1702	}
1703
1704	numToTest := 10
1705	matchUserID := "match"
1706	noMatchUserID := "no_match"
1707	matchingDocIds := make(map[string]struct{})
1708
1709	for i := 0; i < numToTest; i++ {
1710		ds := &TestStruct{"id_" + strconv.Itoa(i), nil}
1711		if i%2 == 0 {
1712			ds.UserID = &noMatchUserID
1713		} else {
1714			ds.UserID = &matchUserID
1715			matchingDocIds[ds.ID] = struct{}{}
1716		}
1717		err = index.Index(ds.ID, ds)
1718		if err != nil {
1719			t.Fatal(err)
1720		}
1721	}
1722
1723	cnt, err := index.DocCount()
1724	if err != nil {
1725		t.Fatal(err)
1726	}
1727	if int(cnt) != numToTest {
1728		t.Fatalf("expected %d documents in index, got %d", numToTest, cnt)
1729	}
1730
1731	q := NewTermQuery(matchUserID)
1732	q.SetField("user_id")
1733	searchReq := NewSearchRequestOptions(q, math.MaxInt32, 0, false)
1734	results, err := index.Search(searchReq)
1735	if err != nil {
1736		t.Fatal(err)
1737	}
1738	if int(results.Total) != numToTest/2 {
1739		t.Fatalf("expected %d search hits, got %d", numToTest/2, results.Total)
1740	}
1741
1742	for _, result := range results.Hits {
1743		if _, found := matchingDocIds[result.ID]; !found {
1744			t.Fatalf("document with ID %s not in results as expected", result.ID)
1745		}
1746	}
1747}
1748
1749func TestIndexAdvancedCountMatchSearch(t *testing.T) {
1750	defer func() {
1751		err := os.RemoveAll("testidx")
1752		if err != nil {
1753			t.Fatal(err)
1754		}
1755	}()
1756
1757	index, err := New("testidx", NewIndexMapping())
1758	if err != nil {
1759		t.Fatal(err)
1760	}
1761
1762	var wg sync.WaitGroup
1763	for i := 0; i < 10; i++ {
1764		wg.Add(1)
1765		go func(i int) {
1766			b := index.NewBatch()
1767			for j := 0; j < 200; j++ {
1768				id := fmt.Sprintf("%d", (i*200)+j)
1769
1770				doc := &document.Document{
1771					ID: id,
1772					Fields: []document.Field{
1773						document.NewTextField("body", []uint64{}, []byte("match")),
1774					},
1775					CompositeFields: []*document.CompositeField{
1776						document.NewCompositeField("_all", true, []string{}, []string{}),
1777					},
1778				}
1779
1780				err := b.IndexAdvanced(doc)
1781				if err != nil {
1782					t.Fatal(err)
1783				}
1784			}
1785			err := index.Batch(b)
1786			if err != nil {
1787				t.Fatal(err)
1788			}
1789			wg.Done()
1790		}(i)
1791	}
1792	wg.Wait()
1793
1794	// search for something that should match all documents
1795	sr, err := index.Search(NewSearchRequest(NewMatchQuery("match")))
1796	if err != nil {
1797		t.Fatal(err)
1798	}
1799
1800	// get the index document count
1801	dc, err := index.DocCount()
1802	if err != nil {
1803		t.Fatal(err)
1804	}
1805
1806	// make sure test is working correctly, doc count should 2000
1807	if dc != 2000 {
1808		t.Errorf("expected doc count 2000, got %d", dc)
1809	}
1810
1811	// make sure our search found all the documents
1812	if dc != sr.Total {
1813		t.Errorf("expected search result total %d to match doc count %d", sr.Total, dc)
1814	}
1815
1816	err = index.Close()
1817	if err != nil {
1818		t.Fatal(err)
1819	}
1820}
1821
1822func benchmarkSearchOverhead(indexType string, b *testing.B) {
1823	defer func() {
1824		err := os.RemoveAll("testidx")
1825		if err != nil {
1826			b.Fatal(err)
1827		}
1828	}()
1829
1830	index, err := NewUsing("testidx", NewIndexMapping(),
1831		indexType, Config.DefaultKVStore, nil)
1832	if err != nil {
1833		b.Fatal(err)
1834	}
1835	defer func() {
1836		err := index.Close()
1837		if err != nil {
1838			b.Fatal(err)
1839		}
1840	}()
1841
1842	elements := []string{"air", "water", "fire", "earth"}
1843	for j := 0; j < 10000; j++ {
1844		err = index.Index(fmt.Sprintf("%d", j),
1845			map[string]interface{}{"name": elements[j%len(elements)]})
1846		if err != nil {
1847			b.Fatal(err)
1848		}
1849	}
1850
1851	query1 := NewTermQuery("water")
1852	query2 := NewTermQuery("fire")
1853	query := NewDisjunctionQuery(query1, query2)
1854	req := NewSearchRequest(query)
1855
1856	b.ResetTimer()
1857
1858	for n := 0; n < b.N; n++ {
1859		_, err = index.Search(req)
1860		if err != nil {
1861			b.Fatal(err)
1862		}
1863	}
1864}
1865
1866func BenchmarkUpsidedownSearchOverhead(b *testing.B) {
1867	benchmarkSearchOverhead(upsidedown.Name, b)
1868}
1869
1870func BenchmarkScorchSearchOverhead(b *testing.B) {
1871	benchmarkSearchOverhead(scorch.Name, b)
1872}
1873
1874func TestSearchQueryCallback(t *testing.T) {
1875	defer func() {
1876		err := os.RemoveAll("testidx")
1877		if err != nil {
1878			t.Fatal(err)
1879		}
1880	}()
1881
1882	index, err := New("testidx", NewIndexMapping())
1883	if err != nil {
1884		t.Fatal(err)
1885	}
1886	defer func() {
1887		err := index.Close()
1888		if err != nil {
1889			t.Fatal(err)
1890		}
1891	}()
1892
1893	elements := []string{"air", "water", "fire", "earth"}
1894	for j := 0; j < 10000; j++ {
1895		err = index.Index(fmt.Sprintf("%d", j),
1896			map[string]interface{}{"name": elements[j%len(elements)]})
1897		if err != nil {
1898			t.Fatal(err)
1899		}
1900	}
1901
1902	query := NewTermQuery("water")
1903	req := NewSearchRequest(query)
1904
1905	expErr := fmt.Errorf("MEM_LIMIT_EXCEEDED")
1906	f := func(size uint64) error {
1907		if size > 1000 {
1908			return expErr
1909		}
1910		return nil
1911	}
1912
1913	ctx := context.WithValue(context.Background(), SearchQueryStartCallbackKey,
1914		SearchQueryStartCallbackFn(f))
1915	_, err = index.SearchInContext(ctx, req)
1916	if err != expErr {
1917		t.Fatalf("Expected: %v, Got: %v", expErr, err)
1918	}
1919}
1920
1921func TestBatchMerge(t *testing.T) {
1922	defer func() {
1923		err := os.RemoveAll("testidx")
1924		if err != nil {
1925			t.Fatal(err)
1926		}
1927	}()
1928
1929	index, err := New("testidx", NewIndexMapping())
1930	if err != nil {
1931		t.Fatal(err)
1932	}
1933	doca := map[string]interface{}{
1934		"name":   "scorch",
1935		"desc":   "gophercon india",
1936		"nation": "india",
1937	}
1938
1939	batchA := index.NewBatch()
1940	err = batchA.Index("a", doca)
1941	if err != nil {
1942		t.Error(err)
1943	}
1944	batchA.SetInternal([]byte("batchkA"), []byte("batchvA"))
1945
1946	docb := map[string]interface{}{
1947		"name": "moss",
1948		"desc": "gophercon MV",
1949	}
1950
1951	batchB := index.NewBatch()
1952	err = batchB.Index("b", docb)
1953	if err != nil {
1954		t.Error(err)
1955	}
1956	batchB.SetInternal([]byte("batchkB"), []byte("batchvB"))
1957
1958	docC := map[string]interface{}{
1959		"name":    "blahblah",
1960		"desc":    "inProgress",
1961		"country": "usa",
1962	}
1963
1964	batchC := index.NewBatch()
1965	err = batchC.Index("c", docC)
1966	if err != nil {
1967		t.Error(err)
1968	}
1969	batchC.SetInternal([]byte("batchkC"), []byte("batchvC"))
1970	batchC.SetInternal([]byte("batchkB"), []byte("batchvBNew"))
1971	batchC.Delete("a")
1972	batchC.DeleteInternal([]byte("batchkA"))
1973
1974	batchA.Merge(batchB)
1975
1976	if batchA.Size() != 4 {
1977		t.Errorf("expected batch size 4, got %d", batchA.Size())
1978	}
1979
1980	batchA.Merge(batchC)
1981
1982	if batchA.Size() != 6 {
1983		t.Errorf("expected batch size 6, got %d", batchA.Size())
1984	}
1985
1986	err = index.Batch(batchA)
1987	if err != nil {
1988		t.Fatal(err)
1989	}
1990
1991	// close the index, open it again, and try some more things
1992	err = index.Close()
1993	if err != nil {
1994		t.Fatal(err)
1995	}
1996
1997	index, err = Open("testidx")
1998	if err != nil {
1999		t.Fatal(err)
2000	}
2001	defer func() {
2002		err := index.Close()
2003		if err != nil {
2004			t.Fatal(err)
2005		}
2006	}()
2007
2008	count, err := index.DocCount()
2009	if err != nil {
2010		t.Fatal(err)
2011	}
2012	if count != 2 {
2013		t.Errorf("expected doc count 2, got %d", count)
2014	}
2015
2016	doc, err := index.Document("c")
2017	if err != nil {
2018		t.Fatal(err)
2019	}
2020	if doc == nil {
2021		t.Errorf("expected doc not nil, got nil")
2022	}
2023
2024	val, err := index.GetInternal([]byte("batchkB"))
2025	if err != nil {
2026		t.Fatal(err)
2027	}
2028	if val == nil || string(val) != "batchvBNew" {
2029		t.Errorf("expected val: batchvBNew , got %s", val)
2030	}
2031
2032	val, err = index.GetInternal([]byte("batchkA"))
2033	if err != nil {
2034		t.Fatal(err)
2035	}
2036	if val != nil {
2037		t.Errorf("expected nil, got %s", val)
2038	}
2039
2040	foundNameField := false
2041	for _, field := range doc.Fields {
2042		if field.Name() == "name" && string(field.Value()) == "blahblah" {
2043			foundNameField = true
2044		}
2045	}
2046	if !foundNameField {
2047		t.Errorf("expected to find field named 'name' with value 'blahblah'")
2048	}
2049
2050	fields, err := index.Fields()
2051	if err != nil {
2052		t.Fatal(err)
2053	}
2054
2055	expectedFields := map[string]bool{
2056		"_all":    false,
2057		"name":    false,
2058		"desc":    false,
2059		"country": false,
2060	}
2061	if len(fields) < len(expectedFields) {
2062		t.Fatalf("expected %d fields got %d", len(expectedFields), len(fields))
2063	}
2064
2065	for _, f := range fields {
2066		expectedFields[f] = true
2067	}
2068
2069	for ef, efp := range expectedFields {
2070		if !efp {
2071			t.Errorf("field %s is missing", ef)
2072		}
2073	}
2074
2075}
2076