diff options
Diffstat (limited to 'vendor/google.golang.org/appengine/search')
| -rw-r--r-- | vendor/google.golang.org/appengine/search/doc.go | 209 | ||||
| -rw-r--r-- | vendor/google.golang.org/appengine/search/field.go | 82 | ||||
| -rw-r--r-- | vendor/google.golang.org/appengine/search/search.go | 1121 | ||||
| -rw-r--r-- | vendor/google.golang.org/appengine/search/search_test.go | 1000 | ||||
| -rw-r--r-- | vendor/google.golang.org/appengine/search/struct.go | 251 | ||||
| -rw-r--r-- | vendor/google.golang.org/appengine/search/struct_test.go | 213 |
6 files changed, 2876 insertions, 0 deletions
diff --git a/vendor/google.golang.org/appengine/search/doc.go b/vendor/google.golang.org/appengine/search/doc.go new file mode 100644 index 0000000..da331ce --- /dev/null +++ b/vendor/google.golang.org/appengine/search/doc.go @@ -0,0 +1,209 @@ +// Copyright 2015 Google Inc. All rights reserved. +// Use of this source code is governed by the Apache 2.0 +// license that can be found in the LICENSE file. + +/* +Package search provides a client for App Engine's search service. + + +Basic Operations + +Indexes contain documents. Each index is identified by its name: a +human-readable ASCII string. + +Within an index, documents are associated with an ID, which is also +a human-readable ASCII string. A document's contents are a mapping from +case-sensitive field names to values. Valid types for field values are: + - string, + - search.Atom, + - search.HTML, + - time.Time (stored with millisecond precision), + - float64 (value between -2,147,483,647 and 2,147,483,647 inclusive), + - appengine.GeoPoint. + +The Get and Put methods on an Index load and save a document. +A document's contents are typically represented by a struct pointer. + +Example code: + + type Doc struct { + Author string + Comment string + Creation time.Time + } + + index, err := search.Open("comments") + if err != nil { + return err + } + newID, err := index.Put(ctx, "", &Doc{ + Author: "gopher", + Comment: "the truth of the matter", + Creation: time.Now(), + }) + if err != nil { + return err + } + +A single document can be retrieved by its ID. Pass a destination struct +to Get to hold the resulting document. + + var doc Doc + err := index.Get(ctx, id, &doc) + if err != nil { + return err + } + + +Search and Listing Documents + +Indexes have two methods for retrieving multiple documents at once: Search and +List. + +Searching an index for a query will result in an iterator. As with an iterator +from package datastore, pass a destination struct to Next to decode the next +result. Next will return Done when the iterator is exhausted. + + for t := index.Search(ctx, "Comment:truth", nil); ; { + var doc Doc + id, err := t.Next(&doc) + if err == search.Done { + break + } + if err != nil { + return err + } + fmt.Fprintf(w, "%s -> %#v\n", id, doc) + } + +Search takes a string query to determine which documents to return. The query +can be simple, such as a single word to match, or complex. The query +language is described at +https://cloud.google.com/appengine/docs/go/search/query_strings + +Search also takes an optional SearchOptions struct which gives much more +control over how results are calculated and returned. + +Call List to iterate over all documents in an index. + + for t := index.List(ctx, nil); ; { + var doc Doc + id, err := t.Next(&doc) + if err == search.Done { + break + } + if err != nil { + return err + } + fmt.Fprintf(w, "%s -> %#v\n", id, doc) + } + + +Fields and Facets + +A document's contents can be represented by a variety of types. These are +typically struct pointers, but they can also be represented by any type +implementing the FieldLoadSaver interface. The FieldLoadSaver allows metadata +to be set for the document with the DocumentMetadata type. Struct pointers are +more strongly typed and are easier to use; FieldLoadSavers are more flexible. + +A document's contents can be expressed in two ways: fields and facets. + +Fields are the most common way of providing content for documents. Fields can +store data in multiple types and can be matched in searches using query +strings. + +Facets provide a way to attach categorical information to a document. The only +valid types for facets are search.Atom and float64. Facets allow search +results to contain summaries of the categories matched in a search, and to +restrict searches to only match against specific categories. + +By default, for struct pointers, all of the struct fields are used as document +fields, and the field name used is the same as on the struct (and hence must +start with an upper case letter). Struct fields may have a +`search:"name,options"` tag. The name must start with a letter and be +composed only of word characters. A "-" tag name means that the field will be +ignored. If options is "facet" then the struct field will be used as a +document facet. If options is "" then the comma may be omitted. There are no +other recognized options. + +Example code: + + // A and B are renamed to a and b. + // A, C and I are facets. + // D's tag is equivalent to having no tag at all (E). + // F and G are ignored entirely by the search package. + // I has tag information for both the search and json packages. + type TaggedStruct struct { + A float64 `search:"a,facet"` + B float64 `search:"b"` + C float64 `search:",facet"` + D float64 `search:""` + E float64 + F float64 `search:"-"` + G float64 `search:"-,facet"` + I float64 `search:",facet" json:"i"` + } + + +The FieldLoadSaver Interface + +A document's contents can also be represented by any type that implements the +FieldLoadSaver interface. This type may be a struct pointer, but it +does not have to be. The search package will call Load when loading the +document's contents, and Save when saving them. In addition to a slice of +Fields, the Load and Save methods also use the DocumentMetadata type to +provide additional information about a document (such as its Rank, or set of +Facets). Possible uses for this interface include deriving non-stored fields, +verifying fields or setting specific languages for string and HTML fields. + +Example code: + + type CustomFieldsExample struct { + // Item's title and which language it is in. + Title string + Lang string + // Mass, in grams. + Mass int + } + + func (x *CustomFieldsExample) Load(fields []search.Field, meta *search.DocumentMetadata) error { + // Load the title field, failing if any other field is found. + for _, f := range fields { + if f.Name != "title" { + return fmt.Errorf("unknown field %q", f.Name) + } + s, ok := f.Value.(string) + if !ok { + return fmt.Errorf("unsupported type %T for field %q", f.Value, f.Name) + } + x.Title = s + x.Lang = f.Language + } + // Load the mass facet, failing if any other facet is found. + for _, f := range meta.Facets { + if f.Name != "mass" { + return fmt.Errorf("unknown facet %q", f.Name) + } + m, ok := f.Value.(float64) + if !ok { + return fmt.Errorf("unsupported type %T for facet %q", f.Value, f.Name) + } + x.Mass = int(m) + } + return nil + } + + func (x *CustomFieldsExample) Save() ([]search.Field, *search.DocumentMetadata, error) { + fields := []search.Field{ + {Name: "title", Value: x.Title, Language: x.Lang}, + } + meta := &search.DocumentMetadata{ + Facets: { + {Name: "mass", Value: float64(x.Mass)}, + }, + } + return fields, meta, nil + } +*/ +package search diff --git a/vendor/google.golang.org/appengine/search/field.go b/vendor/google.golang.org/appengine/search/field.go new file mode 100644 index 0000000..707c2d8 --- /dev/null +++ b/vendor/google.golang.org/appengine/search/field.go @@ -0,0 +1,82 @@ +// Copyright 2014 Google Inc. All rights reserved. +// Use of this source code is governed by the Apache 2.0 +// license that can be found in the LICENSE file. + +package search + +// Field is a name/value pair. A search index's document can be loaded and +// saved as a sequence of Fields. +type Field struct { + // Name is the field name. A valid field name matches /[A-Za-z][A-Za-z0-9_]*/. + Name string + // Value is the field value. The valid types are: + // - string, + // - search.Atom, + // - search.HTML, + // - time.Time (stored with millisecond precision), + // - float64, + // - GeoPoint. + Value interface{} + // Language is a two-letter ISO 639-1 code for the field's language, + // defaulting to "en" if nothing is specified. It may only be specified for + // fields of type string and search.HTML. + Language string + // Derived marks fields that were calculated as a result of a + // FieldExpression provided to Search. This field is ignored when saving a + // document. + Derived bool +} + +// Facet is a name/value pair which is used to add categorical information to a +// document. +type Facet struct { + // Name is the facet name. A valid facet name matches /[A-Za-z][A-Za-z0-9_]*/. + // A facet name cannot be longer than 500 characters. + Name string + // Value is the facet value. + // + // When being used in documents (for example, in + // DocumentMetadata.Facets), the valid types are: + // - search.Atom, + // - float64. + // + // When being used in SearchOptions.Refinements or being returned + // in FacetResult, the valid types are: + // - search.Atom, + // - search.Range. + Value interface{} +} + +// DocumentMetadata is a struct containing information describing a given document. +type DocumentMetadata struct { + // Rank is an integer specifying the order the document will be returned in + // search results. If zero, the rank will be set to the number of seconds since + // 2011-01-01 00:00:00 UTC when being Put into an index. + Rank int + // Facets is the set of facets for this document. + Facets []Facet +} + +// FieldLoadSaver can be converted from and to a slice of Fields +// with additional document metadata. +type FieldLoadSaver interface { + Load([]Field, *DocumentMetadata) error + Save() ([]Field, *DocumentMetadata, error) +} + +// FieldList converts a []Field to implement FieldLoadSaver. +type FieldList []Field + +// Load loads all of the provided fields into l. +// It does not first reset *l to an empty slice. +func (l *FieldList) Load(f []Field, _ *DocumentMetadata) error { + *l = append(*l, f...) + return nil +} + +// Save returns all of l's fields as a slice of Fields. +func (l *FieldList) Save() ([]Field, *DocumentMetadata, error) { + return *l, nil, nil +} + +var _ FieldLoadSaver = (*FieldList)(nil) diff --git a/vendor/google.golang.org/appengine/search/search.go b/vendor/google.golang.org/appengine/search/search.go new file mode 100644 index 0000000..774b051 --- /dev/null +++ b/vendor/google.golang.org/appengine/search/search.go @@ -0,0 +1,1121 @@ +// Copyright 2012 Google Inc. All rights reserved. +// Use of this source code is governed by the Apache 2.0 +// license that can be found in the LICENSE file. + +package search // import "google.golang.org/appengine/search" + +// TODO: let Put specify the document language: "en", "fr", etc. Also: order_id?? storage?? +// TODO: Index.GetAll (or Iterator.GetAll)? +// TODO: struct <-> protobuf tests. +// TODO: enforce Python's MIN_NUMBER_VALUE and MIN_DATE (which would disallow a zero +// time.Time)? _MAXIMUM_STRING_LENGTH? + +import ( + "errors" + "fmt" + "math" + "reflect" + "regexp" + "strconv" + "strings" + "time" + "unicode/utf8" + + "github.com/golang/protobuf/proto" + "golang.org/x/net/context" + + "google.golang.org/appengine" + "google.golang.org/appengine/internal" + pb "google.golang.org/appengine/internal/search" +) + +var ( + // ErrInvalidDocumentType is returned when methods like Put, Get or Next + // are passed a dst or src argument of invalid type. + ErrInvalidDocumentType = errors.New("search: invalid document type") + + // ErrNoSuchDocument is returned when no document was found for a given ID. + ErrNoSuchDocument = errors.New("search: no such document") +) + +// Atom is a document field whose contents are indexed as a single indivisible +// string. +type Atom string + +// HTML is a document field whose contents are indexed as HTML. Only text nodes +// are indexed: "foo<b>bar" will be treated as "foobar". +type HTML string + +// validIndexNameOrDocID is the Go equivalent of Python's +// _ValidateVisiblePrintableAsciiNotReserved. +func validIndexNameOrDocID(s string) bool { + if strings.HasPrefix(s, "!") { + return false + } + for _, c := range s { + if c < 0x21 || 0x7f <= c { + return false + } + } + return true +} + +var ( + fieldNameRE = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9_]*$`) + languageRE = regexp.MustCompile(`^[a-z]{2}$`) +) + +// validFieldName is the Go equivalent of Python's _CheckFieldName. It checks +// the validity of both field and facet names. +func validFieldName(s string) bool { + return len(s) <= 500 && fieldNameRE.MatchString(s) +} + +// validDocRank checks that the ranks is in the range [0, 2^31). +func validDocRank(r int) bool { + return 0 <= r && r <= (1<<31-1) +} + +// validLanguage checks that a language looks like ISO 639-1. +func validLanguage(s string) bool { + return languageRE.MatchString(s) +} + +// validFloat checks that f is in the range [-2147483647, 2147483647]. +func validFloat(f float64) bool { + return -(1<<31-1) <= f && f <= (1<<31-1) +} + +// Index is an index of documents. +type Index struct { + spec pb.IndexSpec +} + +// orderIDEpoch forms the basis for populating OrderId on documents. +var orderIDEpoch = time.Date(2011, 1, 1, 0, 0, 0, 0, time.UTC) + +// Open opens the index with the given name. The index is created if it does +// not already exist. +// +// The name is a human-readable ASCII string. It must contain no whitespace +// characters and not start with "!". +func Open(name string) (*Index, error) { + if !validIndexNameOrDocID(name) { + return nil, fmt.Errorf("search: invalid index name %q", name) + } + return &Index{ + spec: pb.IndexSpec{ + Name: &name, + }, + }, nil +} + +// Put saves src to the index. If id is empty, a new ID is allocated by the +// service and returned. If id is not empty, any existing index entry for that +// ID is replaced. +// +// The ID is a human-readable ASCII string. It must contain no whitespace +// characters and not start with "!". +// +// src must be a non-nil struct pointer or implement the FieldLoadSaver +// interface. +func (x *Index) Put(c context.Context, id string, src interface{}) (string, error) { + d, err := saveDoc(src) + if err != nil { + return "", err + } + if id != "" { + if !validIndexNameOrDocID(id) { + return "", fmt.Errorf("search: invalid ID %q", id) + } + d.Id = proto.String(id) + } + // spec is modified by Call when applying the current Namespace, so copy it to + // avoid retaining the namespace beyond the scope of the Call. + spec := x.spec + req := &pb.IndexDocumentRequest{ + Params: &pb.IndexDocumentParams{ + Document: []*pb.Document{d}, + IndexSpec: &spec, + }, + } + res := &pb.IndexDocumentResponse{} + if err := internal.Call(c, "search", "IndexDocument", req, res); err != nil { + return "", err + } + if len(res.Status) > 0 { + if s := res.Status[0]; s.GetCode() != pb.SearchServiceError_OK { + return "", fmt.Errorf("search: %s: %s", s.GetCode(), s.GetErrorDetail()) + } + } + if len(res.Status) != 1 || len(res.DocId) != 1 { + return "", fmt.Errorf("search: internal error: wrong number of results (%d Statuses, %d DocIDs)", + len(res.Status), len(res.DocId)) + } + return res.DocId[0], nil +} + +// Get loads the document with the given ID into dst. +// +// The ID is a human-readable ASCII string. It must be non-empty, contain no +// whitespace characters and not start with "!". +// +// dst must be a non-nil struct pointer or implement the FieldLoadSaver +// interface. +// +// ErrFieldMismatch is returned when a field is to be loaded into a different +// type than the one it was stored from, or when a field is missing or +// unexported in the destination struct. ErrFieldMismatch is only returned if +// dst is a struct pointer. It is up to the callee to decide whether this error +// is fatal, recoverable, or ignorable. +func (x *Index) Get(c context.Context, id string, dst interface{}) error { + if id == "" || !validIndexNameOrDocID(id) { + return fmt.Errorf("search: invalid ID %q", id) + } + req := &pb.ListDocumentsRequest{ + Params: &pb.ListDocumentsParams{ + IndexSpec: &x.spec, + StartDocId: proto.String(id), + Limit: proto.Int32(1), + }, + } + res := &pb.ListDocumentsResponse{} + if err := internal.Call(c, "search", "ListDocuments", req, res); err != nil { + return err + } + if res.Status == nil || res.Status.GetCode() != pb.SearchServiceError_OK { + return fmt.Errorf("search: %s: %s", res.Status.GetCode(), res.Status.GetErrorDetail()) + } + if len(res.Document) != 1 || res.Document[0].GetId() != id { + return ErrNoSuchDocument + } + return loadDoc(dst, res.Document[0], nil) +} + +// Delete deletes a document from the index. +func (x *Index) Delete(c context.Context, id string) error { + req := &pb.DeleteDocumentRequest{ + Params: &pb.DeleteDocumentParams{ + DocId: []string{id}, + IndexSpec: &x.spec, + }, + } + res := &pb.DeleteDocumentResponse{} + if err := internal.Call(c, "search", "DeleteDocument", req, res); err != nil { + return err + } + if len(res.Status) != 1 { + return fmt.Errorf("search: internal error: wrong number of results (%d)", len(res.Status)) + } + if s := res.Status[0]; s.GetCode() != pb.SearchServiceError_OK { + return fmt.Errorf("search: %s: %s", s.GetCode(), s.GetErrorDetail()) + } + return nil +} + +// List lists all of the documents in an index. The documents are returned in +// increasing ID order. +func (x *Index) List(c context.Context, opts *ListOptions) *Iterator { + t := &Iterator{ + c: c, + index: x, + count: -1, + listInclusive: true, + more: moreList, + } + if opts != nil { + t.listStartID = opts.StartID + t.limit = opts.Limit + t.idsOnly = opts.IDsOnly + } + return t +} + +func moreList(t *Iterator) error { + req := &pb.ListDocumentsRequest{ + Params: &pb.ListDocumentsParams{ + IndexSpec: &t.index.spec, + }, + } + if t.listStartID != "" { + req.Params.StartDocId = &t.listStartID + req.Params.IncludeStartDoc = &t.listInclusive + } + if t.limit > 0 { + req.Params.Limit = proto.Int32(int32(t.limit)) + } + if t.idsOnly { + req.Params.KeysOnly = &t.idsOnly + } + + res := &pb.ListDocumentsResponse{} + if err := internal.Call(t.c, "search", "ListDocuments", req, res); err != nil { + return err + } + if res.Status == nil || res.Status.GetCode() != pb.SearchServiceError_OK { + return fmt.Errorf("search: %s: %s", res.Status.GetCode(), res.Status.GetErrorDetail()) + } + t.listRes = res.Document + t.listStartID, t.listInclusive, t.more = "", false, nil + if len(res.Document) != 0 && t.limit <= 0 { + if id := res.Document[len(res.Document)-1].GetId(); id != "" { + t.listStartID, t.more = id, moreList + } + } + return nil +} + +// ListOptions are the options for listing documents in an index. Passing a nil +// *ListOptions is equivalent to using the default values. +type ListOptions struct { + // StartID is the inclusive lower bound for the ID of the returned + // documents. The zero value means all documents will be returned. + StartID string + + // Limit is the maximum number of documents to return. The zero value + // indicates no limit. + Limit int + + // IDsOnly indicates that only document IDs should be returned for the list + // operation; no document fields are populated. + IDsOnly bool +} + +// Search searches the index for the given query. +func (x *Index) Search(c context.Context, query string, opts *SearchOptions) *Iterator { + t := &Iterator{ + c: c, + index: x, + searchQuery: query, + more: moreSearch, + } + if opts != nil { + if opts.Cursor != "" { + if opts.Offset != 0 { + return errIter("at most one of Cursor and Offset may be specified") + } + t.searchCursor = proto.String(string(opts.Cursor)) + } + t.limit = opts.Limit + t.fields = opts.Fields + t.idsOnly = opts.IDsOnly + t.sort = opts.Sort + t.exprs = opts.Expressions + t.refinements = opts.Refinements + t.facetOpts = opts.Facets + t.searchOffset = opts.Offset + t.countAccuracy = opts.CountAccuracy + } + return t +} + +func moreSearch(t *Iterator) error { + // We use per-result (rather than single/per-page) cursors since this + // lets us return a Cursor for every iterator document. The two cursor + // types are largely interchangeable: a page cursor is the same as the + // last per-result cursor in a given search response. + req := &pb.SearchRequest{ + Params: &pb.SearchParams{ + IndexSpec: &t.index.spec, + Query: &t.searchQuery, + Cursor: t.searchCursor, + CursorType: pb.SearchParams_PER_RESULT.Enum(), + FieldSpec: &pb.FieldSpec{ + Name: t.fields, + }, + }, + } + if t.limit > 0 { + req.Params.Limit = proto.Int32(int32(t.limit)) + } + if t.searchOffset > 0 { + req.Params.Offset = proto.Int32(int32(t.searchOffset)) + t.searchOffset = 0 + } + if t.countAccuracy > 0 { + req.Params.MatchedCountAccuracy = proto.Int32(int32(t.countAccuracy)) + } + if t.idsOnly { + req.Params.KeysOnly = &t.idsOnly + } + if t.sort != nil { + if err := sortToProto(t.sort, req.Params); err != nil { + return err + } + } + if t.refinements != nil { + if err := refinementsToProto(t.refinements, req.Params); err != nil { + return err + } + } + for _, e := range t.exprs { + req.Params.FieldSpec.Expression = append(req.Params.FieldSpec.Expression, &pb.FieldSpec_Expression{ + Name: proto.String(e.Name), + Expression: proto.String(e.Expr), + }) + } + for _, f := range t.facetOpts { + if err := f.setParams(req.Params); err != nil { + return fmt.Errorf("bad FacetSearchOption: %v", err) + } + } + // Don't repeat facet search. + t.facetOpts = nil + + res := &pb.SearchResponse{} + if err := internal.Call(t.c, "search", "Search", req, res); err != nil { + return err + } + if res.Status == nil || res.Status.GetCode() != pb.SearchServiceError_OK { + return fmt.Errorf("search: %s: %s", res.Status.GetCode(), res.Status.GetErrorDetail()) + } + t.searchRes = res.Result + if len(res.FacetResult) > 0 { + t.facetRes = res.FacetResult + } + t.count = int(*res.MatchedCount) + if t.limit > 0 { + t.more = nil + } else { + t.more = moreSearch + } + return nil +} + +// SearchOptions are the options for searching an index. Passing a nil +// *SearchOptions is equivalent to using the default values. +type SearchOptions struct { + // Limit is the maximum number of documents to return. The zero value + // indicates no limit. + Limit int + + // IDsOnly indicates that only document IDs should be returned for the search + // operation; no document fields are populated. + IDsOnly bool + + // Sort controls the ordering of search results. + Sort *SortOptions + + // Fields specifies which document fields to include in the results. If omitted, + // all document fields are returned. No more than 100 fields may be specified. + Fields []string + + // Expressions specifies additional computed fields to add to each returned + // document. + Expressions []FieldExpression + + // Facets controls what facet information is returned for these search results. + // If no options are specified, no facet results will be returned. + Facets []FacetSearchOption + + // Refinements filters the returned documents by requiring them to contain facets + // with specific values. Refinements are applied in conjunction for facets with + // different names, and in disjunction otherwise. + Refinements []Facet + + // Cursor causes the results to commence with the first document after + // the document associated with the cursor. + Cursor Cursor + + // Offset specifies the number of documents to skip over before returning results. + // When specified, Cursor must be nil. + Offset int + + // CountAccuracy specifies the maximum result count that can be expected to + // be accurate. If zero, the count accuracy defaults to 20. + CountAccuracy int +} + +// Cursor represents an iterator's position. +// +// The string value of a cursor is web-safe. It can be saved and restored +// for later use. +type Cursor string + +// FieldExpression defines a custom expression to evaluate for each result. +type FieldExpression struct { + // Name is the name to use for the computed field. + Name string + + // Expr is evaluated to provide a custom content snippet for each document. + // See https://cloud.google.com/appengine/docs/go/search/options for + // the supported expression syntax. + Expr string +} + +// FacetSearchOption controls what facet information is returned in search results. +type FacetSearchOption interface { + setParams(*pb.SearchParams) error +} + +// AutoFacetDiscovery returns a FacetSearchOption which enables automatic facet +// discovery for the search. Automatic facet discovery looks for the facets +// which appear the most often in the aggregate in the matched documents. +// +// The maximum number of facets returned is controlled by facetLimit, and the +// maximum number of values per facet by facetLimit. A limit of zero indicates +// a default limit should be used. +func AutoFacetDiscovery(facetLimit, valueLimit int) FacetSearchOption { + return &autoFacetOpt{facetLimit, valueLimit} +} + +type autoFacetOpt struct { + facetLimit, valueLimit int +} + +const defaultAutoFacetLimit = 10 // As per python runtime search.py. + +func (o *autoFacetOpt) setParams(params *pb.SearchParams) error { + lim := int32(o.facetLimit) + if lim == 0 { + lim = defaultAutoFacetLimit + } + params.AutoDiscoverFacetCount = &lim + if o.valueLimit > 0 { + params.FacetAutoDetectParam = &pb.FacetAutoDetectParam{ + ValueLimit: proto.Int32(int32(o.valueLimit)), + } + } + return nil +} + +// FacetDiscovery returns a FacetSearchOption which selects a facet to be +// returned with the search results. By default, the most frequently +// occurring values for that facet will be returned. However, you can also +// specify a list of particular Atoms or specific Ranges to return. +func FacetDiscovery(name string, value ...interface{}) FacetSearchOption { + return &facetOpt{name, value} +} + +type facetOpt struct { + name string + values []interface{} +} + +func (o *facetOpt) setParams(params *pb.SearchParams) error { + req := &pb.FacetRequest{Name: &o.name} + params.IncludeFacet = append(params.IncludeFacet, req) + if len(o.values) == 0 { + return nil + } + vtype := reflect.TypeOf(o.values[0]) + reqParam := &pb.FacetRequestParam{} + for _, v := range o.values { + if reflect.TypeOf(v) != vtype { + return errors.New("values must all be Atom, or must all be Range") + } + switch v := v.(type) { + case Atom: + reqParam.ValueConstraint = append(reqParam.ValueConstraint, string(v)) + case Range: + rng, err := rangeToProto(v) + if err != nil { + return fmt.Errorf("invalid range: %v", err) + } + reqParam.Range = append(reqParam.Range, rng) + default: + return fmt.Errorf("unsupported value type %T", v) + } + } + req.Params = reqParam + return nil +} + +// FacetDocumentDepth returns a FacetSearchOption which controls the number of +// documents to be evaluated with preparing facet results. +func FacetDocumentDepth(depth int) FacetSearchOption { + return facetDepthOpt(depth) +} + +type facetDepthOpt int + +func (o facetDepthOpt) setParams(params *pb.SearchParams) error { + params.FacetDepth = proto.Int32(int32(o)) + return nil +} + +// FacetResult represents the number of times a particular facet and value +// appeared in the documents matching a search request. +type FacetResult struct { + Facet + + // Count is the number of times this specific facet and value appeared in the + // matching documents. + Count int +} + +// Range represents a numeric range with inclusive start and exclusive end. +// Start may be specified as math.Inf(-1) to indicate there is no minimum +// value, and End may similarly be specified as math.Inf(1); at least one of +// Start or End must be a finite number. +type Range struct { + Start, End float64 +} + +var ( + negInf = math.Inf(-1) + posInf = math.Inf(1) +) + +// AtLeast returns a Range matching any value greater than, or equal to, min. +func AtLeast(min float64) Range { + return Range{Start: min, End: posInf} +} + +// LessThan returns a Range matching any value less than max. +func LessThan(max float64) Range { + return Range{Start: negInf, End: max} +} + +// SortOptions control the ordering and scoring of search results. +type SortOptions struct { + // Expressions is a slice of expressions representing a multi-dimensional + // sort. + Expressions []SortExpression + + // Scorer, when specified, will cause the documents to be scored according to + // search term frequency. + Scorer Scorer + + // Limit is the maximum number of objects to score and/or sort. Limit cannot + // be more than 10,000. The zero value indicates a default limit. + Limit int +} + +// SortExpression defines a single dimension for sorting a document. +type SortExpression struct { + // Expr is evaluated to provide a sorting value for each document. + // See https://cloud.google.com/appengine/docs/go/search/options for + // the supported expression syntax. + Expr string + + // Reverse causes the documents to be sorted in ascending order. + Reverse bool + + // The default value to use when no field is present or the expresion + // cannot be calculated for a document. For text sorts, Default must + // be of type string; for numeric sorts, float64. + Default interface{} +} + +// A Scorer defines how a document is scored. +type Scorer interface { + toProto(*pb.ScorerSpec) +} + +type enumScorer struct { + enum pb.ScorerSpec_Scorer +} + +func (e enumScorer) toProto(spec *pb.ScorerSpec) { + spec.Scorer = e.enum.Enum() +} + +var ( + // MatchScorer assigns a score based on term frequency in a document. + MatchScorer Scorer = enumScorer{pb.ScorerSpec_MATCH_SCORER} + + // RescoringMatchScorer assigns a score based on the quality of the query + // match. It is similar to a MatchScorer but uses a more complex scoring + // algorithm based on match term frequency and other factors like field type. + // Please be aware that this algorithm is continually refined and can change + // over time without notice. This means that the ordering of search results + // that use this scorer can also change without notice. + RescoringMatchScorer Scorer = enumScorer{pb.ScorerSpec_RESCORING_MATCH_SCORER} +) + +func sortToProto(sort *SortOptions, params *pb.SearchParams) error { + for _, e := range sort.Expressions { + spec := &pb.SortSpec{ + SortExpression: proto.String(e.Expr), + } + if e.Reverse { + spec.SortDescending = proto.Bool(false) + } + if e.Default != nil { + switch d := e.Default.(type) { + case float64: + spec.DefaultValueNumeric = &d + case string: + spec.DefaultValueText = &d + default: + return fmt.Errorf("search: invalid Default type %T for expression %q", d, e.Expr) + } + } + params.SortSpec = append(params.SortSpec, spec) + } + + spec := &pb.ScorerSpec{} + if sort.Limit > 0 { + spec.Limit = proto.Int32(int32(sort.Limit)) + params.ScorerSpec = spec + } + if sort.Scorer != nil { + sort.Scorer.toProto(spec) + params.ScorerSpec = spec + } + + return nil +} + +func refinementsToProto(refinements []Facet, params *pb.SearchParams) error { + for _, r := range refinements { + ref := &pb.FacetRefinement{ + Name: proto.String(r.Name), + } + switch v := r.Value.(type) { + case Atom: + ref.Value = proto.String(string(v)) + case Range: + rng, err := rangeToProto(v) + if err != nil { + return fmt.Errorf("search: refinement for facet %q: %v", r.Name, err) + } + // Unfortunately there are two identical messages for identify Facet ranges. + ref.Range = &pb.FacetRefinement_Range{Start: rng.Start, End: rng.End} + default: + return fmt.Errorf("search: unsupported refinement for facet %q of type %T", r.Name, v) + } + params.FacetRefinement = append(params.FacetRefinement, ref) + } + return nil +} + +func rangeToProto(r Range) (*pb.FacetRange, error) { + rng := &pb.FacetRange{} + if r.Start != negInf { + if !validFloat(r.Start) { + return nil, errors.New("invalid value for Start") + } + rng.Start = proto.String(strconv.FormatFloat(r.Start, 'e', -1, 64)) + } else if r.End == posInf { + return nil, errors.New("either Start or End must be finite") + } + if r.End != posInf { + if !validFloat(r.End) { + return nil, errors.New("invalid value for End") + } + rng.End = proto.String(strconv.FormatFloat(r.End, 'e', -1, 64)) + } + return rng, nil +} + +func protoToRange(rng *pb.FacetRefinement_Range) Range { + r := Range{Start: negInf, End: posInf} + if x, err := strconv.ParseFloat(rng.GetStart(), 64); err != nil { + r.Start = x + } + if x, err := strconv.ParseFloat(rng.GetEnd(), 64); err != nil { + r.End = x + } + return r +} + +// Iterator is the result of searching an index for a query or listing an +// index. +type Iterator struct { + c context.Context + index *Index + err error + + listRes []*pb.Document + listStartID string + listInclusive bool + + searchRes []*pb.SearchResult + facetRes []*pb.FacetResult + searchQuery string + searchCursor *string + searchOffset int + sort *SortOptions + + fields []string + exprs []FieldExpression + refinements []Facet + facetOpts []FacetSearchOption + + more func(*Iterator) error + + count int + countAccuracy int + limit int // items left to return; 0 for unlimited. + idsOnly bool +} + +// errIter returns an iterator that only returns the given error. +func errIter(err string) *Iterator { + return &Iterator{ + err: errors.New(err), + } +} + +// Done is returned when a query iteration has completed. +var Done = errors.New("search: query has no more results") + +// Count returns an approximation of the number of documents matched by the +// query. It is only valid to call for iterators returned by Search. +func (t *Iterator) Count() int { return t.count } + +// fetchMore retrieves more results, if there are no errors or pending results. +func (t *Iterator) fetchMore() { + if t.err == nil && len(t.listRes)+len(t.searchRes) == 0 && t.more != nil { + t.err = t.more(t) + } +} + +// Next returns the ID of the next result. When there are no more results, +// Done is returned as the error. +// +// dst must be a non-nil struct pointer, implement the FieldLoadSaver +// interface, or be a nil interface value. If a non-nil dst is provided, it +// will be filled with the indexed fields. dst is ignored if this iterator was +// created with an IDsOnly option. +func (t *Iterator) Next(dst interface{}) (string, error) { + t.fetchMore() + if t.err != nil { + return "", t.err + } + + var doc *pb.Document + var exprs []*pb.Field + switch { + case len(t.listRes) != 0: + doc = t.listRes[0] + t.listRes = t.listRes[1:] + case len(t.searchRes) != 0: + doc = t.searchRes[0].Document + exprs = t.searchRes[0].Expression + t.searchCursor = t.searchRes[0].Cursor + t.searchRes = t.searchRes[1:] + default: + return "", Done + } + if doc == nil { + return "", errors.New("search: internal error: no document returned") + } + if !t.idsOnly && dst != nil { + if err := loadDoc(dst, doc, exprs); err != nil { + return "", err + } + } + return doc.GetId(), nil +} + +// Cursor returns the cursor associated with the current document (that is, +// the document most recently returned by a call to Next). +// +// Passing this cursor in a future call to Search will cause those results +// to commence with the first document after the current document. +func (t *Iterator) Cursor() Cursor { + if t.searchCursor == nil { + return "" + } + return Cursor(*t.searchCursor) +} + +// Facets returns the facets found within the search results, if any facets +// were requested in the SearchOptions. +func (t *Iterator) Facets() ([][]FacetResult, error) { + t.fetchMore() + if t.err != nil && t.err != Done { + return nil, t.err + } + + var facets [][]FacetResult + for _, f := range t.facetRes { + fres := make([]FacetResult, 0, len(f.Value)) + for _, v := range f.Value { + ref := v.Refinement + facet := FacetResult{ + Facet: Facet{Name: ref.GetName()}, + Count: int(v.GetCount()), + } + if ref.Value != nil { + facet.Value = Atom(*ref.Value) + } else { + facet.Value = protoToRange(ref.Range) + } + fres = append(fres, facet) + } + facets = append(facets, fres) + } + return facets, nil +} + +// saveDoc converts from a struct pointer or +// FieldLoadSaver/FieldMetadataLoadSaver to the Document protobuf. +func saveDoc(src interface{}) (*pb.Document, error) { + var err error + var fields []Field + var meta *DocumentMetadata + switch x := src.(type) { + case FieldLoadSaver: + fields, meta, err = x.Save() + default: + fields, meta, err = saveStructWithMeta(src) + } + if err != nil { + return nil, err + } + + fieldsProto, err := fieldsToProto(fields) + if err != nil { + return nil, err + } + d := &pb.Document{ + Field: fieldsProto, + OrderId: proto.Int32(int32(time.Since(orderIDEpoch).Seconds())), + } + if meta != nil { + if meta.Rank != 0 { + if !validDocRank(meta.Rank) { + return nil, fmt.Errorf("search: invalid rank %d, must be [0, 2^31)", meta.Rank) + } + *d.OrderId = int32(meta.Rank) + } + if len(meta.Facets) > 0 { + facets, err := facetsToProto(meta.Facets) + if err != nil { + return nil, err + } + d.Facet = facets + } + } + return d, nil +} + +func fieldsToProto(src []Field) ([]*pb.Field, error) { + // Maps to catch duplicate time or numeric fields. + timeFields, numericFields := make(map[string]bool), make(map[string]bool) + dst := make([]*pb.Field, 0, len(src)) + for _, f := range src { + if !validFieldName(f.Name) { + return nil, fmt.Errorf("search: invalid field name %q", f.Name) + } + fieldValue := &pb.FieldValue{} + switch x := f.Value.(type) { + case string: + fieldValue.Type = pb.FieldValue_TEXT.Enum() + fieldValue.StringValue = proto.String(x) + case Atom: + fieldValue.Type = pb.FieldValue_ATOM.Enum() + fieldValue.StringValue = proto.String(string(x)) + case HTML: + fieldValue.Type = pb.FieldValue_HTML.Enum() + fieldValue.StringValue = proto.String(string(x)) + case time.Time: + if timeFields[f.Name] { + return nil, fmt.Errorf("search: duplicate time field %q", f.Name) + } + timeFields[f.Name] = true + fieldValue.Type = pb.FieldValue_DATE.Enum() + fieldValue.StringValue = proto.String(strconv.FormatInt(x.UnixNano()/1e6, 10)) + case float64: + if numericFields[f.Name] { + return nil, fmt.Errorf("search: duplicate numeric field %q", f.Name) + } + if !validFloat(x) { + return nil, fmt.Errorf("search: numeric field %q with invalid value %f", f.Name, x) + } + numericFields[f.Name] = true + fieldValue.Type = pb.FieldValue_NUMBER.Enum() + fieldValue.StringValue = proto.String(strconv.FormatFloat(x, 'e', -1, 64)) + case appengine.GeoPoint: + if !x.Valid() { + return nil, fmt.Errorf( + "search: GeoPoint field %q with invalid value %v", + f.Name, x) + } + fieldValue.Type = pb.FieldValue_GEO.Enum() + fieldValue.Geo = &pb.FieldValue_Geo{ + Lat: proto.Float64(x.Lat), + Lng: proto.Float64(x.Lng), + } + default: + return nil, fmt.Errorf("search: unsupported field type: %v", reflect.TypeOf(f.Value)) + } + if f.Language != "" { + switch f.Value.(type) { + case string, HTML: + if !validLanguage(f.Language) { + return nil, fmt.Errorf("search: invalid language for field %q: %q", f.Name, f.Language) + } + fieldValue.Language = proto.String(f.Language) + default: + return nil, fmt.Errorf("search: setting language not supported for field %q of type %T", f.Name, f.Value) + } + } + if p := fieldValue.StringValue; p != nil && !utf8.ValidString(*p) { + return nil, fmt.Errorf("search: %q field is invalid UTF-8: %q", f.Name, *p) + } + dst = append(dst, &pb.Field{ + Name: proto.String(f.Name), + Value: fieldValue, + }) + } + return dst, nil +} + +func facetsToProto(src []Facet) ([]*pb.Facet, error) { + dst := make([]*pb.Facet, 0, len(src)) + for _, f := range src { + if !validFieldName(f.Name) { + return nil, fmt.Errorf("search: invalid facet name %q", f.Name) + } + facetValue := &pb.FacetValue{} + switch x := f.Value.(type) { + case Atom: + if !utf8.ValidString(string(x)) { + return nil, fmt.Errorf("search: %q facet is invalid UTF-8: %q", f.Name, x) + } + facetValue.Type = pb.FacetValue_ATOM.Enum() + facetValue.StringValue = proto.String(string(x)) + case float64: + if !validFloat(x) { + return nil, fmt.Errorf("search: numeric facet %q with invalid value %f", f.Name, x) + } + facetValue.Type = pb.FacetValue_NUMBER.Enum() + facetValue.StringValue = proto.String(strconv.FormatFloat(x, 'e', -1, 64)) + default: + return nil, fmt.Errorf("search: unsupported facet type: %v", reflect.TypeOf(f.Value)) + } + dst = append(dst, &pb.Facet{ + Name: proto.String(f.Name), + Value: facetValue, + }) + } + return dst, nil +} + +// loadDoc converts from protobufs to a struct pointer or +// FieldLoadSaver/FieldMetadataLoadSaver. The src param provides the document's +// stored fields and facets, and any document metadata. An additional slice of +// fields, exprs, may optionally be provided to contain any derived expressions +// requested by the developer. +func loadDoc(dst interface{}, src *pb.Document, exprs []*pb.Field) (err error) { + fields, err := protoToFields(src.Field) + if err != nil { + return err + } + facets, err := protoToFacets(src.Facet) + if err != nil { + return err + } + if len(exprs) > 0 { + exprFields, err := protoToFields(exprs) + if err != nil { + return err + } + // Mark each field as derived. + for i := range exprFields { + exprFields[i].Derived = true + } + fields = append(fields, exprFields...) + } + meta := &DocumentMetadata{ + Rank: int(src.GetOrderId()), + Facets: facets, + } + switch x := dst.(type) { + case FieldLoadSaver: + return x.Load(fields, meta) + default: + return loadStructWithMeta(dst, fields, meta) + } +} + +func protoToFields(fields []*pb.Field) ([]Field, error) { + dst := make([]Field, 0, len(fields)) + for _, field := range fields { + fieldValue := field.GetValue() + f := Field{ + Name: field.GetName(), + } + switch fieldValue.GetType() { + case pb.FieldValue_TEXT: + f.Value = fieldValue.GetStringValue() + f.Language = fieldValue.GetLanguage() + case pb.FieldValue_ATOM: + f.Value = Atom(fieldValue.GetStringValue()) + case pb.FieldValue_HTML: + f.Value = HTML(fieldValue.GetStringValue()) + f.Language = fieldValue.GetLanguage() + case pb.FieldValue_DATE: + sv := fieldValue.GetStringValue() + millis, err := strconv.ParseInt(sv, 10, 64) + if err != nil { + return nil, fmt.Errorf("search: internal error: bad time.Time encoding %q: %v", sv, err) + } + f.Value = time.Unix(0, millis*1e6) + case pb.FieldValue_NUMBER: + sv := fieldValue.GetStringValue() + x, err := strconv.ParseFloat(sv, 64) + if err != nil { + return nil, err + } + f.Value = x + case pb.FieldValue_GEO: + geoValue := fieldValue.GetGeo() + geoPoint := appengine.GeoPoint{geoValue.GetLat(), geoValue.GetLng()} + if !geoPoint.Valid() { + return nil, fmt.Errorf("search: internal error: invalid GeoPoint encoding: %v", geoPoint) + } + f.Value = geoPoint + default: + return nil, fmt.Errorf("search: internal error: unknown data type %s", fieldValue.GetType()) + } + dst = append(dst, f) + } + return dst, nil +} + +func protoToFacets(facets []*pb.Facet) ([]Facet, error) { + if len(facets) == 0 { + return nil, nil + } + dst := make([]Facet, 0, len(facets)) + for _, facet := range facets { + facetValue := facet.GetValue() + f := Facet{ + Name: facet.GetName(), + } + switch facetValue.GetType() { + case pb.FacetValue_ATOM: + f.Value = Atom(facetValue.GetStringValue()) + case pb.FacetValue_NUMBER: + sv := facetValue.GetStringValue() + x, err := strconv.ParseFloat(sv, 64) + if err != nil { + return nil, err + } + f.Value = x + default: + return nil, fmt.Errorf("search: internal error: unknown data type %s", facetValue.GetType()) + } + dst = append(dst, f) + } + return dst, nil +} + +func namespaceMod(m proto.Message, namespace string) { + set := func(s **string) { + if *s == nil { + *s = &namespace + } + } + switch m := m.(type) { + case *pb.IndexDocumentRequest: + set(&m.Params.IndexSpec.Namespace) + case *pb.ListDocumentsRequest: + set(&m.Params.IndexSpec.Namespace) + case *pb.DeleteDocumentRequest: + set(&m.Params.IndexSpec.Namespace) + case *pb.SearchRequest: + set(&m.Params.IndexSpec.Namespace) + } +} + +func init() { + internal.RegisterErrorCodeMap("search", pb.SearchServiceError_ErrorCode_name) + internal.NamespaceMods["search"] = namespaceMod +} diff --git a/vendor/google.golang.org/appengine/search/search_test.go b/vendor/google.golang.org/appengine/search/search_test.go new file mode 100644 index 0000000..f7c339b --- /dev/null +++ b/vendor/google.golang.org/appengine/search/search_test.go @@ -0,0 +1,1000 @@ +// Copyright 2012 Google Inc. All rights reserved. +// Use of this source code is governed by the Apache 2.0 +// license that can be found in the LICENSE file. + +package search + +import ( + "errors" + "fmt" + "reflect" + "strings" + "testing" + "time" + + "github.com/golang/protobuf/proto" + + "google.golang.org/appengine" + "google.golang.org/appengine/internal/aetesting" + pb "google.golang.org/appengine/internal/search" +) + +type TestDoc struct { + String string + Atom Atom + HTML HTML + Float float64 + Location appengine.GeoPoint + Time time.Time +} + +type FieldListWithMeta struct { + Fields FieldList + Meta *DocumentMetadata +} + +func (f *FieldListWithMeta) Load(fields []Field, meta *DocumentMetadata) error { + f.Meta = meta + return f.Fields.Load(fields, nil) +} + +func (f *FieldListWithMeta) Save() ([]Field, *DocumentMetadata, error) { + fields, _, err := f.Fields.Save() + return fields, f.Meta, err +} + +// Assert that FieldListWithMeta satisfies FieldLoadSaver +var _ FieldLoadSaver = &FieldListWithMeta{} + +var ( + float = 3.14159 + floatOut = "3.14159e+00" + latitude = 37.3894 + longitude = 122.0819 + testGeo = appengine.GeoPoint{latitude, longitude} + testString = "foo<b>bar" + testTime = time.Unix(1337324400, 0) + testTimeOut = "1337324400000" + searchMeta = &DocumentMetadata{ + Rank: 42, + } + searchDoc = TestDoc{ + String: testString, + Atom: Atom(testString), + HTML: HTML(testString), + Float: float, + Location: testGeo, + Time: testTime, + } + searchFields = FieldList{ + Field{Name: "String", Value: testString}, + Field{Name: "Atom", Value: Atom(testString)}, + Field{Name: "HTML", Value: HTML(testString)}, + Field{Name: "Float", Value: float}, + Field{Name: "Location", Value: testGeo}, + Field{Name: "Time", Value: testTime}, + } + // searchFieldsWithLang is a copy of the searchFields with the Language field + // set on text/HTML Fields. + searchFieldsWithLang = FieldList{} + protoFields = []*pb.Field{ + newStringValueField("String", testString, pb.FieldValue_TEXT), + newStringValueField("Atom", testString, pb.FieldValue_ATOM), + newStringValueField("HTML", testString, pb.FieldValue_HTML), + newStringValueField("Float", floatOut, pb.FieldValue_NUMBER), + { + Name: proto.String("Location"), + Value: &pb.FieldValue{ + Geo: &pb.FieldValue_Geo{ + Lat: proto.Float64(latitude), + Lng: proto.Float64(longitude), + }, + Type: pb.FieldValue_GEO.Enum(), + }, + }, + newStringValueField("Time", testTimeOut, pb.FieldValue_DATE), + } +) + +func init() { + for _, f := range searchFields { + if f.Name == "String" || f.Name == "HTML" { + f.Language = "en" + } + searchFieldsWithLang = append(searchFieldsWithLang, f) + } +} + +func newStringValueField(name, value string, valueType pb.FieldValue_ContentType) *pb.Field { + return &pb.Field{ + Name: proto.String(name), + Value: &pb.FieldValue{ + StringValue: proto.String(value), + Type: valueType.Enum(), + }, + } +} + +func newFacet(name, value string, valueType pb.FacetValue_ContentType) *pb.Facet { + return &pb.Facet{ + Name: proto.String(name), + Value: &pb.FacetValue{ + StringValue: proto.String(value), + Type: valueType.Enum(), + }, + } +} + +func TestValidIndexNameOrDocID(t *testing.T) { + testCases := []struct { + s string + want bool + }{ + {"", true}, + {"!", false}, + {"$", true}, + {"!bad", false}, + {"good!", true}, + {"alsoGood", true}, + {"has spaces", false}, + {"is_inva\xffid_UTF-8", false}, + {"is_non-ASCïI", false}, + {"underscores_are_ok", true}, + } + for _, tc := range testCases { + if got := validIndexNameOrDocID(tc.s); got != tc.want { + t.Errorf("%q: got %v, want %v", tc.s, got, tc.want) + } + } +} + +func TestLoadDoc(t *testing.T) { + got, want := TestDoc{}, searchDoc + if err := loadDoc(&got, &pb.Document{Field: protoFields}, nil); err != nil { + t.Fatalf("loadDoc: %v", err) + } + if got != want { + t.Errorf("loadDoc: got %v, wanted %v", got, want) + } +} + +func TestSaveDoc(t *testing.T) { + got, err := saveDoc(&searchDoc) + if err != nil { + t.Fatalf("saveDoc: %v", err) + } + want := protoFields + if !reflect.DeepEqual(got.Field, want) { + t.Errorf("\ngot %v\nwant %v", got, want) + } +} + +func TestLoadFieldList(t *testing.T) { + var got FieldList + want := searchFieldsWithLang + if err := loadDoc(&got, &pb.Document{Field: protoFields}, nil); err != nil { + t.Fatalf("loadDoc: %v", err) + } + if !reflect.DeepEqual(got, want) { + t.Errorf("\ngot %v\nwant %v", got, want) + } +} + +func TestLangFields(t *testing.T) { + fl := &FieldList{ + {Name: "Foo", Value: "I am English", Language: "en"}, + {Name: "Bar", Value: "私は日本人だ", Language: "jp"}, + } + var got FieldList + doc, err := saveDoc(fl) + if err != nil { + t.Fatalf("saveDoc: %v", err) + } + if err := loadDoc(&got, doc, nil); err != nil { + t.Fatalf("loadDoc: %v", err) + } + if want := fl; !reflect.DeepEqual(&got, want) { + t.Errorf("got %v\nwant %v", got, want) + } +} + +func TestSaveFieldList(t *testing.T) { + got, err := saveDoc(&searchFields) + if err != nil { + t.Fatalf("saveDoc: %v", err) + } + want := protoFields + if !reflect.DeepEqual(got.Field, want) { + t.Errorf("\ngot %v\nwant %v", got, want) + } +} + +func TestLoadFieldAndExprList(t *testing.T) { + var got, want FieldList + for i, f := range searchFieldsWithLang { + f.Derived = (i >= 2) // First 2 elements are "fields", next are "expressions". + want = append(want, f) + } + doc, expr := &pb.Document{Field: protoFields[:2]}, protoFields[2:] + if err := loadDoc(&got, doc, expr); err != nil { + t.Fatalf("loadDoc: %v", err) + } + if !reflect.DeepEqual(got, want) { + t.Errorf("got %v\nwant %v", got, want) + } +} + +func TestLoadMeta(t *testing.T) { + var got FieldListWithMeta + want := FieldListWithMeta{ + Meta: searchMeta, + Fields: searchFieldsWithLang, + } + doc := &pb.Document{ + Field: protoFields, + OrderId: proto.Int32(42), + } + if err := loadDoc(&got, doc, nil); err != nil { + t.Fatalf("loadDoc: %v", err) + } + if !reflect.DeepEqual(got, want) { + t.Errorf("\ngot %v\nwant %v", got, want) + } +} + +func TestSaveMeta(t *testing.T) { + got, err := saveDoc(&FieldListWithMeta{ + Meta: searchMeta, + Fields: searchFields, + }) + if err != nil { + t.Fatalf("saveDoc: %v", err) + } + want := &pb.Document{ + Field: protoFields, + OrderId: proto.Int32(42), + } + if !proto.Equal(got, want) { + t.Errorf("\ngot %v\nwant %v", got, want) + } +} + +func TestLoadSaveWithStruct(t *testing.T) { + type gopher struct { + Name string + Info string `search:"about"` + Legs float64 `search:",facet"` + Fuzz Atom `search:"Fur,facet"` + } + + doc := gopher{"Gopher", "Likes slide rules.", 4, Atom("furry")} + pb := &pb.Document{ + Field: []*pb.Field{ + newStringValueField("Name", "Gopher", pb.FieldValue_TEXT), + newStringValueField("about", "Likes slide rules.", pb.FieldValue_TEXT), + }, + Facet: []*pb.Facet{ + newFacet("Legs", "4e+00", pb.FacetValue_NUMBER), + newFacet("Fur", "furry", pb.FacetValue_ATOM), + }, + } + + var gotDoc gopher + if err := loadDoc(&gotDoc, pb, nil); err != nil { + t.Fatalf("loadDoc: %v", err) + } + if !reflect.DeepEqual(gotDoc, doc) { + t.Errorf("loading doc\ngot %v\nwant %v", gotDoc, doc) + } + + gotPB, err := saveDoc(&doc) + if err != nil { + t.Fatalf("saveDoc: %v", err) + } + gotPB.OrderId = nil // Don't test: it's time dependent. + if !proto.Equal(gotPB, pb) { + t.Errorf("saving doc\ngot %v\nwant %v", gotPB, pb) + } +} + +func TestValidFieldNames(t *testing.T) { + testCases := []struct { + name string + valid bool + }{ + {"Normal", true}, + {"Also_OK_123", true}, + {"Not so great", false}, + {"lower_case", true}, + {"Exclaim!", false}, + {"Hello세상아 안녕", false}, + {"", false}, + {"Hεllo", false}, + {strings.Repeat("A", 500), true}, + {strings.Repeat("A", 501), false}, + } + + for _, tc := range testCases { + _, err := saveDoc(&FieldList{ + Field{Name: tc.name, Value: "val"}, + }) + if err != nil && !strings.Contains(err.Error(), "invalid field name") { + t.Errorf("unexpected err %q for field name %q", err, tc.name) + } + if (err == nil) != tc.valid { + t.Errorf("field %q: expected valid %t, received err %v", tc.name, tc.valid, err) + } + } +} + +func TestValidLangs(t *testing.T) { + testCases := []struct { + field Field + valid bool + }{ + {Field{Name: "Foo", Value: "String", Language: ""}, true}, + {Field{Name: "Foo", Value: "String", Language: "en"}, true}, + {Field{Name: "Foo", Value: "String", Language: "aussie"}, false}, + {Field{Name: "Foo", Value: "String", Language: "12"}, false}, + {Field{Name: "Foo", Value: HTML("String"), Language: "en"}, true}, + {Field{Name: "Foo", Value: Atom("String"), Language: "en"}, false}, + {Field{Name: "Foo", Value: 42, Language: "en"}, false}, + } + + for _, tt := range testCases { + _, err := saveDoc(&FieldList{tt.field}) + if err == nil != tt.valid { + t.Errorf("Field %v, got error %v, wanted valid %t", tt.field, err, tt.valid) + } + } +} + +func TestDuplicateFields(t *testing.T) { + testCases := []struct { + desc string + fields FieldList + errMsg string // Non-empty if we expect an error + }{ + { + desc: "multi string", + fields: FieldList{{Name: "FieldA", Value: "val1"}, {Name: "FieldA", Value: "val2"}, {Name: "FieldA", Value: "val3"}}, + }, + { + desc: "multi atom", + fields: FieldList{{Name: "FieldA", Value: Atom("val1")}, {Name: "FieldA", Value: Atom("val2")}, {Name: "FieldA", Value: Atom("val3")}}, + }, + { + desc: "mixed", + fields: FieldList{{Name: "FieldA", Value: testString}, {Name: "FieldA", Value: testTime}, {Name: "FieldA", Value: float}}, + }, + { + desc: "multi time", + fields: FieldList{{Name: "FieldA", Value: testTime}, {Name: "FieldA", Value: testTime}}, + errMsg: `duplicate time field "FieldA"`, + }, + { + desc: "multi num", + fields: FieldList{{Name: "FieldA", Value: float}, {Name: "FieldA", Value: float}}, + errMsg: `duplicate numeric field "FieldA"`, + }, + } + for _, tc := range testCases { + _, err := saveDoc(&tc.fields) + if (err == nil) != (tc.errMsg == "") || (err != nil && !strings.Contains(err.Error(), tc.errMsg)) { + t.Errorf("%s: got err %v, wanted %q", tc.desc, err, tc.errMsg) + } + } +} + +func TestLoadErrFieldMismatch(t *testing.T) { + testCases := []struct { + desc string + dst interface{} + src []*pb.Field + err error + }{ + { + desc: "missing", + dst: &struct{ One string }{}, + src: []*pb.Field{newStringValueField("Two", "woop!", pb.FieldValue_TEXT)}, + err: &ErrFieldMismatch{ + FieldName: "Two", + Reason: "no such struct field", + }, + }, + { + desc: "wrong type", + dst: &struct{ Num float64 }{}, + src: []*pb.Field{newStringValueField("Num", "woop!", pb.FieldValue_TEXT)}, + err: &ErrFieldMismatch{ + FieldName: "Num", + Reason: "type mismatch: float64 for string data", + }, + }, + { + desc: "unsettable", + dst: &struct{ lower string }{}, + src: []*pb.Field{newStringValueField("lower", "woop!", pb.FieldValue_TEXT)}, + err: &ErrFieldMismatch{ + FieldName: "lower", + Reason: "cannot set struct field", + }, + }, + } + for _, tc := range testCases { + err := loadDoc(tc.dst, &pb.Document{Field: tc.src}, nil) + if !reflect.DeepEqual(err, tc.err) { + t.Errorf("%s, got err %v, wanted %v", tc.desc, err, tc.err) + } + } +} + +func TestLimit(t *testing.T) { + index, err := Open("Doc") + if err != nil { + t.Fatalf("err from Open: %v", err) + } + c := aetesting.FakeSingleContext(t, "search", "Search", func(req *pb.SearchRequest, res *pb.SearchResponse) error { + limit := 20 // Default per page. + if req.Params.Limit != nil { + limit = int(*req.Params.Limit) + } + res.Status = &pb.RequestStatus{Code: pb.SearchServiceError_OK.Enum()} + res.MatchedCount = proto.Int64(int64(limit)) + for i := 0; i < limit; i++ { + res.Result = append(res.Result, &pb.SearchResult{Document: &pb.Document{}}) + res.Cursor = proto.String("moreresults") + } + return nil + }) + + const maxDocs = 500 // Limit maximum number of docs. + testCases := []struct { + limit, want int + }{ + {limit: 0, want: maxDocs}, + {limit: 42, want: 42}, + {limit: 100, want: 100}, + {limit: 1000, want: maxDocs}, + } + + for _, tt := range testCases { + it := index.Search(c, "gopher", &SearchOptions{Limit: tt.limit, IDsOnly: true}) + count := 0 + for ; count < maxDocs; count++ { + _, err := it.Next(nil) + if err == Done { + break + } + if err != nil { + t.Fatalf("err after %d: %v", count, err) + } + } + if count != tt.want { + t.Errorf("got %d results, expected %d", count, tt.want) + } + } +} + +func TestPut(t *testing.T) { + index, err := Open("Doc") + if err != nil { + t.Fatalf("err from Open: %v", err) + } + + c := aetesting.FakeSingleContext(t, "search", "IndexDocument", func(in *pb.IndexDocumentRequest, out *pb.IndexDocumentResponse) error { + expectedIn := &pb.IndexDocumentRequest{ + Params: &pb.IndexDocumentParams{ + Document: []*pb.Document{ + {Field: protoFields, OrderId: proto.Int32(42)}, + }, + IndexSpec: &pb.IndexSpec{ + Name: proto.String("Doc"), + }, + }, + } + if !proto.Equal(in, expectedIn) { + return fmt.Errorf("unsupported argument:\ngot %v\nwant %v", in, expectedIn) + } + *out = pb.IndexDocumentResponse{ + Status: []*pb.RequestStatus{ + {Code: pb.SearchServiceError_OK.Enum()}, + }, + DocId: []string{ + "doc_id", + }, + } + return nil + }) + + id, err := index.Put(c, "", &FieldListWithMeta{ + Meta: searchMeta, + Fields: searchFields, + }) + if err != nil { + t.Fatal(err) + } + if want := "doc_id"; id != want { + t.Errorf("Got doc ID %q, want %q", id, want) + } +} + +func TestPutAutoOrderID(t *testing.T) { + index, err := Open("Doc") + if err != nil { + t.Fatalf("err from Open: %v", err) + } + + c := aetesting.FakeSingleContext(t, "search", "IndexDocument", func(in *pb.IndexDocumentRequest, out *pb.IndexDocumentResponse) error { + if len(in.Params.GetDocument()) < 1 { + return fmt.Errorf("expected at least one Document, got %v", in) + } + got, want := in.Params.Document[0].GetOrderId(), int32(time.Since(orderIDEpoch).Seconds()) + if d := got - want; -5 > d || d > 5 { + return fmt.Errorf("got OrderId %d, want near %d", got, want) + } + *out = pb.IndexDocumentResponse{ + Status: []*pb.RequestStatus{ + {Code: pb.SearchServiceError_OK.Enum()}, + }, + DocId: []string{ + "doc_id", + }, + } + return nil + }) + + if _, err := index.Put(c, "", &searchFields); err != nil { + t.Fatal(err) + } +} + +func TestPutBadStatus(t *testing.T) { + index, err := Open("Doc") + if err != nil { + t.Fatalf("err from Open: %v", err) + } + + c := aetesting.FakeSingleContext(t, "search", "IndexDocument", func(_ *pb.IndexDocumentRequest, out *pb.IndexDocumentResponse) error { + *out = pb.IndexDocumentResponse{ + Status: []*pb.RequestStatus{ + { + Code: pb.SearchServiceError_INVALID_REQUEST.Enum(), + ErrorDetail: proto.String("insufficient gophers"), + }, + }, + } + return nil + }) + + wantErr := "search: INVALID_REQUEST: insufficient gophers" + if _, err := index.Put(c, "", &searchFields); err == nil || err.Error() != wantErr { + t.Fatalf("Put: got %v error, want %q", err, wantErr) + } +} + +func TestSortOptions(t *testing.T) { + index, err := Open("Doc") + if err != nil { + t.Fatalf("err from Open: %v", err) + } + + noErr := errors.New("") // Sentinel err to return to prevent sending request. + + testCases := []struct { + desc string + sort *SortOptions + wantSort []*pb.SortSpec + wantScorer *pb.ScorerSpec + wantErr string + }{ + { + desc: "No SortOptions", + }, + { + desc: "Basic", + sort: &SortOptions{ + Expressions: []SortExpression{ + {Expr: "dog"}, + {Expr: "cat", Reverse: true}, + {Expr: "gopher", Default: "blue"}, + {Expr: "fish", Default: 2.0}, + }, + Limit: 42, + Scorer: MatchScorer, + }, + wantSort: []*pb.SortSpec{ + {SortExpression: proto.String("dog")}, + {SortExpression: proto.String("cat"), SortDescending: proto.Bool(false)}, + {SortExpression: proto.String("gopher"), DefaultValueText: proto.String("blue")}, + {SortExpression: proto.String("fish"), DefaultValueNumeric: proto.Float64(2)}, + }, + wantScorer: &pb.ScorerSpec{ + Limit: proto.Int32(42), + Scorer: pb.ScorerSpec_MATCH_SCORER.Enum(), + }, + }, + { + desc: "Bad expression default", + sort: &SortOptions{ + Expressions: []SortExpression{ + {Expr: "dog", Default: true}, + }, + }, + wantErr: `search: invalid Default type bool for expression "dog"`, + }, + { + desc: "RescoringMatchScorer", + sort: &SortOptions{Scorer: RescoringMatchScorer}, + wantScorer: &pb.ScorerSpec{Scorer: pb.ScorerSpec_RESCORING_MATCH_SCORER.Enum()}, + }, + } + + for _, tt := range testCases { + c := aetesting.FakeSingleContext(t, "search", "Search", func(req *pb.SearchRequest, _ *pb.SearchResponse) error { + params := req.Params + if !reflect.DeepEqual(params.SortSpec, tt.wantSort) { + t.Errorf("%s: params.SortSpec=%v; want %v", tt.desc, params.SortSpec, tt.wantSort) + } + if !reflect.DeepEqual(params.ScorerSpec, tt.wantScorer) { + t.Errorf("%s: params.ScorerSpec=%v; want %v", tt.desc, params.ScorerSpec, tt.wantScorer) + } + return noErr // Always return some error to prevent response parsing. + }) + + it := index.Search(c, "gopher", &SearchOptions{Sort: tt.sort}) + _, err := it.Next(nil) + if err == nil { + t.Fatalf("%s: err==nil; should not happen", tt.desc) + } + if err.Error() != tt.wantErr { + t.Errorf("%s: got error %q, want %q", tt.desc, err, tt.wantErr) + } + } +} + +func TestFieldSpec(t *testing.T) { + index, err := Open("Doc") + if err != nil { + t.Fatalf("err from Open: %v", err) + } + + errFoo := errors.New("foo") // sentinel error when there isn't one. + + testCases := []struct { + desc string + opts *SearchOptions + want *pb.FieldSpec + }{ + { + desc: "No options", + want: &pb.FieldSpec{}, + }, + { + desc: "Fields", + opts: &SearchOptions{ + Fields: []string{"one", "two"}, + }, + want: &pb.FieldSpec{ + Name: []string{"one", "two"}, + }, + }, + { + desc: "Expressions", + opts: &SearchOptions{ + Expressions: []FieldExpression{ + {Name: "one", Expr: "price * quantity"}, + {Name: "two", Expr: "min(daily_use, 10) * rate"}, + }, + }, + want: &pb.FieldSpec{ + Expression: []*pb.FieldSpec_Expression{ + {Name: proto.String("one"), Expression: proto.String("price * quantity")}, + {Name: proto.String("two"), Expression: proto.String("min(daily_use, 10) * rate")}, + }, + }, + }, + } + + for _, tt := range testCases { + c := aetesting.FakeSingleContext(t, "search", "Search", func(req *pb.SearchRequest, _ *pb.SearchResponse) error { + params := req.Params + if !reflect.DeepEqual(params.FieldSpec, tt.want) { + t.Errorf("%s: params.FieldSpec=%v; want %v", tt.desc, params.FieldSpec, tt.want) + } + return errFoo // Always return some error to prevent response parsing. + }) + + it := index.Search(c, "gopher", tt.opts) + if _, err := it.Next(nil); err != errFoo { + t.Fatalf("%s: got error %v; want %v", tt.desc, err, errFoo) + } + } +} + +func TestBasicSearchOpts(t *testing.T) { + index, err := Open("Doc") + if err != nil { + t.Fatalf("err from Open: %v", err) + } + + noErr := errors.New("") // Sentinel err to return to prevent sending request. + + testCases := []struct { + desc string + facetOpts []FacetSearchOption + cursor Cursor + offset int + countAccuracy int + want *pb.SearchParams + wantErr string + }{ + { + desc: "No options", + want: &pb.SearchParams{}, + }, + { + desc: "Default auto discovery", + facetOpts: []FacetSearchOption{ + AutoFacetDiscovery(0, 0), + }, + want: &pb.SearchParams{ + AutoDiscoverFacetCount: proto.Int32(10), + }, + }, + { + desc: "Auto discovery", + facetOpts: []FacetSearchOption{ + AutoFacetDiscovery(7, 12), + }, + want: &pb.SearchParams{ + AutoDiscoverFacetCount: proto.Int32(7), + FacetAutoDetectParam: &pb.FacetAutoDetectParam{ + ValueLimit: proto.Int32(12), + }, + }, + }, + { + desc: "Param Depth", + facetOpts: []FacetSearchOption{ + AutoFacetDiscovery(7, 12), + }, + want: &pb.SearchParams{ + AutoDiscoverFacetCount: proto.Int32(7), + FacetAutoDetectParam: &pb.FacetAutoDetectParam{ + ValueLimit: proto.Int32(12), + }, + }, + }, + { + desc: "Doc depth", + facetOpts: []FacetSearchOption{ + FacetDocumentDepth(123), + }, + want: &pb.SearchParams{ + FacetDepth: proto.Int32(123), + }, + }, + { + desc: "Facet discovery", + facetOpts: []FacetSearchOption{ + FacetDiscovery("colour"), + FacetDiscovery("size", Atom("M"), Atom("L")), + FacetDiscovery("price", LessThan(7), Range{7, 14}, AtLeast(14)), + }, + want: &pb.SearchParams{ + IncludeFacet: []*pb.FacetRequest{ + {Name: proto.String("colour")}, + {Name: proto.String("size"), Params: &pb.FacetRequestParam{ + ValueConstraint: []string{"M", "L"}, + }}, + {Name: proto.String("price"), Params: &pb.FacetRequestParam{ + Range: []*pb.FacetRange{ + {End: proto.String("7e+00")}, + {Start: proto.String("7e+00"), End: proto.String("1.4e+01")}, + {Start: proto.String("1.4e+01")}, + }, + }}, + }, + }, + }, + { + desc: "Facet discovery - bad value", + facetOpts: []FacetSearchOption{ + FacetDiscovery("colour", true), + }, + wantErr: "bad FacetSearchOption: unsupported value type bool", + }, + { + desc: "Facet discovery - mix value types", + facetOpts: []FacetSearchOption{ + FacetDiscovery("colour", Atom("blue"), AtLeast(7)), + }, + wantErr: "bad FacetSearchOption: values must all be Atom, or must all be Range", + }, + { + desc: "Facet discovery - invalid range", + facetOpts: []FacetSearchOption{ + FacetDiscovery("colour", Range{negInf, posInf}), + }, + wantErr: "bad FacetSearchOption: invalid range: either Start or End must be finite", + }, + { + desc: "Cursor", + cursor: Cursor("mycursor"), + want: &pb.SearchParams{ + Cursor: proto.String("mycursor"), + }, + }, + { + desc: "Offset", + offset: 121, + want: &pb.SearchParams{ + Offset: proto.Int32(121), + }, + }, + { + desc: "Cursor and Offset set", + cursor: Cursor("mycursor"), + offset: 121, + wantErr: "at most one of Cursor and Offset may be specified", + }, + { + desc: "Count accuracy", + countAccuracy: 100, + want: &pb.SearchParams{ + MatchedCountAccuracy: proto.Int32(100), + }, + }, + } + + for _, tt := range testCases { + c := aetesting.FakeSingleContext(t, "search", "Search", func(req *pb.SearchRequest, _ *pb.SearchResponse) error { + if tt.want == nil { + t.Errorf("%s: expected call to fail", tt.desc) + return nil + } + // Set default fields. + tt.want.Query = proto.String("gopher") + tt.want.IndexSpec = &pb.IndexSpec{Name: proto.String("Doc")} + tt.want.CursorType = pb.SearchParams_PER_RESULT.Enum() + tt.want.FieldSpec = &pb.FieldSpec{} + if got := req.Params; !reflect.DeepEqual(got, tt.want) { + t.Errorf("%s: params=%v; want %v", tt.desc, got, tt.want) + } + return noErr // Always return some error to prevent response parsing. + }) + + it := index.Search(c, "gopher", &SearchOptions{ + Facets: tt.facetOpts, + Cursor: tt.cursor, + Offset: tt.offset, + CountAccuracy: tt.countAccuracy, + }) + _, err := it.Next(nil) + if err == nil { + t.Fatalf("%s: err==nil; should not happen", tt.desc) + } + if err.Error() != tt.wantErr { + t.Errorf("%s: got error %q, want %q", tt.desc, err, tt.wantErr) + } + } +} + +func TestFacetRefinements(t *testing.T) { + index, err := Open("Doc") + if err != nil { + t.Fatalf("err from Open: %v", err) + } + + noErr := errors.New("") // Sentinel err to return to prevent sending request. + + testCases := []struct { + desc string + refine []Facet + want []*pb.FacetRefinement + wantErr string + }{ + { + desc: "No refinements", + }, + { + desc: "Basic", + refine: []Facet{ + {Name: "fur", Value: Atom("fluffy")}, + {Name: "age", Value: LessThan(123)}, + {Name: "age", Value: AtLeast(0)}, + {Name: "legs", Value: Range{Start: 3, End: 5}}, + }, + want: []*pb.FacetRefinement{ + {Name: proto.String("fur"), Value: proto.String("fluffy")}, + {Name: proto.String("age"), Range: &pb.FacetRefinement_Range{End: proto.String("1.23e+02")}}, + {Name: proto.String("age"), Range: &pb.FacetRefinement_Range{Start: proto.String("0e+00")}}, + {Name: proto.String("legs"), Range: &pb.FacetRefinement_Range{Start: proto.String("3e+00"), End: proto.String("5e+00")}}, + }, + }, + { + desc: "Infinite range", + refine: []Facet{ + {Name: "age", Value: Range{Start: negInf, End: posInf}}, + }, + wantErr: `search: refinement for facet "age": either Start or End must be finite`, + }, + { + desc: "Bad End value in range", + refine: []Facet{ + {Name: "age", Value: LessThan(2147483648)}, + }, + wantErr: `search: refinement for facet "age": invalid value for End`, + }, + { + desc: "Bad Start value in range", + refine: []Facet{ + {Name: "age", Value: AtLeast(-2147483649)}, + }, + wantErr: `search: refinement for facet "age": invalid value for Start`, + }, + { + desc: "Unknown value type", + refine: []Facet{ + {Name: "age", Value: "you can't use strings!"}, + }, + wantErr: `search: unsupported refinement for facet "age" of type string`, + }, + } + + for _, tt := range testCases { + c := aetesting.FakeSingleContext(t, "search", "Search", func(req *pb.SearchRequest, _ *pb.SearchResponse) error { + if got := req.Params.FacetRefinement; !reflect.DeepEqual(got, tt.want) { + t.Errorf("%s: params.FacetRefinement=%v; want %v", tt.desc, got, tt.want) + } + return noErr // Always return some error to prevent response parsing. + }) + + it := index.Search(c, "gopher", &SearchOptions{Refinements: tt.refine}) + _, err := it.Next(nil) + if err == nil { + t.Fatalf("%s: err==nil; should not happen", tt.desc) + } + if err.Error() != tt.wantErr { + t.Errorf("%s: got error %q, want %q", tt.desc, err, tt.wantErr) + } + } +} + +func TestNamespaceResetting(t *testing.T) { + namec := make(chan *string, 1) + c0 := aetesting.FakeSingleContext(t, "search", "IndexDocument", func(req *pb.IndexDocumentRequest, res *pb.IndexDocumentResponse) error { + namec <- req.Params.IndexSpec.Namespace + return fmt.Errorf("RPC error") + }) + + // Check that wrapping c0 in a namespace twice works correctly. + c1, err := appengine.Namespace(c0, "A") + if err != nil { + t.Fatalf("appengine.Namespace: %v", err) + } + c2, err := appengine.Namespace(c1, "") // should act as the original context + if err != nil { + t.Fatalf("appengine.Namespace: %v", err) + } + + i := (&Index{}) + + i.Put(c0, "something", &searchDoc) + if ns := <-namec; ns != nil { + t.Errorf(`Put with c0: ns = %q, want nil`, *ns) + } + + i.Put(c1, "something", &searchDoc) + if ns := <-namec; ns == nil { + t.Error(`Put with c1: ns = nil, want "A"`) + } else if *ns != "A" { + t.Errorf(`Put with c1: ns = %q, want "A"`, *ns) + } + + i.Put(c2, "something", &searchDoc) + if ns := <-namec; ns != nil { + t.Errorf(`Put with c2: ns = %q, want nil`, *ns) + } +} diff --git a/vendor/google.golang.org/appengine/search/struct.go b/vendor/google.golang.org/appengine/search/struct.go new file mode 100644 index 0000000..e73d2f2 --- /dev/null +++ b/vendor/google.golang.org/appengine/search/struct.go @@ -0,0 +1,251 @@ +// Copyright 2015 Google Inc. All rights reserved. +// Use of this source code is governed by the Apache 2.0 +// license that can be found in the LICENSE file. + +package search + +import ( + "fmt" + "reflect" + "strings" + "sync" +) + +// ErrFieldMismatch is returned when a field is to be loaded into a different +// than the one it was stored from, or when a field is missing or unexported in +// the destination struct. +type ErrFieldMismatch struct { + FieldName string + Reason string +} + +func (e *ErrFieldMismatch) Error() string { + return fmt.Sprintf("search: cannot load field %q: %s", e.FieldName, e.Reason) +} + +// ErrFacetMismatch is returned when a facet is to be loaded into a different +// type than the one it was stored from, or when a field is missing or +// unexported in the destination struct. StructType is the type of the struct +// pointed to by the destination argument passed to Iterator.Next. +type ErrFacetMismatch struct { + StructType reflect.Type + FacetName string + Reason string +} + +func (e *ErrFacetMismatch) Error() string { + return fmt.Sprintf("search: cannot load facet %q into a %q: %s", e.FacetName, e.StructType, e.Reason) +} + +// structCodec defines how to convert a given struct to/from a search document. +type structCodec struct { + // byIndex returns the struct tag for the i'th struct field. + byIndex []structTag + + // fieldByName returns the index of the struct field for the given field name. + fieldByName map[string]int + + // facetByName returns the index of the struct field for the given facet name, + facetByName map[string]int +} + +// structTag holds a structured version of each struct field's parsed tag. +type structTag struct { + name string + facet bool + ignore bool +} + +var ( + codecsMu sync.RWMutex + codecs = map[reflect.Type]*structCodec{} +) + +func loadCodec(t reflect.Type) (*structCodec, error) { + codecsMu.RLock() + codec, ok := codecs[t] + codecsMu.RUnlock() + if ok { + return codec, nil + } + + codecsMu.Lock() + defer codecsMu.Unlock() + if codec, ok := codecs[t]; ok { + return codec, nil + } + + codec = &structCodec{ + fieldByName: make(map[string]int), + facetByName: make(map[string]int), + } + + for i, I := 0, t.NumField(); i < I; i++ { + f := t.Field(i) + name, opts := f.Tag.Get("search"), "" + if i := strings.Index(name, ","); i != -1 { + name, opts = name[:i], name[i+1:] + } + ignore := false + if name == "-" { + ignore = true + } else if name == "" { + name = f.Name + } else if !validFieldName(name) { + return nil, fmt.Errorf("search: struct tag has invalid field name: %q", name) + } + facet := opts == "facet" + codec.byIndex = append(codec.byIndex, structTag{name: name, facet: facet, ignore: ignore}) + if facet { + codec.facetByName[name] = i + } else { + codec.fieldByName[name] = i + } + } + + codecs[t] = codec + return codec, nil +} + +// structFLS adapts a struct to be a FieldLoadSaver. +type structFLS struct { + v reflect.Value + codec *structCodec +} + +func (s structFLS) Load(fields []Field, meta *DocumentMetadata) error { + var err error + for _, field := range fields { + i, ok := s.codec.fieldByName[field.Name] + if !ok { + // Note the error, but keep going. + err = &ErrFieldMismatch{ + FieldName: field.Name, + Reason: "no such struct field", + } + continue + + } + f := s.v.Field(i) + if !f.CanSet() { + // Note the error, but keep going. + err = &ErrFieldMismatch{ + FieldName: field.Name, + Reason: "cannot set struct field", + } + continue + } + v := reflect.ValueOf(field.Value) + if ft, vt := f.Type(), v.Type(); ft != vt { + err = &ErrFieldMismatch{ + FieldName: field.Name, + Reason: fmt.Sprintf("type mismatch: %v for %v data", ft, vt), + } + continue + } + f.Set(v) + } + if meta == nil { + return err + } + for _, facet := range meta.Facets { + i, ok := s.codec.facetByName[facet.Name] + if !ok { + // Note the error, but keep going. + if err == nil { + err = &ErrFacetMismatch{ + StructType: s.v.Type(), + FacetName: facet.Name, + Reason: "no matching field found", + } + } + continue + } + f := s.v.Field(i) + if !f.CanSet() { + // Note the error, but keep going. + if err == nil { + err = &ErrFacetMismatch{ + StructType: s.v.Type(), + FacetName: facet.Name, + Reason: "unable to set unexported field of struct", + } + } + continue + } + v := reflect.ValueOf(facet.Value) + if ft, vt := f.Type(), v.Type(); ft != vt { + if err == nil { + err = &ErrFacetMismatch{ + StructType: s.v.Type(), + FacetName: facet.Name, + Reason: fmt.Sprintf("type mismatch: %v for %d data", ft, vt), + } + continue + } + } + f.Set(v) + } + return err +} + +func (s structFLS) Save() ([]Field, *DocumentMetadata, error) { + fields := make([]Field, 0, len(s.codec.fieldByName)) + var facets []Facet + for i, tag := range s.codec.byIndex { + if tag.ignore { + continue + } + f := s.v.Field(i) + if !f.CanSet() { + continue + } + if tag.facet { + facets = append(facets, Facet{Name: tag.name, Value: f.Interface()}) + } else { + fields = append(fields, Field{Name: tag.name, Value: f.Interface()}) + } + } + return fields, &DocumentMetadata{Facets: facets}, nil +} + +// newStructFLS returns a FieldLoadSaver for the struct pointer p. +func newStructFLS(p interface{}) (FieldLoadSaver, error) { + v := reflect.ValueOf(p) + if v.Kind() != reflect.Ptr || v.IsNil() || v.Elem().Kind() != reflect.Struct { + return nil, ErrInvalidDocumentType + } + codec, err := loadCodec(v.Elem().Type()) + if err != nil { + return nil, err + } + return structFLS{v.Elem(), codec}, nil +} + +func loadStructWithMeta(dst interface{}, f []Field, meta *DocumentMetadata) error { + x, err := newStructFLS(dst) + if err != nil { + return err + } + return x.Load(f, meta) +} + +func saveStructWithMeta(src interface{}) ([]Field, *DocumentMetadata, error) { + x, err := newStructFLS(src) + if err != nil { + return nil, nil, err + } + return x.Save() +} + +// LoadStruct loads the fields from f to dst. dst must be a struct pointer. +func LoadStruct(dst interface{}, f []Field) error { + return loadStructWithMeta(dst, f, nil) +} + +// SaveStruct returns the fields from src as a slice of Field. +// src must be a struct pointer. +func SaveStruct(src interface{}) ([]Field, error) { + f, _, err := saveStructWithMeta(src) + return f, err +} diff --git a/vendor/google.golang.org/appengine/search/struct_test.go b/vendor/google.golang.org/appengine/search/struct_test.go new file mode 100644 index 0000000..4e5b5d1 --- /dev/null +++ b/vendor/google.golang.org/appengine/search/struct_test.go @@ -0,0 +1,213 @@ +// Copyright 2015 Google Inc. All rights reserved. +// Use of this source code is governed by the Apache 2.0 +// license that can be found in the LICENSE file. + +package search + +import ( + "reflect" + "testing" +) + +func TestLoadingStruct(t *testing.T) { + testCases := []struct { + desc string + fields []Field + meta *DocumentMetadata + want interface{} + wantErr bool + }{ + { + desc: "Basic struct", + fields: []Field{ + {Name: "Name", Value: "Gopher"}, + {Name: "Legs", Value: float64(4)}, + }, + want: &struct { + Name string + Legs float64 + }{"Gopher", 4}, + }, + { + desc: "Struct with tags", + fields: []Field{ + {Name: "Name", Value: "Gopher"}, + {Name: "about", Value: "Likes slide rules."}, + }, + meta: &DocumentMetadata{Facets: []Facet{ + {Name: "Legs", Value: float64(4)}, + {Name: "Fur", Value: Atom("furry")}, + }}, + want: &struct { + Name string + Info string `search:"about"` + Legs float64 `search:",facet"` + Fuzz Atom `search:"Fur,facet"` + }{"Gopher", "Likes slide rules.", 4, Atom("furry")}, + }, + { + desc: "Bad field from tag", + want: &struct { + AlphaBeta string `search:"αβ"` + }{}, + wantErr: true, + }, + { + desc: "Ignore missing field", + fields: []Field{ + {Name: "Meaning", Value: float64(42)}, + }, + want: &struct{}{}, + wantErr: true, + }, + { + desc: "Ignore unsettable field", + fields: []Field{ + {Name: "meaning", Value: float64(42)}, + }, + want: &struct{ meaning float64 }{}, // field not populated. + wantErr: true, + }, + { + desc: "Error on missing facet", + meta: &DocumentMetadata{Facets: []Facet{ + {Name: "Set", Value: Atom("yes")}, + {Name: "Missing", Value: Atom("no")}, + }}, + want: &struct { + Set Atom `search:",facet"` + }{Atom("yes")}, + wantErr: true, + }, + { + desc: "Error on unsettable facet", + meta: &DocumentMetadata{Facets: []Facet{ + {Name: "Set", Value: Atom("yes")}, + {Name: "unset", Value: Atom("no")}, + }}, + want: &struct { + Set Atom `search:",facet"` + }{Atom("yes")}, + wantErr: true, + }, + { + desc: "Error setting ignored field", + fields: []Field{ + {Name: "Set", Value: "yes"}, + {Name: "Ignored", Value: "no"}, + }, + want: &struct { + Set string + Ignored string `search:"-"` + }{Set: "yes"}, + wantErr: true, + }, + { + desc: "Error setting ignored facet", + meta: &DocumentMetadata{Facets: []Facet{ + {Name: "Set", Value: Atom("yes")}, + {Name: "Ignored", Value: Atom("no")}, + }}, + want: &struct { + Set Atom `search:",facet"` + Ignored Atom `search:"-,facet"` + }{Set: Atom("yes")}, + wantErr: true, + }, + } + + for _, tt := range testCases { + // Make a pointer to an empty version of what want points to. + dst := reflect.New(reflect.TypeOf(tt.want).Elem()).Interface() + err := loadStructWithMeta(dst, tt.fields, tt.meta) + if err != nil != tt.wantErr { + t.Errorf("%s: got err %v; want err %t", tt.desc, err, tt.wantErr) + continue + } + if !reflect.DeepEqual(dst, tt.want) { + t.Errorf("%s: doesn't match\ngot: %v\nwant: %v", tt.desc, dst, tt.want) + } + } +} + +func TestSavingStruct(t *testing.T) { + testCases := []struct { + desc string + doc interface{} + wantFields []Field + wantFacets []Facet + }{ + { + desc: "Basic struct", + doc: &struct { + Name string + Legs float64 + }{"Gopher", 4}, + wantFields: []Field{ + {Name: "Name", Value: "Gopher"}, + {Name: "Legs", Value: float64(4)}, + }, + }, + { + desc: "Struct with tags", + doc: &struct { + Name string + Info string `search:"about"` + Legs float64 `search:",facet"` + Fuzz Atom `search:"Fur,facet"` + }{"Gopher", "Likes slide rules.", 4, Atom("furry")}, + wantFields: []Field{ + {Name: "Name", Value: "Gopher"}, + {Name: "about", Value: "Likes slide rules."}, + }, + wantFacets: []Facet{ + {Name: "Legs", Value: float64(4)}, + {Name: "Fur", Value: Atom("furry")}, + }, + }, + { + desc: "Ignore unexported struct fields", + doc: &struct { + Name string + info string + Legs float64 `search:",facet"` + fuzz Atom `search:",facet"` + }{"Gopher", "Likes slide rules.", 4, Atom("furry")}, + wantFields: []Field{ + {Name: "Name", Value: "Gopher"}, + }, + wantFacets: []Facet{ + {Name: "Legs", Value: float64(4)}, + }, + }, + { + desc: "Ignore fields marked -", + doc: &struct { + Name string + Info string `search:"-"` + Legs float64 `search:",facet"` + Fuzz Atom `search:"-,facet"` + }{"Gopher", "Likes slide rules.", 4, Atom("furry")}, + wantFields: []Field{ + {Name: "Name", Value: "Gopher"}, + }, + wantFacets: []Facet{ + {Name: "Legs", Value: float64(4)}, + }, + }, + } + + for _, tt := range testCases { + fields, meta, err := saveStructWithMeta(tt.doc) + if err != nil { + t.Errorf("%s: got err %v; want nil", tt.desc, err) + continue + } + if !reflect.DeepEqual(fields, tt.wantFields) { + t.Errorf("%s: fields don't match\ngot: %v\nwant: %v", tt.desc, fields, tt.wantFields) + } + if facets := meta.Facets; !reflect.DeepEqual(facets, tt.wantFacets) { + t.Errorf("%s: facets don't match\ngot: %v\nwant: %v", tt.desc, facets, tt.wantFacets) + } + } +} |