Add Gist code search (#194)

internal/index/bleve.go

@@ -0,0 +1,157 @@
+package index
+
+import (
+	"errors"
+	"github.com/blevesearch/bleve/v2"
+	"github.com/blevesearch/bleve/v2/analysis/analyzer/custom"
+	"github.com/blevesearch/bleve/v2/analysis/token/camelcase"
+	"github.com/blevesearch/bleve/v2/analysis/token/lowercase"
+	"github.com/blevesearch/bleve/v2/analysis/token/unicodenorm"
+	"github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode"
+	"github.com/blevesearch/bleve/v2/search/query"
+	"github.com/thomiceli/opengist/internal/config"
+	"strconv"
+)
+
+var bleveIndex bleve.Index
+
+func Enabled() bool {
+	return config.C.IndexEnabled
+}
+
+func Open(indexFilename string) error {
+	var err error
+	bleveIndex, err = bleve.Open(indexFilename)
+	if err == nil {
+		return nil
+	}
+
+	if !errors.Is(err, bleve.ErrorIndexPathDoesNotExist) {
+		return err
+	}
+
+	docMapping := bleve.NewDocumentMapping()
+	docMapping.AddFieldMappingsAt("GistID", bleve.NewNumericFieldMapping())
+	docMapping.AddFieldMappingsAt("Content", bleve.NewTextFieldMapping())
+
+	mapping := bleve.NewIndexMapping()
+
+	if err = mapping.AddCustomTokenFilter("unicodeNormalize", map[string]any{
+		"type": unicodenorm.Name,
+		"form": unicodenorm.NFC,
+	}); err != nil {
+		return err
+	}
+
+	if err = mapping.AddCustomAnalyzer("gistAnalyser", map[string]interface{}{
+		"type":          custom.Name,
+		"char_filters":  []string{},
+		"tokenizer":     unicode.Name,
+		"token_filters": []string{"unicodeNormalize", camelcase.Name, lowercase.Name},
+	}); err != nil {
+		return err
+	}
+
+	docMapping.DefaultAnalyzer = "gistAnalyser"
+
+	bleveIndex, err = bleve.New(indexFilename, mapping)
+
+	return err
+}
+
+func Close() error {
+	return bleveIndex.Close()
+}
+
+func AddInIndex(gist *Gist) error {
+	if !Enabled() {
+		return nil
+	}
+
+	if gist == nil {
+		return errors.New("failed to add nil gist to index")
+	}
+	return bleveIndex.Index(strconv.Itoa(int(gist.GistID)), gist)
+}
+
+func RemoveFromIndex(gistID uint) error {
+	if !Enabled() {
+		return nil
+	}
+
+	return bleveIndex.Delete(strconv.Itoa(int(gistID)))
+}
+
+func SearchGists(queryStr string, queryMetadata SearchGistMetadata, gistsIds []uint, page int) ([]uint, uint64, map[string]int, error) {
+	if !Enabled() {
+		return nil, 0, nil, nil
+	}
+
+	var err error
+	var indexerQuery query.Query
+	if queryStr != "" {
+		contentQuery := bleve.NewMatchPhraseQuery(queryStr)
+		contentQuery.FieldVal = "Content"
+		indexerQuery = contentQuery
+	} else {
+		contentQuery := bleve.NewMatchAllQuery()
+		indexerQuery = contentQuery
+	}
+
+	if len(gistsIds) > 0 {
+		repoQueries := make([]query.Query, 0, len(gistsIds))
+
+		truee := true
+		for _, id := range gistsIds {
+			f := float64(id)
+			qq := bleve.NewNumericRangeInclusiveQuery(&f, &f, &truee, &truee)
+			qq.SetField("GistID")
+			repoQueries = append(repoQueries, qq)
+		}
+
+		indexerQuery = bleve.NewConjunctionQuery(bleve.NewDisjunctionQuery(repoQueries...), indexerQuery)
+	}
+
+	addQuery := func(field, value string) {
+		if value != "" && value != "." {
+			q := bleve.NewMatchPhraseQuery(value)
+			q.FieldVal = field
+			indexerQuery = bleve.NewConjunctionQuery(indexerQuery, q)
+		}
+	}
+
+	addQuery("Username", queryMetadata.Username)
+	addQuery("Title", queryMetadata.Title)
+	addQuery("Extensions", "."+queryMetadata.Extension)
+	addQuery("Filenames", queryMetadata.Filename)
+	addQuery("Languages", queryMetadata.Language)
+
+	languageFacet := bleve.NewFacetRequest("Languages", 10)
+
+	perPage := 10
+	offset := (page - 1) * perPage
+
+	s := bleve.NewSearchRequestOptions(indexerQuery, perPage, offset, false)
+	s.AddFacet("languageFacet", languageFacet)
+	s.Fields = []string{"GistID"}
+	s.IncludeLocations = false
+
+	results, err := bleveIndex.Search(s)
+	if err != nil {
+		return nil, 0, nil, err
+	}
+
+	gistIds := make([]uint, 0, len(results.Hits))
+	for _, hit := range results.Hits {
+		gistIds = append(gistIds, uint(hit.Fields["GistID"].(float64)))
+	}
+
+	languageCounts := make(map[string]int)
+	if facets, found := results.Facets["languageFacet"]; found {
+		for _, term := range facets.Terms.Terms() {
+			languageCounts[term.Term] = term.Count
+		}
+	}
+
+	return gistIds, results.Total, languageCounts, nil
+}