gitea/opengist
0
0
Fork 0
mirror of https://github.com/thomiceli/opengist.git synced 2025-06-19 00:27:11 +02:00
Code Issues Packages Projects Releases Wiki Activity

Add Meilisearch indexer (#444)

Browse Source
This commit is contained in:
Thomas Miceli
2025-03-19 23:28:04 +01:00
committed by GitHub
parent dbdfcd4e85
commit efba783c56
17 changed files with 373 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
internal/cli/main.go
View File

@ -124,9 +124,9 @@ func Initialize(ctx *cli.Context) {
log.Error().Err(err).Msg("Failed to initialize WebAuthn")
}
if config.C.IndexEnabled {
log.Info().Msg("Index directory: " + filepath.Join(homePath, config.C.IndexDirname))
index.Init(filepath.Join(homePath, config.C.IndexDirname))
index.DepreactionIndexDirname()
if index.IndexEnabled() {
index.NewIndexer(index.IndexType())
}
}
@ -136,7 +136,7 @@ func shutdown() {
log.Error().Err(err).Msg("Failed to close database")
}
if config.C.IndexEnabled {
if index.IndexEnabled() {
log.Info().Msg("Shutting down index...")
index.Close()
}

10
internal/config/config.go
View File

@ -37,8 +37,11 @@ type config struct {
DBUri string `yaml:"db-uri" env:"OG_DB_URI"`
DBFilename string `yaml:"db-filename" env:"OG_DB_FILENAME"` // deprecated
IndexEnabled bool `yaml:"index.enabled" env:"OG_INDEX_ENABLED"`
IndexDirname string `yaml:"index.dirname" env:"OG_INDEX_DIRNAME"`
IndexEnabled bool `yaml:"index.enabled" env:"OG_INDEX_ENABLED"` // deprecated
Index string `yaml:"index" env:"OG_INDEX"`
BleveDirname string `yaml:"index.dirname" env:"OG_INDEX_DIRNAME"` // deprecated
MeiliHost string `yaml:"index.meili.host" env:"OG_MEILI_HOST"`
MeiliAPIKey string `yaml:"index.meili.api-key" env:"OG_MEILI_API_KEY"`
GitDefaultBranch string `yaml:"git.default-branch" env:"OG_GIT_DEFAULT_BRANCH"`
@ -94,8 +97,7 @@ func configWithDefaults() (*config, error) {
c.LogOutput = "stdout,file"
c.OpengistHome = ""
c.DBUri = "opengist.db"
c.IndexEnabled = true
c.IndexDirname = "opengist.index"
c.Index = "bleve"
c.SqliteJournalMode = "WAL"

10
internal/db/gist.go
View File

@ -40,6 +40,10 @@ func (v Visibility) String() string {
}
}
func (v Visibility) Uint() uint {
return uint(v)
}
func (v Visibility) Next() Visibility {
switch v {
case PublicVisibility:
@ -788,6 +792,8 @@ func (gist *Gist) ToIndexedGist() (*index.Gist, error) {
indexedGist := &index.Gist{
GistID: gist.ID,
UserID: gist.UserID,
Visibility: gist.Private.Uint(),
Username: gist.User.Username,
Title: gist.Title,
Content: wholeContent,
@ -803,7 +809,7 @@ func (gist *Gist) ToIndexedGist() (*index.Gist, error) {
}
func (gist *Gist) AddInIndex() {
if !index.Enabled() {
if !index.IndexEnabled() {
return
}
@ -821,7 +827,7 @@ func (gist *Gist) AddInIndex() {
}
func (gist *Gist) RemoveFromIndex() {
if !index.Enabled() {
if !index.IndexEnabled() {
return
}

104
internal/index/bleve.go
View File

@ -10,37 +10,32 @@ import (
"github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode"
"github.com/blevesearch/bleve/v2/search/query"
"github.com/rs/zerolog/log"
"github.com/thomiceli/opengist/internal/config"
"strconv"
"sync/atomic"
)
var atomicIndexer atomic.Pointer[Indexer]
type Indexer struct {
Index bleve.Index
type BleveIndexer struct {
index bleve.Index
path string
}
func Enabled() bool {
return config.C.IndexEnabled
func NewBleveIndexer(path string) *BleveIndexer {
return &BleveIndexer{path: path}
}
func Init(indexFilename string) {
atomicIndexer.Store(&Indexer{Index: nil})
func (i *BleveIndexer) Init() {
go func() {
bleveIndex, err := open(indexFilename)
bleveIndex, err := i.open()
if err != nil {
log.Error().Err(err).Msg("Failed to open index")
(*atomicIndexer.Load()).close()
log.Error().Err(err).Msg("Failed to open Bleve index")
i.Close()
}
atomicIndexer.Store(&Indexer{Index: bleveIndex})
log.Info().Msg("Indexer initialized")
i.index = bleveIndex
log.Info().Msg("Bleve indexer initialized")
}()
}
func open(indexFilename string) (bleve.Index, error) {
bleveIndex, err := bleve.Open(indexFilename)
func (i *BleveIndexer) open() (bleve.Index, error) {
bleveIndex, err := bleve.Open(i.path)
if err == nil {
return bleveIndex, nil
}
@ -73,67 +68,33 @@ func open(indexFilename string) (bleve.Index, error) {
docMapping.DefaultAnalyzer = "gistAnalyser"
return bleve.New(indexFilename, mapping)
return bleve.New(i.path, mapping)
}
func Close() {
(*atomicIndexer.Load()).close()
}
func (i *Indexer) close() {
if i == nil || i.Index == nil {
func (i *BleveIndexer) Close() {
if i == nil || i.index == nil {
return
}
err := i.Index.Close()
err := i.index.Close()
if err != nil {
log.Error().Err(err).Msg("Failed to close bleve index")
log.Error().Err(err).Msg("Failed to close Bleve index")
}
log.Info().Msg("Indexer closed")
atomicIndexer.Store(&Indexer{Index: nil})
log.Info().Msg("Bleve indexer closed")
}
func checkForIndexer() error {
if (*atomicIndexer.Load()).Index == nil {
return errors.New("indexer is not initialized")
}
return nil
}
func AddInIndex(gist *Gist) error {
if !Enabled() {
return nil
}
if err := checkForIndexer(); err != nil {
return err
}
func (i *BleveIndexer) Add(gist *Gist) error {
if gist == nil {
return errors.New("failed to add nil gist to index")
}
return (*atomicIndexer.Load()).Index.Index(strconv.Itoa(int(gist.GistID)), gist)
return (*atomicIndexer.Load()).(*BleveIndexer).index.Index(strconv.Itoa(int(gist.GistID)), gist)
}
func RemoveFromIndex(gistID uint) error {
if !Enabled() {
return nil
}
if err := checkForIndexer(); err != nil {
return err
}
return (*atomicIndexer.Load()).Index.Delete(strconv.Itoa(int(gistID)))
func (i *BleveIndexer) Remove(gistID uint) error {
return (*atomicIndexer.Load()).(*BleveIndexer).index.Delete(strconv.Itoa(int(gistID)))
}
func SearchGists(queryStr string, queryMetadata SearchGistMetadata, gistsIds []uint, page int) ([]uint, uint64, map[string]int, error) {
if !Enabled() {
return nil, 0, nil, nil
}
if err := checkForIndexer(); err != nil {
return nil, 0, nil, err
}
func (i *BleveIndexer) Search(queryStr string, queryMetadata SearchGistMetadata, userId uint, page int) ([]uint, uint64, map[string]int, error) {
var err error
var indexerQuery query.Query
if queryStr != "" {
@ -145,17 +106,16 @@ func SearchGists(queryStr string, queryMetadata SearchGistMetadata, gistsIds []u
indexerQuery = contentQuery
}
repoQueries := make([]query.Query, 0, len(gistsIds))
privateQuery := bleve.NewBoolFieldQuery(false)
privateQuery.SetField("Private")
userIdMatch := float64(userId)
truee := true
for _, id := range gistsIds {
f := float64(id)
qq := bleve.NewNumericRangeInclusiveQuery(&f, &f, &truee, &truee)
qq.SetField("GistID")
repoQueries = append(repoQueries, qq)
}
userIdQuery := bleve.NewNumericRangeInclusiveQuery(&userIdMatch, &userIdMatch, &truee, &truee)
userIdQuery.SetField("UserID")
indexerQuery = bleve.NewConjunctionQuery(bleve.NewDisjunctionQuery(repoQueries...), indexerQuery)
accessQuery := bleve.NewDisjunctionQuery(privateQuery, userIdQuery)
indexerQuery = bleve.NewConjunctionQuery(accessQuery, indexerQuery)
addQuery := func(field, value string) {
if value != "" && value != "." {
@ -182,7 +142,7 @@ func SearchGists(queryStr string, queryMetadata SearchGistMetadata, gistsIds []u
s.Fields = []string{"GistID"}
s.IncludeLocations = false
results, err := (*atomicIndexer.Load()).Index.Search(s)
results, err := (*atomicIndexer.Load()).(*BleveIndexer).index.Search(s)
if err != nil {
return nil, 0, nil, err
}

2
internal/index/gist.go
View File

@ -2,6 +2,8 @@ package index
type Gist struct {
GistID uint
UserID uint
Visibility uint
Username string
Title string
Content string

136
internal/index/indexer.go Normal file
View File

@ -0,0 +1,136 @@
package index
import (
"fmt"
"github.com/rs/zerolog/log"
"github.com/thomiceli/opengist/internal/config"
"path/filepath"
"sync/atomic"
)
var atomicIndexer atomic.Pointer[Indexer]
type Indexer interface {
Init()
Close()
Add(gist *Gist) error
Remove(gistID uint) error
Search(query string, metadata SearchGistMetadata, userId uint, page int) ([]uint, uint64, map[string]int, error)
}
type IndexerType string
const (
Bleve IndexerType = "bleve"
Meilisearch IndexerType = "meilisearch"
None IndexerType = ""
)
func IndexType() IndexerType {
switch config.C.Index {
case "bleve":
return Bleve
case "meilisearch":
return Meilisearch
default:
return None
}
}
func IndexEnabled() bool {
switch config.C.Index {
case "bleve", "meilisearch":
return true
default:
return false
}
}
func NewIndexer(idxType IndexerType) {
if !IndexEnabled() {
return
}
atomicIndexer.Store(nil)
var idx Indexer
switch idxType {
case Bleve:
idx = NewBleveIndexer(filepath.Join(config.GetHomeDir(), "opengist.index"))
case Meilisearch:
idx = NewMeiliIndexer(config.C.MeiliHost, config.C.MeiliAPIKey, "opengist")
default:
log.Warn().Msgf("Failed to create indexer, unknown indexer type: %s", idxType)
return
}
idx.Init()
atomicIndexer.Store(&idx)
}
func Close() {
if !IndexEnabled() {
return
}
idx := *atomicIndexer.Load()
if idx == nil {
return
}
idx.Close()
atomicIndexer.Store(nil)
}
func AddInIndex(gist *Gist) error {
if !IndexEnabled() {
return nil
}
idx := *atomicIndexer.Load()
if idx == nil {
return fmt.Errorf("indexer is not initialized")
}
return idx.Add(gist)
}
func RemoveFromIndex(gistID uint) error {
if !IndexEnabled() {
return nil
}
idx := *atomicIndexer.Load()
if idx == nil {
return fmt.Errorf("indexer is not initialized")
}
return idx.Remove(gistID)
}
func SearchGists(query string, metadata SearchGistMetadata, userId uint, page int) ([]uint, uint64, map[string]int, error) {
if !IndexEnabled() {
return nil, 0, nil, nil
}
idx := *atomicIndexer.Load()
if idx == nil {
return nil, 0, nil, fmt.Errorf("indexer is not initialized")
}
return idx.Search(query, metadata, userId, page)
}
func DepreactionIndexDirname() {
if config.C.IndexEnabled {
log.Warn().Msg("The 'index.enabled'/'OG_INDEX_ENABLED' configuration option is deprecated and will be removed in a future version. Please use 'index'/'OG_INDEX' instead.")
}
if config.C.Index == "" {
config.C.Index = "bleve"
}
if config.C.BleveDirname != "" {
log.Warn().Msg("The 'index.dirname'/'OG_INDEX_DIRNAME' configuration option is deprecated and will be removed in a future version.")
}
}

146
internal/index/meilisearch.go Normal file
View File

@ -0,0 +1,146 @@
package index
import (
"errors"
"fmt"
"github.com/meilisearch/meilisearch-go"
"github.com/rs/zerolog/log"
"strconv"
"strings"
)
type MeiliIndexer struct {
client meilisearch.ServiceManager
index meilisearch.IndexManager
indexName string
host string
apikey string
}
func NewMeiliIndexer(host, apikey, indexName string) *MeiliIndexer {
return &MeiliIndexer{
host: host,
apikey: apikey,
indexName: indexName,
}
}
func (i *MeiliIndexer) Init() {
go func() {
meiliIndex, err := i.open()
if err != nil {
log.Error().Err(err).Msg("Failed to open Meilisearch index")
i.Close()
}
i.index = meiliIndex
log.Info().Msg("Meilisearch indexer initialized")
}()
}
func (i *MeiliIndexer) open() (meilisearch.IndexManager, error) {
client := meilisearch.New(i.host, meilisearch.WithAPIKey(i.apikey))
indexResult, err := client.GetIndex(i.indexName)
if err != nil {
return nil, err
}
if indexResult != nil {
return indexResult.IndexManager, nil
}
_, err = client.CreateIndex(&meilisearch.IndexConfig{
Uid: i.indexName,
PrimaryKey: "GistID",
})
if err != nil {
return nil, err
}
_, _ = client.Index(i.indexName).UpdateSettings(&meilisearch.Settings{
FilterableAttributes: []string{"GistID", "UserID", "Visibility", "Username", "Title", "Filenames", "Extensions", "Languages", "Topics"},
DisplayedAttributes: []string{"GistID"},
SearchableAttributes: []string{"Content", "Username", "Title", "Filenames", "Extensions", "Languages", "Topics"},
RankingRules: []string{"words"},
})
return client.Index(i.indexName), nil
}
func (i *MeiliIndexer) Close() {
if i.client != nil {
i.client.Close()
log.Info().Msg("Meilisearch indexer closed")
}
i.client = nil
}
func (i *MeiliIndexer) Add(gist *Gist) error {
if gist == nil {
return errors.New("failed to add nil gist to index")
}
_, err := (*atomicIndexer.Load()).(*MeiliIndexer).index.AddDocuments(gist, "GistID")
return err
}
func (i *MeiliIndexer) Remove(gistID uint) error {
_, err := (*atomicIndexer.Load()).(*MeiliIndexer).index.DeleteDocument(strconv.Itoa(int(gistID)))
return err
}
func (i *MeiliIndexer) Search(queryStr string, queryMetadata SearchGistMetadata, userId uint, page int) ([]uint, uint64, map[string]int, error) {
searchRequest := &meilisearch.SearchRequest{
Offset: int64((page - 1) * 10),
Limit: 11,
AttributesToRetrieve: []string{"GistID", "Languages"},
Facets: []string{"Languages"},
AttributesToSearchOn: []string{"Content"},
}
var filters []string
filters = append(filters, fmt.Sprintf("(Visibility = 0 OR UserID = %d)", userId))
addFilter := func(field, value string) {
if value != "" && value != "." {
filters = append(filters, fmt.Sprintf("%s = \"%s\"", field, escapeFilterValue(value)))
}
}
addFilter("Username", queryMetadata.Username)
addFilter("Title", queryMetadata.Title)
addFilter("Filenames", queryMetadata.Filename)
addFilter("Extensions", queryMetadata.Extension)
addFilter("Languages", queryMetadata.Language)
addFilter("Topics", queryMetadata.Topic)
if len(filters) > 0 {
searchRequest.Filter = strings.Join(filters, " AND ")
}
response, err := (*atomicIndexer.Load()).(*MeiliIndexer).index.Search(queryStr, searchRequest)
if err != nil {
log.Error().Err(err).Msg("Failed to search Meilisearch index")
return nil, 0, nil, err
}
gistIds := make([]uint, 0, len(response.Hits))
for _, hit := range response.Hits {
if gistID, ok := hit.(map[string]interface{})["GistID"].(float64); ok {
gistIds = append(gistIds, uint(gistID))
}
}
languageCounts := make(map[string]int)
if facets, ok := response.FacetDistribution.(map[string]interface{})["Languages"]; ok {
for language, count := range facets.(map[string]interface{}) {
if countValue, ok := count.(float64); ok {
languageCounts[language] = int(countValue)
}
}
}
return gistIds, uint64(response.EstimatedTotalHits), languageCounts, nil
}
func escapeFilterValue(value string) string {
escaped := strings.ReplaceAll(value, "\\", "\\\\")
escaped = strings.ReplaceAll(escaped, "\"", "\\\"")
return escaped
}

8
internal/web/handlers/gist/all.go
View File

@ -179,12 +179,6 @@ func Search(ctx *context.Context) error {
currentUserId = 0
}
var visibleGistsIds []uint
visibleGistsIds, err = db.GetAllGistsVisibleByUser(currentUserId)
if err != nil {
return ctx.ErrorRes(500, "Error fetching gists", err)
}
gistsIds, nbHits, langs, err := index.SearchGists(content, index.SearchGistMetadata{
Username: meta["user"],
Title: meta["title"],
@ -192,7 +186,7 @@ func Search(ctx *context.Context) error {
Extension: meta["extension"],
Language: meta["language"],
Topic: meta["topic"],
}, visibleGistsIds, pageInt)
}, currentUserId, pageInt)
if err != nil {
return ctx.ErrorRes(500, "Error searching gists", err)
}

2
internal/web/handlers/gist/edit.go
View File

@ -70,6 +70,8 @@ func EditVisibility(ctx *context.Context) error {
return ctx.ErrorRes(500, "Error updating this gist", err)
}
gist.AddInIndex()
ctx.AddFlash(ctx.Tr("flash.gist.visibility-changed"), "success")
return ctx.RedirectTo("/" + gist.User.Username + "/" + gist.Identifier())
}

2
internal/web/server/renderer.go
View File

@ -171,7 +171,7 @@ func (s *Server) setFuncMap() {
return strings.TrimSpace(resultBuilder.String())
},
"indexEnabled": index.Enabled,
"indexEnabled": index.IndexEnabled,
"isUrl": func(s string) bool {
_, err := url.ParseRequestURI(s)
return err == nil

2
internal/web/server/router.go
View File

@ -98,7 +98,7 @@ func (s *Server) registerRoutes() {
r.GET("/all", gist.AllGists, checkRequireLogin, setAllGistsMode("all"))
if index.Enabled() {
if index.IndexEnabled() {
r.GET("/search", gist.Search, checkRequireLogin)
} else {
r.GET("/search", gist.AllGists, checkRequireLogin, setAllGistsMode("search"))

2
internal/web/test/server.go
View File

@ -151,7 +151,7 @@ func Setup(t *testing.T) *TestServer {
git.ReposDirectory = filepath.Join("tests")
config.C.IndexEnabled = false
config.C.Index = ""
config.C.LogLevel = "error"
config.InitLog()