gitea/opengist
0
0
Fork 0
mirror of https://github.com/thomiceli/opengist.git synced 2025-05-11 23:10:02 +02:00
Code Issues Packages Projects Releases Wiki Activity

Add Meilisearch indexer (#444)

Browse Source
This commit is contained in:
Thomas Miceli 2025-03-19 23:28:04 +01:00 committed by GitHub
parent dbdfcd4e85
commit efba783c56
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
17 changed files with 373 additions and 102 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
config.yml
View File

@ -23,11 +23,14 @@ secret-key:
# MySQL/MariaDB: mysql://user:password@host:port/database
db-uri: opengist.db
# Enable or disable the code search index (either `true` or `false`). Default: true
index.enabled: true
# Define the code indexer (either `bleve`, `meilisearch`, or empty for no index). Default: bleve
index: bleve
# Name of the directory where the code search index is stored. Default: opengist.index
index.dirname: opengist.index
# Set the host for the Meiliseach server
index.meili.host:
# Set the API key for the Meiliseach server
index.meili.api-key:
# Default branch name used by Opengist when initializing Git repositories.
# If not set, uses the Git default branch name. See https://git-scm.com/book/en/v2/Getting-Started-First-Time-Git-Setup#_new_default_branch

5
docs/configuration/cheat-sheet.md
View File

@ -12,8 +12,9 @@ aside: false
| opengist-home | OG_OPENGIST_HOME | home directory | Path to the directory where Opengist stores its data. |
| secret-key | OG_SECRET_KEY | randomized 32 bytes | Secret key used for session store & encrypt MFA data on database. |
| db-uri | OG_DB_URI | `opengist.db` | URI of the database. |
| index.enabled | OG_INDEX_ENABLED | `true` | Enable or disable the code search index (`true` or `false`) |
| index.dirname | OG_INDEX_DIRNAME | `opengist.index` | Name of the directory where the code search index is stored. |
| index | OG_INDEX | `bleve` | Define the code indexer (either `bleve`, `meilisearch`, or empty for no index). |
| index.meili.host | OG_MEILI_HOST | none | Set the host for the Meiliseach server. |
| index.meili.api-key | OG_MEILI_API_KEY | none | Set the API key for the Meiliseach server. |
| git.default-branch | OG_GIT_DEFAULT_BRANCH | none | Default branch name used by Opengist when initializing Git repositories. If not set, uses the Git default branch name. More info [here](https://git-scm.com/book/en/v2/Getting-Started-First-Time-Git-Setup#_new_default_branch) |
| sqlite.journal-mode | OG_SQLITE_JOURNAL_MODE | `WAL` | Set the journal mode for SQLite. More info [here](https://www.sqlite.org/pragma.html#pragma_journal_mode) |
| http.host | OG_HTTP_HOST | `0.0.0.0` | The host on which the HTTP server should bind. |

5
go.mod
View File

@ -17,6 +17,7 @@ require (
github.com/labstack/echo-contrib v0.17.2
github.com/labstack/echo/v4 v4.13.3
github.com/markbates/goth v1.80.0
github.com/meilisearch/meilisearch-go v0.31.0
github.com/pquerna/otp v1.4.0
github.com/prometheus/client_golang v1.20.5
github.com/rs/zerolog v1.33.0
@ -37,6 +38,7 @@ require (
require (
filippo.io/edwards25519 v1.1.0 // indirect
github.com/RoaringBitmap/roaring v1.9.4 // indirect
github.com/andybalholm/brotli v1.1.1 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bits-and-blooms/bitset v1.17.0 // indirect
github.com/blevesearch/bleve_index_api v1.1.13 // indirect
@ -68,6 +70,7 @@ require (
github.com/go-playground/universal-translator v0.18.1 // indirect
github.com/go-sql-driver/mysql v1.8.1 // indirect
github.com/go-webauthn/x v0.1.15 // indirect
github.com/golang-jwt/jwt/v4 v4.5.1 // indirect
github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
github.com/golang/geo v0.0.0-20230421003525-6adc56603217 // indirect
github.com/golang/protobuf v1.5.4 // indirect
@ -80,11 +83,13 @@ require (
github.com/jackc/puddle/v2 v2.2.2 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/compress v1.17.11 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/labstack/gommon v0.4.2 // indirect
github.com/leodido/go-urn v1.4.0 // indirect
github.com/mailru/easyjson v0.9.0 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-runewidth v0.0.16 // indirect

17
go.sum
View File

@ -12,6 +12,8 @@ github.com/alecthomas/chroma/v2 v2.15.0/go.mod h1:gUhVLrPDXPtp/f+L1jo9xepo9gL4eL
github.com/alecthomas/repr v0.0.0-20220113201626-b1b626ac65ae/go.mod h1:2kn6fqh/zIyPLmm3ugklbEi5hg5wS435eygvNfaDQL8=
github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc=
github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA=
github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/bits-and-blooms/bitset v1.12.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
@ -107,6 +109,8 @@ github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6Wezm
github.com/gobwas/ws v1.1.0 h1:7RFti/xnNkMJnrK7D1yQ/iCIB5OrrY/54/H930kIbHA=
github.com/gobwas/ws v1.1.0/go.mod h1:nzvNcVha5eUziGrbxFCo6qFIojQHjJV5cLYIbezhfL0=
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
github.com/golang-jwt/jwt/v4 v4.5.1 h1:JdqV9zKUdtaa9gdPlywC3aeoEsR681PlKC+4F5gQgeo=
github.com/golang-jwt/jwt/v4 v4.5.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk=
github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
github.com/golang/geo v0.0.0-20230421003525-6adc56603217 h1:HKlyj6in2JV6wVkmQ4XmG/EIm+SCYlPZ+V4GWit7Z+I=
@ -168,8 +172,8 @@ github.com/labstack/gommon v0.4.2 h1:F8qTUNXgG1+6WQmqoUWnz8WiEU60mXVVw0P4ht1WRA0
github.com/labstack/gommon v0.4.2/go.mod h1:QlUFxVM+SNXhDL/Z7YhocGIBYOiwB0mXm1+1bAPHPyU=
github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4=
github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU=
github.com/markbates/goth v1.80.0 h1:NnvatczZDzOs1hn9Ug+dVYf2Viwwkp/ZDX5K+GLjan8=
github.com/markbates/goth v1.80.0/go.mod h1:4/GYHo+W6NWisrMPZnq0Yr2Q70UntNLn7KXEFhrIdAY=
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
@ -180,6 +184,8 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/meilisearch/meilisearch-go v0.31.0 h1:yZRhY1qJqdH8h6GFZALGtkDLyj8f9v5aJpsNMyrUmnY=
github.com/meilisearch/meilisearch-go v0.31.0/go.mod h1:aNtyuwurDg/ggxQIcKqWH6G9g2ptc8GyY7PLY4zMn/g=
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@ -219,8 +225,13 @@ github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWR
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/urfave/cli/v2 v2.27.5 h1:WoHEJLdsXr6dDWoJgMq/CboDmyY/8HMMH1fTECbih+w=
@ -233,6 +244,8 @@ github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4=
github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM=
github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
github.com/yuin/goldmark v1.4.15/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
github.com/yuin/goldmark v1.7.1/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E=
github.com/yuin/goldmark v1.7.8 h1:iERMLn0/QJeHFhxSt3p6PeN9mGnvIKSpG9YYorDMnic=

8
internal/cli/main.go
View File

@ -124,9 +124,9 @@ func Initialize(ctx *cli.Context) {
log.Error().Err(err).Msg("Failed to initialize WebAuthn")
}
if config.C.IndexEnabled {
log.Info().Msg("Index directory: " + filepath.Join(homePath, config.C.IndexDirname))
index.Init(filepath.Join(homePath, config.C.IndexDirname))
index.DepreactionIndexDirname()
if index.IndexEnabled() {
index.NewIndexer(index.IndexType())
}
}
@ -136,7 +136,7 @@ func shutdown() {
log.Error().Err(err).Msg("Failed to close database")
}
if config.C.IndexEnabled {
if index.IndexEnabled() {
log.Info().Msg("Shutting down index...")
index.Close()
}

10
internal/config/config.go
View File

@ -37,8 +37,11 @@ type config struct {
DBUri string `yaml:"db-uri" env:"OG_DB_URI"`
DBFilename string `yaml:"db-filename" env:"OG_DB_FILENAME"` // deprecated
IndexEnabled bool `yaml:"index.enabled" env:"OG_INDEX_ENABLED"`
IndexDirname string `yaml:"index.dirname" env:"OG_INDEX_DIRNAME"`
IndexEnabled bool `yaml:"index.enabled" env:"OG_INDEX_ENABLED"` // deprecated
Index string `yaml:"index" env:"OG_INDEX"`
BleveDirname string `yaml:"index.dirname" env:"OG_INDEX_DIRNAME"` // deprecated
MeiliHost string `yaml:"index.meili.host" env:"OG_MEILI_HOST"`
MeiliAPIKey string `yaml:"index.meili.api-key" env:"OG_MEILI_API_KEY"`
GitDefaultBranch string `yaml:"git.default-branch" env:"OG_GIT_DEFAULT_BRANCH"`
@ -94,8 +97,7 @@ func configWithDefaults() (*config, error) {
c.LogOutput = "stdout,file"
c.OpengistHome = ""
c.DBUri = "opengist.db"
c.IndexEnabled = true
c.IndexDirname = "opengist.index"
c.Index = "bleve"
c.SqliteJournalMode = "WAL"

10
internal/db/gist.go
View File

@ -40,6 +40,10 @@ func (v Visibility) String() string {
}
}
func (v Visibility) Uint() uint {
return uint(v)
}
func (v Visibility) Next() Visibility {
switch v {
case PublicVisibility:
@ -788,6 +792,8 @@ func (gist *Gist) ToIndexedGist() (*index.Gist, error) {
indexedGist := &index.Gist{
GistID: gist.ID,
UserID: gist.UserID,
Visibility: gist.Private.Uint(),
Username: gist.User.Username,
Title: gist.Title,
Content: wholeContent,
@ -803,7 +809,7 @@ func (gist *Gist) ToIndexedGist() (*index.Gist, error) {
}
func (gist *Gist) AddInIndex() {
if !index.Enabled() {
if !index.IndexEnabled() {
return
}
@ -821,7 +827,7 @@ func (gist *Gist) AddInIndex() {
}
func (gist *Gist) RemoveFromIndex() {
if !index.Enabled() {
if !index.IndexEnabled() {
return
}

104
internal/index/bleve.go
View File

@ -10,37 +10,32 @@ import (
"github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode"
"github.com/blevesearch/bleve/v2/search/query"
"github.com/rs/zerolog/log"
"github.com/thomiceli/opengist/internal/config"
"strconv"
"sync/atomic"
)
var atomicIndexer atomic.Pointer[Indexer]
type Indexer struct {
Index bleve.Index
type BleveIndexer struct {
index bleve.Index
path string
}
func Enabled() bool {
return config.C.IndexEnabled
func NewBleveIndexer(path string) *BleveIndexer {
return &BleveIndexer{path: path}
}
func Init(indexFilename string) {
atomicIndexer.Store(&Indexer{Index: nil})
func (i *BleveIndexer) Init() {
go func() {
bleveIndex, err := open(indexFilename)
bleveIndex, err := i.open()
if err != nil {
log.Error().Err(err).Msg("Failed to open index")
(*atomicIndexer.Load()).close()
log.Error().Err(err).Msg("Failed to open Bleve index")
i.Close()
}
atomicIndexer.Store(&Indexer{Index: bleveIndex})
log.Info().Msg("Indexer initialized")
i.index = bleveIndex
log.Info().Msg("Bleve indexer initialized")
}()
}
func open(indexFilename string) (bleve.Index, error) {
bleveIndex, err := bleve.Open(indexFilename)
func (i *BleveIndexer) open() (bleve.Index, error) {
bleveIndex, err := bleve.Open(i.path)
if err == nil {
return bleveIndex, nil
}
@ -73,67 +68,33 @@ func open(indexFilename string) (bleve.Index, error) {
docMapping.DefaultAnalyzer = "gistAnalyser"
return bleve.New(indexFilename, mapping)
return bleve.New(i.path, mapping)
}
func Close() {
(*atomicIndexer.Load()).close()
}
func (i *Indexer) close() {
if i == nil || i.Index == nil {
func (i *BleveIndexer) Close() {
if i == nil || i.index == nil {
return
}
err := i.Index.Close()
err := i.index.Close()
if err != nil {
log.Error().Err(err).Msg("Failed to close bleve index")
log.Error().Err(err).Msg("Failed to close Bleve index")
}
log.Info().Msg("Indexer closed")
atomicIndexer.Store(&Indexer{Index: nil})
log.Info().Msg("Bleve indexer closed")
}
func checkForIndexer() error {
if (*atomicIndexer.Load()).Index == nil {
return errors.New("indexer is not initialized")
}
return nil
}
func AddInIndex(gist *Gist) error {
if !Enabled() {
return nil
}
if err := checkForIndexer(); err != nil {
return err
}
func (i *BleveIndexer) Add(gist *Gist) error {
if gist == nil {
return errors.New("failed to add nil gist to index")
}
return (*atomicIndexer.Load()).Index.Index(strconv.Itoa(int(gist.GistID)), gist)
return (*atomicIndexer.Load()).(*BleveIndexer).index.Index(strconv.Itoa(int(gist.GistID)), gist)
}
func RemoveFromIndex(gistID uint) error {
if !Enabled() {
return nil
}
if err := checkForIndexer(); err != nil {
return err
}
return (*atomicIndexer.Load()).Index.Delete(strconv.Itoa(int(gistID)))
func (i *BleveIndexer) Remove(gistID uint) error {
return (*atomicIndexer.Load()).(*BleveIndexer).index.Delete(strconv.Itoa(int(gistID)))
}
func SearchGists(queryStr string, queryMetadata SearchGistMetadata, gistsIds []uint, page int) ([]uint, uint64, map[string]int, error) {
if !Enabled() {
return nil, 0, nil, nil
}
if err := checkForIndexer(); err != nil {
return nil, 0, nil, err
}
func (i *BleveIndexer) Search(queryStr string, queryMetadata SearchGistMetadata, userId uint, page int) ([]uint, uint64, map[string]int, error) {
var err error
var indexerQuery query.Query
if queryStr != "" {
@ -145,17 +106,16 @@ func SearchGists(queryStr string, queryMetadata SearchGistMetadata, gistsIds []u
indexerQuery = contentQuery
}
repoQueries := make([]query.Query, 0, len(gistsIds))
privateQuery := bleve.NewBoolFieldQuery(false)
privateQuery.SetField("Private")
userIdMatch := float64(userId)
truee := true
for _, id := range gistsIds {
f := float64(id)
qq := bleve.NewNumericRangeInclusiveQuery(&f, &f, &truee, &truee)
qq.SetField("GistID")
repoQueries = append(repoQueries, qq)
}
userIdQuery := bleve.NewNumericRangeInclusiveQuery(&userIdMatch, &userIdMatch, &truee, &truee)
userIdQuery.SetField("UserID")
indexerQuery = bleve.NewConjunctionQuery(bleve.NewDisjunctionQuery(repoQueries...), indexerQuery)
accessQuery := bleve.NewDisjunctionQuery(privateQuery, userIdQuery)
indexerQuery = bleve.NewConjunctionQuery(accessQuery, indexerQuery)
addQuery := func(field, value string) {
if value != "" && value != "." {
@ -182,7 +142,7 @@ func SearchGists(queryStr string, queryMetadata SearchGistMetadata, gistsIds []u
s.Fields = []string{"GistID"}
s.IncludeLocations = false
results, err := (*atomicIndexer.Load()).Index.Search(s)
results, err := (*atomicIndexer.Load()).(*BleveIndexer).index.Search(s)
if err != nil {
return nil, 0, nil, err
}

2
internal/index/gist.go
View File

@ -2,6 +2,8 @@ package index
type Gist struct {
GistID uint
UserID uint
Visibility uint
Username string
Title string
Content string

136
internal/index/indexer.go Normal file
View File

@ -0,0 +1,136 @@
package index
import (
"fmt"
"github.com/rs/zerolog/log"
"github.com/thomiceli/opengist/internal/config"
"path/filepath"
"sync/atomic"
)
var atomicIndexer atomic.Pointer[Indexer]
type Indexer interface {
Init()
Close()
Add(gist *Gist) error
Remove(gistID uint) error
Search(query string, metadata SearchGistMetadata, userId uint, page int) ([]uint, uint64, map[string]int, error)
}
type IndexerType string
const (
Bleve IndexerType = "bleve"
Meilisearch IndexerType = "meilisearch"
None IndexerType = ""
)
func IndexType() IndexerType {
switch config.C.Index {
case "bleve":
return Bleve
case "meilisearch":
return Meilisearch
default:
return None
}
}
func IndexEnabled() bool {
switch config.C.Index {
case "bleve", "meilisearch":
return true
default:
return false
}
}
func NewIndexer(idxType IndexerType) {
if !IndexEnabled() {
return
}
atomicIndexer.Store(nil)
var idx Indexer
switch idxType {
case Bleve:
idx = NewBleveIndexer(filepath.Join(config.GetHomeDir(), "opengist.index"))
case Meilisearch:
idx = NewMeiliIndexer(config.C.MeiliHost, config.C.MeiliAPIKey, "opengist")
default:
log.Warn().Msgf("Failed to create indexer, unknown indexer type: %s", idxType)
return
}
idx.Init()
atomicIndexer.Store(&idx)
}
func Close() {
if !IndexEnabled() {
return
}
idx := *atomicIndexer.Load()
if idx == nil {
return
}
idx.Close()
atomicIndexer.Store(nil)
}
func AddInIndex(gist *Gist) error {
if !IndexEnabled() {
return nil
}
idx := *atomicIndexer.Load()
if idx == nil {
return fmt.Errorf("indexer is not initialized")
}
return idx.Add(gist)
}
func RemoveFromIndex(gistID uint) error {
if !IndexEnabled() {
return nil
}
idx := *atomicIndexer.Load()
if idx == nil {
return fmt.Errorf("indexer is not initialized")
}
return idx.Remove(gistID)
}
func SearchGists(query string, metadata SearchGistMetadata, userId uint, page int) ([]uint, uint64, map[string]int, error) {
if !IndexEnabled() {
return nil, 0, nil, nil
}
idx := *atomicIndexer.Load()
if idx == nil {
return nil, 0, nil, fmt.Errorf("indexer is not initialized")
}
return idx.Search(query, metadata, userId, page)
}
func DepreactionIndexDirname() {
if config.C.IndexEnabled {
log.Warn().Msg("The 'index.enabled'/'OG_INDEX_ENABLED' configuration option is deprecated and will be removed in a future version. Please use 'index'/'OG_INDEX' instead.")
}
if config.C.Index == "" {
config.C.Index = "bleve"
}
if config.C.BleveDirname != "" {
log.Warn().Msg("The 'index.dirname'/'OG_INDEX_DIRNAME' configuration option is deprecated and will be removed in a future version.")
}
}

146
internal/index/meilisearch.go Normal file
View File

@ -0,0 +1,146 @@
package index
import (
"errors"
"fmt"
"github.com/meilisearch/meilisearch-go"
"github.com/rs/zerolog/log"
"strconv"
"strings"
)
type MeiliIndexer struct {
client meilisearch.ServiceManager
index meilisearch.IndexManager
indexName string
host string
apikey string
}
func NewMeiliIndexer(host, apikey, indexName string) *MeiliIndexer {
return &MeiliIndexer{
host: host,
apikey: apikey,
indexName: indexName,
}
}
func (i *MeiliIndexer) Init() {
go func() {
meiliIndex, err := i.open()
if err != nil {
log.Error().Err(err).Msg("Failed to open Meilisearch index")
i.Close()
}
i.index = meiliIndex
log.Info().Msg("Meilisearch indexer initialized")
}()
}
func (i *MeiliIndexer) open() (meilisearch.IndexManager, error) {
client := meilisearch.New(i.host, meilisearch.WithAPIKey(i.apikey))
indexResult, err := client.GetIndex(i.indexName)
if err != nil {
return nil, err
}
if indexResult != nil {
return indexResult.IndexManager, nil
}
_, err = client.CreateIndex(&meilisearch.IndexConfig{
Uid: i.indexName,
PrimaryKey: "GistID",
})
if err != nil {
return nil, err
}
_, _ = client.Index(i.indexName).UpdateSettings(&meilisearch.Settings{
FilterableAttributes: []string{"GistID", "UserID", "Visibility", "Username", "Title", "Filenames", "Extensions", "Languages", "Topics"},
DisplayedAttributes: []string{"GistID"},
SearchableAttributes: []string{"Content", "Username", "Title", "Filenames", "Extensions", "Languages", "Topics"},
RankingRules: []string{"words"},
})
return client.Index(i.indexName), nil
}
func (i *MeiliIndexer) Close() {
if i.client != nil {
i.client.Close()
log.Info().Msg("Meilisearch indexer closed")
}
i.client = nil
}
func (i *MeiliIndexer) Add(gist *Gist) error {
if gist == nil {
return errors.New("failed to add nil gist to index")
}
_, err := (*atomicIndexer.Load()).(*MeiliIndexer).index.AddDocuments(gist, "GistID")
return err
}
func (i *MeiliIndexer) Remove(gistID uint) error {
_, err := (*atomicIndexer.Load()).(*MeiliIndexer).index.DeleteDocument(strconv.Itoa(int(gistID)))
return err
}
func (i *MeiliIndexer) Search(queryStr string, queryMetadata SearchGistMetadata, userId uint, page int) ([]uint, uint64, map[string]int, error) {
searchRequest := &meilisearch.SearchRequest{
Offset: int64((page - 1) * 10),
Limit: 11,
AttributesToRetrieve: []string{"GistID", "Languages"},
Facets: []string{"Languages"},
AttributesToSearchOn: []string{"Content"},
}
var filters []string
filters = append(filters, fmt.Sprintf("(Visibility = 0 OR UserID = %d)", userId))
addFilter := func(field, value string) {
if value != "" && value != "." {
filters = append(filters, fmt.Sprintf("%s = \"%s\"", field, escapeFilterValue(value)))
}
}
addFilter("Username", queryMetadata.Username)
addFilter("Title", queryMetadata.Title)
addFilter("Filenames", queryMetadata.Filename)
addFilter("Extensions", queryMetadata.Extension)
addFilter("Languages", queryMetadata.Language)
addFilter("Topics", queryMetadata.Topic)
if len(filters) > 0 {
searchRequest.Filter = strings.Join(filters, " AND ")
}
response, err := (*atomicIndexer.Load()).(*MeiliIndexer).index.Search(queryStr, searchRequest)
if err != nil {
log.Error().Err(err).Msg("Failed to search Meilisearch index")
return nil, 0, nil, err
}
gistIds := make([]uint, 0, len(response.Hits))
for _, hit := range response.Hits {
if gistID, ok := hit.(map[string]interface{})["GistID"].(float64); ok {
gistIds = append(gistIds, uint(gistID))
}
}
languageCounts := make(map[string]int)
if facets, ok := response.FacetDistribution.(map[string]interface{})["Languages"]; ok {
for language, count := range facets.(map[string]interface{}) {
if countValue, ok := count.(float64); ok {
languageCounts[language] = int(countValue)
}
}
}
return gistIds, uint64(response.EstimatedTotalHits), languageCounts, nil
}
func escapeFilterValue(value string) string {
escaped := strings.ReplaceAll(value, "\\", "\\\\")
escaped = strings.ReplaceAll(escaped, "\"", "\\\"")
return escaped
}

8
internal/web/handlers/gist/all.go
View File

@ -179,12 +179,6 @@ func Search(ctx *context.Context) error {
currentUserId = 0
}
var visibleGistsIds []uint
visibleGistsIds, err = db.GetAllGistsVisibleByUser(currentUserId)
if err != nil {
return ctx.ErrorRes(500, "Error fetching gists", err)
}
gistsIds, nbHits, langs, err := index.SearchGists(content, index.SearchGistMetadata{
Username: meta["user"],
Title: meta["title"],
@ -192,7 +186,7 @@ func Search(ctx *context.Context) error {
Extension: meta["extension"],
Language: meta["language"],
Topic: meta["topic"],
}, visibleGistsIds, pageInt)
}, currentUserId, pageInt)
if err != nil {
return ctx.ErrorRes(500, "Error searching gists", err)
}

2
internal/web/handlers/gist/edit.go
View File

@ -70,6 +70,8 @@ func EditVisibility(ctx *context.Context) error {
return ctx.ErrorRes(500, "Error updating this gist", err)
}
gist.AddInIndex()
ctx.AddFlash(ctx.Tr("flash.gist.visibility-changed"), "success")
return ctx.RedirectTo("/" + gist.User.Username + "/" + gist.Identifier())
}

2
internal/web/server/renderer.go
View File

@ -171,7 +171,7 @@ func (s *Server) setFuncMap() {
return strings.TrimSpace(resultBuilder.String())
},
"indexEnabled": index.Enabled,
"indexEnabled": index.IndexEnabled,
"isUrl": func(s string) bool {
_, err := url.ParseRequestURI(s)
return err == nil

2
internal/web/server/router.go
View File

@ -98,7 +98,7 @@ func (s *Server) registerRoutes() {
r.GET("/all", gist.AllGists, checkRequireLogin, setAllGistsMode("all"))
if index.Enabled() {
if index.IndexEnabled() {
r.GET("/search", gist.Search, checkRequireLogin)
} else {
r.GET("/search", gist.AllGists, checkRequireLogin, setAllGistsMode("search"))

2
internal/web/test/server.go
View File

@ -151,7 +151,7 @@ func Setup(t *testing.T) *TestServer {
git.ReposDirectory = filepath.Join("tests")
config.C.IndexEnabled = false
config.C.Index = ""
config.C.LogLevel = "error"
config.InitLog()

5
templates/pages/admin_config.html vendored
View File

@ -19,8 +19,9 @@
<dt>Opengist home</dt><dd>{{ .c.OpengistHome }}</dd>
<dt>Database type</dt><dd>{{ .dbtype }}{{ if eq .dbtype "SQLite" }} ({{ .c.SqliteJournalMode }}){{ end }}</dd>
<dt>Database name</dt><dd>{{ .dbname }}</dd>
<dt>Index Enabled</dt><dd>{{ .c.IndexEnabled }}</dd>
<dt>Index Dirname</dt><dd>{{ .c.IndexDirname }}</dd>
<dt>Index</dt><dd>{{ .c.Index }}</dd>
<dt>MeiliSearch Host</dt><dd>{{ .c.MeiliHost }}</dd>
<dt>MeiliSearch API Key</dt><dd>{{ if .c.MeiliAPIKey }}&#60;defined&#62;{{ end }}</dd>
<dt>Git default branch</dt><dd>{{ .c.GitDefaultBranch }}</dd>
<div class="relative col-span-3 mt-4">
<div class="absolute inset-0 flex items-center" aria-hidden="true">