gitea/opengist
0
0
Fork 0
mirror of https://github.com/thomiceli/opengist.git synced 2025-06-13 05:47:12 +02:00
Code Issues Packages Projects Releases Wiki Activity

Search gists on user profile with title, visibility, language & topics (#422)

Browse Source
This commit is contained in:
Thomas Miceli
2025-02-02 18:14:03 +01:00
committed by GitHub
parent 76fc129c09
commit 7aa8f84eff
23 changed files with 429 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
internal/actions/actions.go
View File

@ -23,6 +23,7 @@ const (
SyncGistPreviews
ResetHooks
IndexGists
SyncGistLanguages
)
var (
@ -73,6 +74,8 @@ func Run(actionType int) {
functionToRun = resetHooks
case IndexGists:
functionToRun = indexGists
case SyncGistLanguages:
functionToRun = syncGistLanguages
default:
log.Error().Msg("Unknown action type")
}
@ -166,3 +169,17 @@ func indexGists() {
}
}
}
func syncGistLanguages() {
log.Info().Msg("Syncing all Gist languages...")
gists, err := db.GetAllGistsRows()
if err != nil {
log.Error().Err(err).Msg("Cannot get gists")
return
}
for _, gist := range gists {
log.Info().Msgf("Syncing languages for gist %d", gist.ID)
gist.UpdateLanguages()
}
}

4
internal/db/db.go
View File

@ -144,7 +144,7 @@ func Setup(dbUri string) error {
return err
}
if err = db.AutoMigrate(&User{}, &Gist{}, &SSHKey{}, &AdminSetting{}, &Invitation{}, &WebAuthnCredential{}, &TOTP{}, &GistTopic{}); err != nil {
if err = db.AutoMigrate(&User{}, &Gist{}, &SSHKey{}, &AdminSetting{}, &Invitation{}, &WebAuthnCredential{}, &TOTP{}, &GistTopic{}, &GistLanguage{}); err != nil {
return err
}
@ -258,5 +258,5 @@ func DeprecationDBFilename() {
}
func TruncateDatabase() error {
return db.Migrator().DropTable("likes", &User{}, "gists", &SSHKey{}, &AdminSetting{}, &Invitation{}, &WebAuthnCredential{}, &TOTP{}, &GistTopic{})
return db.Migrator().DropTable("likes", &User{}, "gists", &SSHKey{}, &AdminSetting{}, &Invitation{}, &WebAuthnCredential{}, &TOTP{}, &GistTopic{}, &GistLanguage{})
}

113
internal/db/gist.go
View File

@ -6,6 +6,7 @@ import (
"fmt"
"os/exec"
"path/filepath"
"slices"
"strings"
"time"
@ -50,16 +51,16 @@ func (v Visibility) Next() Visibility {
}
}
func ParseVisibility[T string | int](v T) (Visibility, error) {
func ParseVisibility[T string | int](v T) Visibility {
switch s := fmt.Sprint(v); s {
case "0", "public":
return PublicVisibility, nil
return PublicVisibility
case "1", "unlisted":
return UnlistedVisibility, nil
return UnlistedVisibility
case "2", "private":
return PrivateVisibility, nil
return PrivateVisibility
default:
return -1, fmt.Errorf("unknown visibility %q", s)
return PublicVisibility
}
}
@ -84,7 +85,8 @@ type Gist struct {
Forked *Gist `gorm:"foreignKey:ForkedID;constraint:OnUpdate:CASCADE,OnDelete:SET NULL"`
ForkedID uint
Topics []GistTopic `gorm:"constraint:OnUpdate:CASCADE,OnDelete:CASCADE"`
Topics []GistTopic `gorm:"constraint:OnUpdate:CASCADE,OnDelete:CASCADE"`
Languages []GistLanguage `gorm:"constraint:OnUpdate:CASCADE,OnDelete:CASCADE"`
}
type Like struct {
@ -166,25 +168,59 @@ func GetAllGistsFromSearch(currentUserId uint, query string, offset int, sort st
}
func gistsFromUserStatement(fromUserId uint, currentUserId uint) *gorm.DB {
return db.
Where("((gists.private = 0) or (gists.private > 0 and gists.user_id = ?))", currentUserId).
Where("users.id = ?", fromUserId).
Joins("join users on gists.user_id = users.id")
}
func gistsFromUserStatementWithPreloads(fromUserId uint, currentUserId uint) *gorm.DB {
return db.Preload("User").Preload("Forked.User").Preload("Topics").
Where("((gists.private = 0) or (gists.private > 0 and gists.user_id = ?))", currentUserId).
Where("users.id = ?", fromUserId).
Joins("join users on gists.user_id = users.id")
}
func GetAllGistsFromUser(fromUserId uint, currentUserId uint, offset int, sort string, order string) ([]*Gist, error) {
func GetAllGistsFromUser(fromUserId uint, currentUserId uint, title string, language string, visibility string, topics []string, offset int, sort string, order string) ([]*Gist, int64, error) {
var gists []*Gist
err := gistsFromUserStatement(fromUserId, currentUserId).Limit(11).
var count int64
baseQuery := gistsFromUserStatementWithPreloads(fromUserId, currentUserId).Model(&Gist{})
if title != "" {
baseQuery = baseQuery.Where("gists.title like ?", "%"+title+"%")
}
if language != "" {
baseQuery = baseQuery.Joins("join gist_languages on gists.id = gist_languages.gist_id").
Where("gist_languages.language = ?", language)
}
if visibility != "" {
baseQuery = baseQuery.Where("gists.private = ?", ParseVisibility(visibility))
}
if len(topics) > 0 {
baseQuery = baseQuery.Joins("join gist_topics on gists.id = gist_topics.gist_id").
Where("gist_topics.topic in ?", topics)
}
err := baseQuery.Count(&count).Error
if err != nil {
return nil, 0, err
}
err = baseQuery.Limit(11).
Offset(offset * 10).
Order("gists." + sort + "_at " + order).
Find(&gists).Error
return gists, err
return gists, count, err
}
func CountAllGistsFromUser(fromUserId uint, currentUserId uint) (int64, error) {
var count int64
err := gistsFromUserStatement(fromUserId, currentUserId).Model(&Gist{}).Count(&count).Error
err := gistsFromUserStatementWithPreloads(fromUserId, currentUserId).Model(&Gist{}).Count(&count).Error
return count, err
}
@ -258,7 +294,18 @@ func GetAllGistsByIds(ids []uint) ([]*Gist, error) {
Where("id in ?", ids).
Find(&gists).Error
return gists, err
// keep order
ordered := make([]*Gist, 0, len(ids))
for _, wantedId := range ids {
for _, gist := range gists {
if gist.ID == wantedId {
ordered = append(ordered, gist)
break
}
}
}
return ordered, err
}
func (gist *Gist) Create() error {
@ -593,6 +640,47 @@ func DeserialiseInitRepository(user string) (*Gist, error) {
return &gist, nil
}
func (gist *Gist) UpdateLanguages() {
languages, err := gist.GetLanguagesFromFiles()
if err != nil {
log.Error().Err(err).Msgf("Cannot get languages for gist %d", gist.ID)
return
}
slices.Sort(languages)
languages = slices.Compact(languages)
tx := db.Begin()
if tx.Error != nil {
log.Error().Err(tx.Error).Msgf("Cannot start transaction for gist %d", gist.ID)
return
}
if err := tx.Where("gist_id = ?", gist.ID).Delete(&GistLanguage{}).Error; err != nil {
tx.Rollback()
log.Error().Err(err).Msgf("Cannot delete languages for gist %d", gist.ID)
return
}
for _, language := range languages {
gistLanguage := &GistLanguage{
GistID: gist.ID,
Language: language,
}
if err := tx.Create(gistLanguage).Error; err != nil {
tx.Rollback()
log.Error().Err(err).Msgf("Cannot create gist language %s for gist %d", language, gist.ID)
return
}
}
if err := tx.Commit().Error; err != nil {
tx.Rollback()
log.Error().Err(err).Msgf("Cannot commit transaction for gist %d", gist.ID)
return
}
}
func (gist *Gist) ToDTO() (*GistDTO, error) {
files, err := gist.Files("HEAD", false)
if err != nil {
@ -684,6 +772,9 @@ func (gist *Gist) ToIndexedGist() (*index.Gist, error) {
wholeContent := ""
for _, file := range files {
wholeContent += file.Content
if !strings.HasSuffix(wholeContent, "\n") {
wholeContent += "\n"
}
exts = append(exts, filepath.Ext(file.Filename))
}

27
internal/db/gist_language.go Normal file
View File

@ -0,0 +1,27 @@
package db
type GistLanguage struct {
GistID uint `gorm:"primaryKey"`
Language string `gorm:"primaryKey;size:100"`
}
func GetGistLanguagesForUser(fromUserId, currentUserId uint) ([]struct {
Language string
Count int64
}, error) {
var results []struct {
Language string
Count int64
}
err := gistsFromUserStatement(fromUserId, currentUserId).Model(&GistLanguage{}).
Select("language, count(*) as count").
Joins("JOIN gists ON gists.id = gist_languages.gist_id").
Where("gists.user_id = ?", fromUserId).
Group("language").
Order("count DESC").
Limit(15). // Added limit of 15
Find(&results).Error
return results, err
}

0
internal/db/gist_tag.go → internal/db/gist_topic.go
View File

2
internal/hooks/post_receive.go
View File

@ -46,7 +46,7 @@ func PostReceive(in io.Reader, out, er io.Writer) error {
}
if slices.Contains([]string{"public", "unlisted", "private"}, opts["visibility"]) {
gist.Private, _ = db.ParseVisibility(opts["visibility"])
gist.Private = db.ParseVisibility(opts["visibility"])
outputSb.WriteString(fmt.Sprintf("Gist visibility set to %s\n\n", opts["visibility"]))
}

12
internal/i18n/locales/en-US.yml
View File

@ -87,7 +87,15 @@ gist.search.help.filename: gists having files with given name
gist.search.help.extension: gists having files with given extension
gist.search.help.language: gists having files with given language
gist.search.help.topic: gists with given topic
gist.search.placeholder.title: Title
gist.search.placeholder.visibility: Visibility
gist.search.placeholder.public: Public
gist.search.placeholder.unlisted: Unlisted
gist.search.placeholder.private: Private
gist.search.placeholder.language: Language
gist.search.placeholder.all: All
gist.search.placeholder.topics: Topics
gist.search.placeholder.search: Search
gist.forks: Forks
gist.forks.view: View fork
@ -234,6 +242,7 @@ admin.actions.git-gc: Garbage collect all git repositories
admin.actions.sync-previews: Synchronize all gists previews
admin.actions.reset-hooks: Reset Git server hooks for all repositories
admin.actions.index-gists: Index all gists
admin.actions.sync-gist-languages: Synchronize all gists languages
admin.id: ID
admin.user: User
admin.delete: Delete
@ -279,6 +288,7 @@ flash.admin.git-gc: Garbage collecting repositories...
flash.admin.sync-previews: Syncing Gist previews...
flash.admin.reset-hooks: Resetting Git server hooks for all repositories...
flash.admin.index-gists: Indexing all gists...
flash.admin.sync-gist-languages: Syncing Gist languages...
flash.auth.username-exists: Username already exists
flash.auth.invalid-credentials: Invalid credentials

2
internal/index/bleve.go
View File

@ -177,7 +177,7 @@ func SearchGists(queryStr string, queryMetadata SearchGistMetadata, gistsIds []u
perPage := 10
offset := (page - 1) * perPage
s := bleve.NewSearchRequestOptions(indexerQuery, perPage, offset, false)
s := bleve.NewSearchRequestOptions(indexerQuery, perPage+1, offset, false)
s.AddFacet("languageFacet", languageFacet)
s.Fields = []string{"GistID"}
s.IncludeLocations = false

6
internal/web/handlers/admin/actions.go
View File

@ -40,3 +40,9 @@ func AdminIndexGists(ctx *context.Context) error {
go actions.Run(actions.IndexGists)
return ctx.RedirectTo("/admin-panel")
}
func AdminSyncGistLanguages(ctx *context.Context) error {
ctx.AddFlash(ctx.Tr("flash.admin.sync-gist-languages"), "success")
go actions.Run(actions.SyncGistLanguages)
return ctx.RedirectTo("/admin-panel")
}

5
internal/web/handlers/admin/admin.go
View File

@ -48,6 +48,7 @@ func AdminIndex(ctx *context.Context) error {
ctx.SetData("syncGistPreviews", actions.IsRunning(actions.SyncGistPreviews))
ctx.SetData("resetHooks", actions.IsRunning(actions.ResetHooks))
ctx.SetData("indexGists", actions.IsRunning(actions.IndexGists))
ctx.SetData("syncGistLanguages", actions.IsRunning(actions.SyncGistLanguages))
return ctx.Html("admin_index.html")
}
@ -64,7 +65,7 @@ func AdminUsers(ctx *context.Context) error {
return ctx.ErrorRes(500, "Cannot get users", err)
}
if err = handlers.Paginate(ctx, data, pageInt, 10, "data", "admin-panel/users", 1); err != nil {
if err = handlers.Paginate(ctx, data, pageInt, 10, "data", "admin-panel/users", 1, nil); err != nil {
return ctx.ErrorRes(404, ctx.Tr("error.page-not-found"), nil)
}
@ -82,7 +83,7 @@ func AdminGists(ctx *context.Context) error {
return ctx.ErrorRes(500, "Cannot get gists", err)
}
if err = handlers.Paginate(ctx, data, pageInt, 10, "data", "admin-panel/gists", 1); err != nil {
if err = handlers.Paginate(ctx, data, pageInt, 10, "data", "admin-panel/gists", 1, nil); err != nil {
return ctx.ErrorRes(404, ctx.Tr("error.page-not-found"), nil)
}

66
internal/web/handlers/gist/all.go
View File

@ -9,7 +9,8 @@ import (
"github.com/thomiceli/opengist/internal/web/context"
"github.com/thomiceli/opengist/internal/web/handlers"
"gorm.io/gorm"
"html/template"
"slices"
"strings"
)
func AllGists(ctx *context.Context) error {
@ -35,6 +36,11 @@ func AllGists(ctx *context.Context) error {
orderText = ctx.TrH("gist.list.order-by-asc")
}
pagination := &handlers.PaginationParams{
Sort: sort,
Order: order,
}
ctx.SetData("sort", sortText)
ctx.SetData("order", orderText)
@ -51,7 +57,7 @@ func AllGists(ctx *context.Context) error {
if mode == "search" {
ctx.SetData("htmlTitle", ctx.TrH("gist.list.search-results"))
ctx.SetData("searchQuery", ctx.QueryParam("q"))
ctx.SetData("searchQueryUrl", template.URL("&q="+ctx.QueryParam("q")))
pagination.Query = ctx.QueryParam("q")
urlPage = "search"
gists, err = db.GetAllGistsFromSearch(currentUserId, ctx.QueryParam("q"), pageInt-1, sort, order, "")
} else if mode == "topics" {
@ -66,6 +72,7 @@ func AllGists(ctx *context.Context) error {
}
} else {
var fromUser *db.User
var count int64
fromUser, err = db.GetUserByUsername(fromUserStr)
if err != nil {
@ -104,10 +111,39 @@ func AllGists(ctx *context.Context) error {
gists, err = db.GetAllGistsForkedByUser(fromUser.ID, currentUserId, pageInt-1, sort, order)
} else if mode == "fromUser" {
urlPage = fromUserStr
if languages, err := db.GetGistLanguagesForUser(fromUser.ID, currentUserId); err != nil {
return ctx.ErrorRes(500, "Error fetching languages", err)
} else {
ctx.SetData("languages", languages)
}
title := ctx.QueryParam("title")
language := ctx.QueryParam("language")
visibility := ctx.QueryParam("visibility")
topicsStr := ctx.QueryParam("topics")
topics := strings.Fields(topicsStr)
if len(topics) > 10 {
topics = topics[:10]
}
slices.Sort(topics)
topics = slices.Compact(topics)
pagination.Title = title
pagination.Language = language
pagination.Visibility = visibility
pagination.Topics = topicsStr
ctx.SetData("title", title)
ctx.SetData("language", language)
ctx.SetData("visibility", visibility)
ctx.SetData("topics", topicsStr)
ctx.SetData("htmlTitle", ctx.TrH("gist.list.all-from", fromUserStr))
gists, err = db.GetAllGistsFromUser(fromUser.ID, currentUserId, pageInt-1, sort, order)
gists, count, err = db.GetAllGistsFromUser(fromUser.ID, currentUserId, title, language, visibility, topics, pageInt-1, sort, order)
ctx.SetData("countFromUser", count)
}
}
if err != nil {
return ctx.ErrorRes(500, "Error fetching gists", err)
}
renderedGists := make([]*render.RenderedGist, 0, len(gists))
for _, gist := range gists {
@ -118,21 +154,20 @@ func AllGists(ctx *context.Context) error {
renderedGists = append(renderedGists, &rendered)
}
if err != nil {
return ctx.ErrorRes(500, "Error fetching gists", err)
}
if err = handlers.Paginate(ctx, renderedGists, pageInt, 10, "gists", fromUserStr, 2, "&sort="+sort+"&order="+order); err != nil {
if err = handlers.Paginate(ctx, renderedGists, pageInt, 10, "gists", urlPage, 2, pagination); err != nil {
return ctx.ErrorRes(404, ctx.Tr("error.page-not-found"), nil)
}
ctx.SetData("urlPage", urlPage)
return ctx.Html("all.html")
}
func Search(ctx *context.Context) error {
var err error
pagination := &handlers.PaginationParams{
Query: ctx.QueryParam("q"),
}
content, meta := handlers.ParseSearchQueryStr(ctx.QueryParam("q"))
pageInt := handlers.GetPage(ctx)
@ -176,19 +211,12 @@ func Search(ctx *context.Context) error {
renderedGists = append(renderedGists, &rendered)
}
if pageInt > 1 && len(renderedGists) != 0 {
ctx.SetData("prevPage", pageInt-1)
if err = handlers.Paginate(ctx, renderedGists, pageInt, 10, "gists", "search", 2, pagination); err != nil {
return ctx.ErrorRes(404, ctx.Tr("error.page-not-found"), nil)
}
if 10*pageInt < int(nbHits) {
ctx.SetData("nextPage", pageInt+1)
}
ctx.SetData("prevLabel", ctx.TrH("pagination.previous"))
ctx.SetData("nextLabel", ctx.TrH("pagination.next"))
ctx.SetData("urlPage", "search")
ctx.SetData("urlParams", template.URL("&q="+ctx.QueryParam("q")))
ctx.SetData("htmlTitle", ctx.TrH("gist.list.search-results"))
ctx.SetData("nbHits", nbHits)
ctx.SetData("gists", renderedGists)
ctx.SetData("langs", langs)
ctx.SetData("searchQuery", ctx.QueryParam("q"))
return ctx.Html("search.html")

1
internal/web/handlers/gist/create.go
View File

@ -137,6 +137,7 @@ func ProcessCreate(ctx *context.Context) error {
}
gist.AddInIndex()
gist.UpdateLanguages()
return ctx.RedirectTo("/" + user.Username + "/" + gist.Identifier())
}

2
internal/web/handlers/gist/fork.go
View File

@ -77,7 +77,7 @@ func Forks(ctx *context.Context) error {
return ctx.ErrorRes(500, "Error getting users who liked this gist", err)
}
if err = handlers.Paginate(ctx, forks, pageInt, 30, "forks", gist.User.Username+"/"+gist.Identifier()+"/forks", 2); err != nil {
if err = handlers.Paginate(ctx, forks, pageInt, 30, "forks", gist.User.Username+"/"+gist.Identifier()+"/forks", 2, nil); err != nil {
return ctx.ErrorRes(404, ctx.Tr("error.page-not-found"), nil)
}

2
internal/web/handlers/gist/like.go
View File

@ -42,7 +42,7 @@ func Likes(ctx *context.Context) error {
return ctx.ErrorRes(500, "Error getting users who liked this gist", err)
}
if err = handlers.Paginate(ctx, likers, pageInt, 30, "likers", gist.User.Username+"/"+gist.Identifier()+"/likes", 1); err != nil {
if err = handlers.Paginate(ctx, likers, pageInt, 30, "likers", gist.User.Username+"/"+gist.Identifier()+"/likes", 1, nil); err != nil {
return ctx.ErrorRes(404, ctx.Tr("error.page-not-found"), nil)
}

2
internal/web/handlers/gist/revisions.go
View File

@ -19,7 +19,7 @@ func Revisions(ctx *context.Context) error {
return ctx.ErrorRes(500, "Error fetching commits log", err)
}
if err := handlers.Paginate(ctx, commits, pageInt, 10, "commits", userName+"/"+gistName+"/revisions", 2); err != nil {
if err := handlers.Paginate(ctx, commits, pageInt, 10, "commits", userName+"/"+gistName+"/revisions", 2, nil); err != nil {
return ctx.ErrorRes(404, ctx.Tr("error.page-not-found"), nil)
}

73
internal/web/handlers/util.go
View File

@ -2,7 +2,9 @@ package handlers
import (
"errors"
"github.com/gorilla/schema"
"html/template"
"net/url"
"path/filepath"
"strconv"
"strings"
@ -24,7 +26,68 @@ func GetPage(ctx *context.Context) int {
return pageInt
}
func Paginate[T any](ctx *context.Context, data []*T, pageInt int, perPage int, templateDataName string, urlPage string, labels int, urlParams ...string) error {
type PaginationParams struct {
Page int `schema:"page,omitempty"`
Sort string `schema:"sort,omitempty"`
Order string `schema:"order,omitempty"`
Title string `schema:"title,omitempty"`
Visibility string `schema:"visibility,omitempty"`
Language string `schema:"language,omitempty"`
Topics string `schema:"topics,omitempty"`
Query string `schema:"q,omitempty"`
HasPrevious bool `schema:"-"` // Exclude from URL parameters
HasNext bool `schema:"-"`
}
var encoder = schema.NewEncoder()
func (p PaginationParams) String() string {
values := url.Values{}
err := encoder.Encode(p, values)
if err != nil {
return ""
}
if len(values) == 0 {
return ""
}
return "?" + values.Encode()
}
func (p PaginationParams) NextURL() template.URL {
p.Page++
return template.URL(p.String())
}
func (p PaginationParams) PreviousURL() template.URL {
p.Page--
return template.URL(p.String())
}
func (p PaginationParams) WithParams(pairs ...string) template.URL {
values := url.Values{}
_ = encoder.Encode(p, values)
// reset page
values.Del("page")
for i := 0; i < len(pairs); i += 2 {
values.Set(pairs[i], pairs[i+1])
}
return template.URL("?" + values.Encode())
}
func Paginate[T any](ctx *context.Context, data []*T, pageInt int, perPage int, templateDataName string, urlPage string, labels int, params *PaginationParams) error {
var paginationParams PaginationParams
if params == nil {
paginationParams = PaginationParams{}
} else {
paginationParams = *params
}
paginationParams.Page = pageInt
lenData := len(data)
if lenData == 0 && pageInt != 1 {
return errors.New("page not found")
@ -34,15 +97,13 @@ func Paginate[T any](ctx *context.Context, data []*T, pageInt int, perPage int,
if lenData > 1 {
data = data[:lenData-1]
}
ctx.SetData("nextPage", pageInt+1)
paginationParams.HasNext = true
}
if pageInt > 1 {
ctx.SetData("prevPage", pageInt-1)
paginationParams.HasPrevious = true
}
if len(urlParams) > 0 {
ctx.SetData("urlParams", template.URL(urlParams[0]))
}
ctx.SetData("pagination", paginationParams)
switch labels {
case 1:

1
internal/web/server/router.go
View File

@ -82,6 +82,7 @@ func (s *Server) registerRoutes() {
sB.POST("/sync-previews", admin.AdminSyncGistPreviews)
sB.POST("/reset-hooks", admin.AdminResetHooks)
sB.POST("/index-gists", admin.AdminIndexGists)
sB.POST("/sync-languages", admin.AdminSyncGistLanguages)
sB.GET("/configuration", admin.AdminConfig)
sB.PUT("/set-config", admin.AdminSetConfig)
}