Add support for incoming emails (#22056)

closes #13585
fixes #9067
fixes #2386
ref #6226
ref #6219
fixes #745

This PR adds support to process incoming emails to perform actions.
Currently I added handling of replies and unsubscribing from
issues/pulls. In contrast to #13585 the IMAP IDLE command is used
instead of polling which results (in my opinion 😉) in cleaner code.

Procedure:
- When sending an issue/pull reply email, a token is generated which is
present in the Reply-To and References header.
- IMAP IDLE waits until a new email arrives
- The token tells which action should be performed

A possible signature and/or reply gets stripped from the content.

I added a new service to the drone pipeline to test the receiving of
incoming mails. If we keep this in, we may test our outgoing emails too
in future.

Co-authored-by: silverwind <me@silverwind.io>
Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>

services/mailer/incoming/incoming.go

@@ -0,0 +1,375 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package incoming
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	net_mail "net/mail"
+	"regexp"
+	"strings"
+	"time"
+
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/process"
+	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/services/mailer/token"
+
+	"github.com/dimiro1/reply"
+	"github.com/emersion/go-imap"
+	"github.com/emersion/go-imap/client"
+	"github.com/jhillyerd/enmime"
+)
+
+var (
+	addressTokenRegex   *regexp.Regexp
+	referenceTokenRegex *regexp.Regexp
+)
+
+func Init(ctx context.Context) error {
+	if !setting.IncomingEmail.Enabled {
+		return nil
+	}
+
+	var err error
+	addressTokenRegex, err = regexp.Compile(
+		fmt.Sprintf(
+			`\A%s\z`,
+			strings.Replace(regexp.QuoteMeta(setting.IncomingEmail.ReplyToAddress), regexp.QuoteMeta(setting.IncomingEmail.TokenPlaceholder), "(.+)", 1),
+		),
+	)
+	if err != nil {
+		return err
+	}
+	referenceTokenRegex, err = regexp.Compile(fmt.Sprintf(`\Areply-(.+)@%s\z`, regexp.QuoteMeta(setting.Domain)))
+	if err != nil {
+		return err
+	}
+
+	go func() {
+		ctx, _, finished := process.GetManager().AddTypedContext(ctx, "Incoming Email", process.SystemProcessType, true)
+		defer finished()
+
+		// This background job processes incoming emails. It uses the IMAP IDLE command to get notified about incoming emails.
+		// The following loop restarts the processing logic after errors until ctx indicates to stop.
+
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			default:
+				if err := processIncomingEmails(ctx); err != nil {
+					log.Error("Error while processing incoming emails: %v", err)
+				}
+				select {
+				case <-ctx.Done():
+					return
+				case <-time.NewTimer(10 * time.Second).C:
+				}
+			}
+		}
+	}()
+
+	return nil
+}
+
+// processIncomingEmails is the "main" method with the wait/process loop
+func processIncomingEmails(ctx context.Context) error {
+	server := fmt.Sprintf("%s:%d", setting.IncomingEmail.Host, setting.IncomingEmail.Port)
+
+	var c *client.Client
+	var err error
+	if setting.IncomingEmail.UseTLS {
+		c, err = client.DialTLS(server, &tls.Config{InsecureSkipVerify: setting.IncomingEmail.SkipTLSVerify})
+	} else {
+		c, err = client.Dial(server)
+	}
+	if err != nil {
+		return fmt.Errorf("could not connect to server '%s': %w", server, err)
+	}
+
+	if err := c.Login(setting.IncomingEmail.Username, setting.IncomingEmail.Password); err != nil {
+		return fmt.Errorf("could not login: %w", err)
+	}
+	defer func() {
+		if err := c.Logout(); err != nil {
+			log.Error("Logout from incoming email server failed: %v", err)
+		}
+	}()
+
+	if _, err := c.Select(setting.IncomingEmail.Mailbox, false); err != nil {
+		return fmt.Errorf("selecting box '%s' failed: %w", setting.IncomingEmail.Mailbox, err)
+	}
+
+	// The following loop processes messages. If there are no messages available, IMAP IDLE is used to wait for new messages.
+	// This process is repeated until an IMAP error occurs or ctx indicates to stop.
+
+	for {
+		select {
+		case <-ctx.Done():
+			return nil
+		default:
+			if err := processMessages(ctx, c); err != nil {
+				return fmt.Errorf("could not process messages: %w", err)
+			}
+			if err := waitForUpdates(ctx, c); err != nil {
+				return fmt.Errorf("wait for updates failed: %w", err)
+			}
+			select {
+			case <-ctx.Done():
+				return nil
+			case <-time.NewTimer(time.Second).C:
+			}
+		}
+	}
+}
+
+// waitForUpdates uses IMAP IDLE to wait for new emails
+func waitForUpdates(ctx context.Context, c *client.Client) error {
+	updates := make(chan client.Update, 1)
+
+	c.Updates = updates
+	defer func() {
+		c.Updates = nil
+	}()
+
+	errs := make(chan error, 1)
+	stop := make(chan struct{})
+	go func() {
+		errs <- c.Idle(stop, nil)
+	}()
+
+	stopped := false
+	for {
+		select {
+		case update := <-updates:
+			switch update.(type) {
+			case *client.MailboxUpdate:
+				if !stopped {
+					close(stop)
+					stopped = true
+				}
+			default:
+			}
+		case err := <-errs:
+			if err != nil {
+				return fmt.Errorf("imap idle failed: %w", err)
+			}
+			return nil
+		case <-ctx.Done():
+			return nil
+		}
+	}
+}
+
+// processMessages searches unread mails and processes them.
+func processMessages(ctx context.Context, c *client.Client) error {
+	criteria := imap.NewSearchCriteria()
+	criteria.WithoutFlags = []string{imap.SeenFlag}
+	criteria.Smaller = setting.IncomingEmail.MaximumMessageSize
+	ids, err := c.Search(criteria)
+	if err != nil {
+		return fmt.Errorf("imap search failed: %w", err)
+	}
+
+	if len(ids) == 0 {
+		return nil
+	}
+
+	seqset := new(imap.SeqSet)
+	seqset.AddNum(ids...)
+	messages := make(chan *imap.Message, 10)
+
+	section := &imap.BodySectionName{}
+
+	errs := make(chan error, 1)
+	go func() {
+		errs <- c.Fetch(
+			seqset,
+			[]imap.FetchItem{section.FetchItem()},
+			messages,
+		)
+	}()
+
+	handledSet := new(imap.SeqSet)
+loop:
+	for {
+		select {
+		case <-ctx.Done():
+			break loop
+		case msg, ok := <-messages:
+			if !ok {
+				if setting.IncomingEmail.DeleteHandledMessage && !handledSet.Empty() {
+					if err := c.Store(
+						handledSet,
+						imap.FormatFlagsOp(imap.AddFlags, true),
+						[]interface{}{imap.DeletedFlag},
+						nil,
+					); err != nil {
+						return fmt.Errorf("imap store failed: %w", err)
+					}
+
+					if err := c.Expunge(nil); err != nil {
+						return fmt.Errorf("imap expunge failed: %w", err)
+					}
+				}
+				return nil
+			}
+
+			err := func() error {
+				r := msg.GetBody(section)
+				if r == nil {
+					return fmt.Errorf("could not get body from message: %w", err)
+				}
+
+				env, err := enmime.ReadEnvelope(r)
+				if err != nil {
+					return fmt.Errorf("could not read envelope: %w", err)
+				}
+
+				if isAutomaticReply(env) {
+					log.Debug("Skipping automatic email reply")
+					return nil
+				}
+
+				t := searchTokenInHeaders(env)
+				if t == "" {
+					log.Debug("Incoming email token not found in headers")
+					return nil
+				}
+
+				handlerType, user, payload, err := token.ExtractToken(ctx, t)
+				if err != nil {
+					if _, ok := err.(*token.ErrToken); ok {
+						log.Info("Invalid incoming email token: %v", err)
+						return nil
+					}
+					return err
+				}
+
+				handler, ok := handlers[handlerType]
+				if !ok {
+					return fmt.Errorf("unexpected handler type: %v", handlerType)
+				}
+
+				content := getContentFromMailReader(env)
+
+				if err := handler.Handle(ctx, content, user, payload); err != nil {
+					return fmt.Errorf("could not handle message: %w", err)
+				}
+
+				handledSet.AddNum(msg.SeqNum)
+
+				return nil
+			}()
+			if err != nil {
+				log.Error("Error while processing incoming email[%v]: %v", msg.Uid, err)
+			}
+		}
+	}
+
+	if err := <-errs; err != nil {
+		return fmt.Errorf("imap fetch failed: %w", err)
+	}
+
+	return nil
+}
+
+// isAutomaticReply tests if the headers indicate an automatic reply
+func isAutomaticReply(env *enmime.Envelope) bool {
+	autoSubmitted := env.GetHeader("Auto-Submitted")
+	if autoSubmitted != "" && autoSubmitted != "no" {
+		return true
+	}
+	autoReply := env.GetHeader("X-Autoreply")
+	if autoReply == "yes" {
+		return true
+	}
+	autoRespond := env.GetHeader("X-Autorespond")
+	return autoRespond != ""
+}
+
+// searchTokenInHeaders looks for the token in To, Delivered-To and References
+func searchTokenInHeaders(env *enmime.Envelope) string {
+	if addressTokenRegex != nil {
+		to, _ := env.AddressList("To")
+
+		token := searchTokenInAddresses(to)
+		if token != "" {
+			return token
+		}
+
+		deliveredTo, _ := env.AddressList("Delivered-To")
+
+		token = searchTokenInAddresses(deliveredTo)
+		if token != "" {
+			return token
+		}
+	}
+
+	references := env.GetHeader("References")
+	for {
+		begin := strings.IndexByte(references, '<')
+		if begin == -1 {
+			break
+		}
+		begin++
+
+		end := strings.IndexByte(references, '>')
+		if end == -1 || begin > end {
+			break
+		}
+
+		match := referenceTokenRegex.FindStringSubmatch(references[begin:end])
+		if len(match) == 2 {
+			return match[1]
+		}
+
+		references = references[end+1:]
+	}
+
+	return ""
+}
+
+// searchTokenInAddresses looks for the token in an address
+func searchTokenInAddresses(addresses []*net_mail.Address) string {
+	for _, address := range addresses {
+		match := addressTokenRegex.FindStringSubmatch(address.Address)
+		if len(match) != 2 {
+			continue
+		}
+
+		return match[1]
+	}
+
+	return ""
+}
+
+type MailContent struct {
+	Content     string
+	Attachments []*Attachment
+}
+
+type Attachment struct {
+	Name    string
+	Content []byte
+}
+
+// getContentFromMailReader grabs the plain content and the attachments from the mail.
+// A potential reply/signature gets stripped from the content.
+func getContentFromMailReader(env *enmime.Envelope) *MailContent {
+	attachments := make([]*Attachment, 0, len(env.Attachments))
+	for _, attachment := range env.Attachments {
+		attachments = append(attachments, &Attachment{
+			Name:    attachment.FileName,
+			Content: attachment.Content,
+		})
+	}
+
+	return &MailContent{
+		Content:     reply.FromText(env.Text),
+		Attachments: attachments,
+	}
+}

services/mailer/incoming/incoming_handler.go

@@ -0,0 +1,171 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package incoming
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+
+	issues_model "code.gitea.io/gitea/models/issues"
+	access_model "code.gitea.io/gitea/models/perm/access"
+	repo_model "code.gitea.io/gitea/models/repo"
+	user_model "code.gitea.io/gitea/models/user"
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/upload"
+	"code.gitea.io/gitea/modules/util"
+	attachment_service "code.gitea.io/gitea/services/attachment"
+	issue_service "code.gitea.io/gitea/services/issue"
+	incoming_payload "code.gitea.io/gitea/services/mailer/incoming/payload"
+	"code.gitea.io/gitea/services/mailer/token"
+	pull_service "code.gitea.io/gitea/services/pull"
+)
+
+type MailHandler interface {
+	Handle(ctx context.Context, content *MailContent, doer *user_model.User, payload []byte) error
+}
+
+var handlers = map[token.HandlerType]MailHandler{
+	token.ReplyHandlerType:       &ReplyHandler{},
+	token.UnsubscribeHandlerType: &UnsubscribeHandler{},
+}
+
+// ReplyHandler handles incoming emails to create a reply from them
+type ReplyHandler struct{}
+
+func (h *ReplyHandler) Handle(ctx context.Context, content *MailContent, doer *user_model.User, payload []byte) error {
+	if doer == nil {
+		return util.NewInvalidArgumentErrorf("doer can't be nil")
+	}
+
+	ref, err := incoming_payload.GetReferenceFromPayload(ctx, payload)
+	if err != nil {
+		return err
+	}
+
+	var issue *issues_model.Issue
+
+	switch r := ref.(type) {
+	case *issues_model.Issue:
+		issue = r
+	case *issues_model.Comment:
+		comment := r
+
+		if err := comment.LoadIssue(ctx); err != nil {
+			return err
+		}
+
+		issue = comment.Issue
+	default:
+		return util.NewInvalidArgumentErrorf("unsupported reply reference: %v", ref)
+	}
+
+	if err := issue.LoadRepo(ctx); err != nil {
+		return err
+	}
+
+	perm, err := access_model.GetUserRepoPermission(ctx, issue.Repo, doer)
+	if err != nil {
+		return err
+	}
+
+	if !perm.CanWriteIssuesOrPulls(issue.IsPull) || issue.IsLocked && !doer.IsAdmin {
+		log.Debug("can't write issue or pull")
+		return nil
+	}
+
+	switch r := ref.(type) {
+	case *issues_model.Issue:
+		attachmentIDs := make([]string, 0, len(content.Attachments))
+		if setting.Attachment.Enabled {
+			for _, attachment := range content.Attachments {
+				a, err := attachment_service.UploadAttachment(bytes.NewReader(attachment.Content), setting.Attachment.AllowedTypes, &repo_model.Attachment{
+					Name:       attachment.Name,
+					UploaderID: doer.ID,
+					RepoID:     issue.Repo.ID,
+				})
+				if err != nil {
+					if upload.IsErrFileTypeForbidden(err) {
+						log.Info("Skipping disallowed attachment type: %s", attachment.Name)
+						continue
+					}
+					return err
+				}
+				attachmentIDs = append(attachmentIDs, a.UUID)
+			}
+		}
+
+		if content.Content == "" && len(attachmentIDs) == 0 {
+			return nil
+		}
+
+		_, err = issue_service.CreateIssueComment(ctx, doer, issue.Repo, issue, content.Content, attachmentIDs)
+		if err != nil {
+			return fmt.Errorf("CreateIssueComment failed: %w", err)
+		}
+	case *issues_model.Comment:
+		comment := r
+
+		if content.Content == "" {
+			return nil
+		}
+
+		if comment.Type == issues_model.CommentTypeCode {
+			_, err := pull_service.CreateCodeComment(
+				ctx,
+				doer,
+				nil,
+				issue,
+				comment.Line,
+				content.Content,
+				comment.TreePath,
+				false,
+				comment.ReviewID,
+				"",
+			)
+			if err != nil {
+				return fmt.Errorf("CreateCodeComment failed: %w", err)
+			}
+		}
+	}
+	return nil
+}
+
+// UnsubscribeHandler handles unwatching issues/pulls
+type UnsubscribeHandler struct{}
+
+func (h *UnsubscribeHandler) Handle(ctx context.Context, _ *MailContent, doer *user_model.User, payload []byte) error {
+	if doer == nil {
+		return util.NewInvalidArgumentErrorf("doer can't be nil")
+	}
+
+	ref, err := incoming_payload.GetReferenceFromPayload(ctx, payload)
+	if err != nil {
+		return err
+	}
+
+	switch r := ref.(type) {
+	case *issues_model.Issue:
+		issue := r
+
+		if err := issue.LoadRepo(ctx); err != nil {
+			return err
+		}
+
+		perm, err := access_model.GetUserRepoPermission(ctx, issue.Repo, doer)
+		if err != nil {
+			return err
+		}
+
+		if !perm.CanReadIssuesOrPulls(issue.IsPull) {
+			log.Debug("can't read issue or pull")
+			return nil
+		}
+
+		return issues_model.CreateOrUpdateIssueWatch(doer.ID, issue.ID, false)
+	}
+
+	return fmt.Errorf("unsupported unsubscribe reference: %v", ref)
+}

services/mailer/incoming/incoming_test.go

@@ -0,0 +1,138 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package incoming
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/jhillyerd/enmime"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestIsAutomaticReply(t *testing.T) {
+	cases := []struct {
+		Headers  map[string]string
+		Expected bool
+	}{
+		{
+			Headers:  map[string]string{},
+			Expected: false,
+		},
+		{
+			Headers: map[string]string{
+				"Auto-Submitted": "no",
+			},
+			Expected: false,
+		},
+		{
+			Headers: map[string]string{
+				"Auto-Submitted": "yes",
+			},
+			Expected: true,
+		},
+		{
+			Headers: map[string]string{
+				"X-Autoreply": "no",
+			},
+			Expected: false,
+		},
+		{
+			Headers: map[string]string{
+				"X-Autoreply": "yes",
+			},
+			Expected: true,
+		},
+		{
+			Headers: map[string]string{
+				"X-Autorespond": "yes",
+			},
+			Expected: true,
+		},
+	}
+
+	for _, c := range cases {
+		b := enmime.Builder().
+			From("Dummy", "dummy@gitea.io").
+			To("Dummy", "dummy@gitea.io")
+		for k, v := range c.Headers {
+			b = b.Header(k, v)
+		}
+		root, err := b.Build()
+		assert.NoError(t, err)
+		env, err := enmime.EnvelopeFromPart(root)
+		assert.NoError(t, err)
+
+		assert.Equal(t, c.Expected, isAutomaticReply(env))
+	}
+}
+
+func TestGetContentFromMailReader(t *testing.T) {
+	mailString := "Content-Type: multipart/mixed; boundary=message-boundary\r\n" +
+		"\r\n" +
+		"--message-boundary\r\n" +
+		"Content-Type: multipart/alternative; boundary=text-boundary\r\n" +
+		"\r\n" +
+		"--text-boundary\r\n" +
+		"Content-Type: text/plain\r\n" +
+		"Content-Disposition: inline\r\n" +
+		"\r\n" +
+		"mail content\r\n" +
+		"--text-boundary--\r\n" +
+		"--message-boundary\r\n" +
+		"Content-Type: text/plain\r\n" +
+		"Content-Disposition: attachment; filename=attachment.txt\r\n" +
+		"\r\n" +
+		"attachment content\r\n" +
+		"--message-boundary--\r\n"
+
+	env, err := enmime.ReadEnvelope(strings.NewReader(mailString))
+	assert.NoError(t, err)
+	content := getContentFromMailReader(env)
+	assert.Equal(t, "mail content", content.Content)
+	assert.Len(t, content.Attachments, 1)
+	assert.Equal(t, "attachment.txt", content.Attachments[0].Name)
+	assert.Equal(t, []byte("attachment content"), content.Attachments[0].Content)
+
+	mailString = "Content-Type: multipart/mixed; boundary=message-boundary\r\n" +
+		"\r\n" +
+		"--message-boundary\r\n" +
+		"Content-Type: multipart/alternative; boundary=text-boundary\r\n" +
+		"\r\n" +
+		"--text-boundary\r\n" +
+		"Content-Type: text/html\r\n" +
+		"Content-Disposition: inline\r\n" +
+		"\r\n" +
+		"<p>mail content</p>\r\n" +
+		"--text-boundary--\r\n" +
+		"--message-boundary--\r\n"
+
+	env, err = enmime.ReadEnvelope(strings.NewReader(mailString))
+	assert.NoError(t, err)
+	content = getContentFromMailReader(env)
+	assert.Equal(t, "mail content", content.Content)
+	assert.Empty(t, content.Attachments)
+
+	mailString = "Content-Type: multipart/mixed; boundary=message-boundary\r\n" +
+		"\r\n" +
+		"--message-boundary\r\n" +
+		"Content-Type: multipart/alternative; boundary=text-boundary\r\n" +
+		"\r\n" +
+		"--text-boundary\r\n" +
+		"Content-Type: text/plain\r\n" +
+		"Content-Disposition: inline\r\n" +
+		"\r\n" +
+		"mail content without signature\r\n" +
+		"--\r\n" +
+		"signature\r\n" +
+		"--text-boundary--\r\n" +
+		"--message-boundary--\r\n"
+
+	env, err = enmime.ReadEnvelope(strings.NewReader(mailString))
+	assert.NoError(t, err)
+	content = getContentFromMailReader(env)
+	assert.NoError(t, err)
+	assert.Equal(t, "mail content without signature", content.Content)
+	assert.Empty(t, content.Attachments)
+}

services/mailer/incoming/payload/payload.go

@@ -0,0 +1,70 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package payload
+
+import (
+	"context"
+
+	issues_model "code.gitea.io/gitea/models/issues"
+	"code.gitea.io/gitea/modules/util"
+)
+
+const replyPayloadVersion1 byte = 1
+
+type payloadReferenceType byte
+
+const (
+	payloadReferenceIssue payloadReferenceType = iota
+	payloadReferenceComment
+)
+
+// CreateReferencePayload creates data which GetReferenceFromPayload resolves to the reference again.
+func CreateReferencePayload(reference interface{}) ([]byte, error) {
+	var refType payloadReferenceType
+	var refID int64
+
+	switch r := reference.(type) {
+	case *issues_model.Issue:
+		refType = payloadReferenceIssue
+		refID = r.ID
+	case *issues_model.Comment:
+		refType = payloadReferenceComment
+		refID = r.ID
+	default:
+		return nil, util.NewInvalidArgumentErrorf("unsupported reference type: %T", r)
+	}
+
+	payload, err := util.PackData(refType, refID)
+	if err != nil {
+		return nil, err
+	}
+
+	return append([]byte{replyPayloadVersion1}, payload...), nil
+}
+
+// GetReferenceFromPayload resolves the reference from the payload
+func GetReferenceFromPayload(ctx context.Context, payload []byte) (interface{}, error) {
+	if len(payload) < 1 {
+		return nil, util.NewInvalidArgumentErrorf("payload to small")
+	}
+
+	if payload[0] != replyPayloadVersion1 {
+		return nil, util.NewInvalidArgumentErrorf("unsupported payload version")
+	}
+
+	var ref payloadReferenceType
+	var id int64
+	if err := util.UnpackData(payload[1:], &ref, &id); err != nil {
+		return nil, err
+	}
+
+	switch ref {
+	case payloadReferenceIssue:
+		return issues_model.GetIssueByID(ctx, id)
+	case payloadReferenceComment:
+		return issues_model.GetCommentByID(ctx, id)
+	default:
+		return nil, util.NewInvalidArgumentErrorf("unsupported reference type: %T", ref)
+	}
+}

services/mailer/mail.go

@@ -29,6 +29,8 @@ import (
 	"code.gitea.io/gitea/modules/templates"
 	"code.gitea.io/gitea/modules/timeutil"
 	"code.gitea.io/gitea/modules/translation"
+	incoming_payload "code.gitea.io/gitea/services/mailer/incoming/payload"
+	"code.gitea.io/gitea/services/mailer/token"
 
 	"gopkg.in/gomail.v2"
 )
@@ -302,14 +304,57 @@ func composeIssueCommentMessages(ctx *mailCommentContext, lang string, recipient
 	msgID := createReference(ctx.Issue, ctx.Comment, ctx.ActionType)
 	reference := createReference(ctx.Issue, nil, activities_model.ActionType(0))
 
+	var replyPayload []byte
+	if ctx.Comment != nil && ctx.Comment.Type == issues_model.CommentTypeCode {
+		replyPayload, err = incoming_payload.CreateReferencePayload(ctx.Comment)
+	} else {
+		replyPayload, err = incoming_payload.CreateReferencePayload(ctx.Issue)
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	unsubscribePayload, err := incoming_payload.CreateReferencePayload(ctx.Issue)
+	if err != nil {
+		return nil, err
+	}
+
 	msgs := make([]*Message, 0, len(recipients))
 	for _, recipient := range recipients {
 		msg := NewMessageFrom([]string{recipient.Email}, ctx.Doer.DisplayName(), setting.MailService.FromEmail, subject, mailBody.String())
 		msg.Info = fmt.Sprintf("Subject: %s, %s", subject, info)
 
-		msg.SetHeader("Message-ID", "<"+msgID+">")
-		msg.SetHeader("In-Reply-To", "<"+reference+">")
-		msg.SetHeader("References", "<"+reference+">")
+		msg.SetHeader("Message-ID", msgID)
+		msg.SetHeader("In-Reply-To", reference)
+
+		references := []string{reference}
+		listUnsubscribe := []string{"<" + ctx.Issue.HTMLURL() + ">"}
+
+		if setting.IncomingEmail.Enabled {
+			if ctx.Comment != nil {
+				token, err := token.CreateToken(token.ReplyHandlerType, recipient, replyPayload)
+				if err != nil {
+					log.Error("CreateToken failed: %v", err)
+				} else {
+					replyAddress := strings.Replace(setting.IncomingEmail.ReplyToAddress, setting.IncomingEmail.TokenPlaceholder, token, 1)
+					msg.ReplyTo = replyAddress
+					msg.SetHeader("List-Post", fmt.Sprintf("<mailto:%s>", replyAddress))
+
+					references = append(references, fmt.Sprintf("<reply-%s@%s>", token, setting.Domain))
+				}
+			}
+
+			token, err := token.CreateToken(token.UnsubscribeHandlerType, recipient, unsubscribePayload)
+			if err != nil {
+				log.Error("CreateToken failed: %v", err)
+			} else {
+				unsubAddress := strings.Replace(setting.IncomingEmail.ReplyToAddress, setting.IncomingEmail.TokenPlaceholder, token, 1)
+				listUnsubscribe = append(listUnsubscribe, "<mailto:"+unsubAddress+">")
+			}
+		}
+
+		msg.SetHeader("References", references...)
+		msg.SetHeader("List-Unsubscribe", listUnsubscribe...)
 
 		for key, value := range generateAdditionalHeaders(ctx, actType, recipient) {
 			msg.SetHeader(key, value)
@@ -345,7 +390,7 @@ func createReference(issue *issues_model.Issue, comment *issues_model.Comment, a
 		}
 	}
 
-	return fmt.Sprintf("%s/%s/%d%s@%s", issue.Repo.FullName(), path, issue.Index, extra, setting.Domain)
+	return fmt.Sprintf("<%s/%s/%d%s@%s>", issue.Repo.FullName(), path, issue.Index, extra, setting.Domain)
 }
 
 func generateAdditionalHeaders(ctx *mailCommentContext, reason string, recipient *user_model.User) map[string]string {
@@ -357,8 +402,6 @@ func generateAdditionalHeaders(ctx *mailCommentContext, reason string, recipient
 
 		// https://datatracker.ietf.org/doc/html/rfc2369
 		"List-Archive": fmt.Sprintf("<%s>", repo.HTMLURL()),
-		//"List-Post": https://github.com/go-gitea/gitea/pull/13585
-		"List-Unsubscribe": ctx.Issue.HTMLURL(),
 
 		"X-Mailer":                  "Gitea",
 		"X-Gitea-Reason":            reason,

services/mailer/mail_test.go

@@ -8,6 +8,7 @@ import (
 	"context"
 	"fmt"
 	"html/template"
+	"regexp"
 	"strings"
 	"testing"
 	texttmpl "text/template"
@@ -66,6 +67,9 @@ func prepareMailerTest(t *testing.T) (doer *user_model.User, repo *repo_model.Re
 func TestComposeIssueCommentMessage(t *testing.T) {
 	doer, _, issue, comment := prepareMailerTest(t)
 
+	setting.IncomingEmail.Enabled = true
+	defer func() { setting.IncomingEmail.Enabled = false }()
+
 	subjectTemplates = texttmpl.Must(texttmpl.New("issue/comment").Parse(subjectTpl))
 	bodyTemplates = template.Must(template.New("issue/comment").Parse(bodyTpl))
 
@@ -78,18 +82,20 @@ func TestComposeIssueCommentMessage(t *testing.T) {
 	assert.NoError(t, err)
 	assert.Len(t, msgs, 2)
 	gomailMsg := msgs[0].ToMessage()
-	mailto := gomailMsg.GetHeader("To")
-	subject := gomailMsg.GetHeader("Subject")
-	messageID := gomailMsg.GetHeader("Message-ID")
-	inReplyTo := gomailMsg.GetHeader("In-Reply-To")
-	references := gomailMsg.GetHeader("References")
+	replyTo := gomailMsg.GetHeader("Reply-To")[0]
+	subject := gomailMsg.GetHeader("Subject")[0]
 
-	assert.Len(t, mailto, 1, "exactly one recipient is expected in the To field")
-	assert.Equal(t, "Re: ", subject[0][:4], "Comment reply subject should contain Re:")
-	assert.Equal(t, "Re: [user2/repo1] @user2 #1 - issue1", subject[0])
-	assert.Equal(t, "<user2/repo1/issues/1@localhost>", inReplyTo[0], "In-Reply-To header doesn't match")
-	assert.Equal(t, "<user2/repo1/issues/1@localhost>", references[0], "References header doesn't match")
-	assert.Equal(t, "<user2/repo1/issues/1/comment/2@localhost>", messageID[0], "Message-ID header doesn't match")
+	assert.Len(t, gomailMsg.GetHeader("To"), 1, "exactly one recipient is expected in the To field")
+	tokenRegex := regexp.MustCompile(`\Aincoming\+(.+)@localhost\z`)
+	assert.Regexp(t, tokenRegex, replyTo)
+	token := tokenRegex.FindAllStringSubmatch(replyTo, 1)[0][1]
+	assert.Equal(t, "Re: ", subject[:4], "Comment reply subject should contain Re:")
+	assert.Equal(t, "Re: [user2/repo1] @user2 #1 - issue1", subject)
+	assert.Equal(t, "<user2/repo1/issues/1@localhost>", gomailMsg.GetHeader("In-Reply-To")[0], "In-Reply-To header doesn't match")
+	assert.ElementsMatch(t, []string{"<user2/repo1/issues/1@localhost>", "<reply-" + token + "@localhost>"}, gomailMsg.GetHeader("References"), "References header doesn't match")
+	assert.Equal(t, "<user2/repo1/issues/1/comment/2@localhost>", gomailMsg.GetHeader("Message-ID")[0], "Message-ID header doesn't match")
+	assert.Equal(t, "<mailto:"+replyTo+">", gomailMsg.GetHeader("List-Post")[0])
+	assert.Len(t, gomailMsg.GetHeader("List-Unsubscribe"), 2) // url + mailto
 }
 
 func TestComposeIssueMessage(t *testing.T) {
@@ -119,6 +125,8 @@ func TestComposeIssueMessage(t *testing.T) {
 	assert.Equal(t, "<user2/repo1/issues/1@localhost>", inReplyTo[0], "In-Reply-To header doesn't match")
 	assert.Equal(t, "<user2/repo1/issues/1@localhost>", references[0], "References header doesn't match")
 	assert.Equal(t, "<user2/repo1/issues/1@localhost>", messageID[0], "Message-ID header doesn't match")
+	assert.Empty(t, gomailMsg.GetHeader("List-Post"))         // incoming mail feature disabled
+	assert.Len(t, gomailMsg.GetHeader("List-Unsubscribe"), 1) // url without mailto
 }
 
 func TestTemplateSelection(t *testing.T) {
@@ -238,7 +246,6 @@ func TestGenerateAdditionalHeaders(t *testing.T) {
 	expected := map[string]string{
 		"List-ID":                   "user2/repo1 <repo1.user2.localhost>",
 		"List-Archive":              "<https://try.gitea.io/user2/repo1>",
-		"List-Unsubscribe":          "https://try.gitea.io/user2/repo1/issues/1",
 		"X-Gitea-Reason":            "dummy-reason",
 		"X-Gitea-Sender":            "< U<se>r Tw<o > ><",
 		"X-Gitea-Recipient":         "Test",
@@ -271,7 +278,6 @@ func Test_createReference(t *testing.T) {
 		name   string
 		args   args
 		prefix string
-		suffix string
 	}{
 		{
 			name: "Open Issue",
@@ -279,7 +285,7 @@ func Test_createReference(t *testing.T) {
 				issue:      issue,
 				actionType: activities_model.ActionCreateIssue,
 			},
-			prefix: fmt.Sprintf("%s/issues/%d@%s", issue.Repo.FullName(), issue.Index, setting.Domain),
+			prefix: fmt.Sprintf("<%s/issues/%d@%s>", issue.Repo.FullName(), issue.Index, setting.Domain),
 		},
 		{
 			name: "Open Pull",
@@ -287,7 +293,7 @@ func Test_createReference(t *testing.T) {
 				issue:      pullIssue,
 				actionType: activities_model.ActionCreatePullRequest,
 			},
-			prefix: fmt.Sprintf("%s/pulls/%d@%s", issue.Repo.FullName(), issue.Index, setting.Domain),
+			prefix: fmt.Sprintf("<%s/pulls/%d@%s>", issue.Repo.FullName(), issue.Index, setting.Domain),
 		},
 		{
 			name: "Comment Issue",
@@ -296,7 +302,7 @@ func Test_createReference(t *testing.T) {
 				comment:    comment,
 				actionType: activities_model.ActionCommentIssue,
 			},
-			prefix: fmt.Sprintf("%s/issues/%d/comment/%d@%s", issue.Repo.FullName(), issue.Index, comment.ID, setting.Domain),
+			prefix: fmt.Sprintf("<%s/issues/%d/comment/%d@%s>", issue.Repo.FullName(), issue.Index, comment.ID, setting.Domain),
 		},
 		{
 			name: "Comment Pull",
@@ -305,7 +311,7 @@ func Test_createReference(t *testing.T) {
 				comment:    comment,
 				actionType: activities_model.ActionCommentPull,
 			},
-			prefix: fmt.Sprintf("%s/pulls/%d/comment/%d@%s", issue.Repo.FullName(), issue.Index, comment.ID, setting.Domain),
+			prefix: fmt.Sprintf("<%s/pulls/%d/comment/%d@%s>", issue.Repo.FullName(), issue.Index, comment.ID, setting.Domain),
 		},
 		{
 			name: "Close Issue",
@@ -313,7 +319,7 @@ func Test_createReference(t *testing.T) {
 				issue:      issue,
 				actionType: activities_model.ActionCloseIssue,
 			},
-			prefix: fmt.Sprintf("%s/issues/%d/close/", issue.Repo.FullName(), issue.Index),
+			prefix: fmt.Sprintf("<%s/issues/%d/close/", issue.Repo.FullName(), issue.Index),
 		},
 		{
 			name: "Close Pull",
@@ -321,7 +327,7 @@ func Test_createReference(t *testing.T) {
 				issue:      pullIssue,
 				actionType: activities_model.ActionClosePullRequest,
 			},
-			prefix: fmt.Sprintf("%s/pulls/%d/close/", issue.Repo.FullName(), issue.Index),
+			prefix: fmt.Sprintf("<%s/pulls/%d/close/", issue.Repo.FullName(), issue.Index),
 		},
 		{
 			name: "Reopen Issue",
@@ -329,7 +335,7 @@ func Test_createReference(t *testing.T) {
 				issue:      issue,
 				actionType: activities_model.ActionReopenIssue,
 			},
-			prefix: fmt.Sprintf("%s/issues/%d/reopen/", issue.Repo.FullName(), issue.Index),
+			prefix: fmt.Sprintf("<%s/issues/%d/reopen/", issue.Repo.FullName(), issue.Index),
 		},
 		{
 			name: "Reopen Pull",
@@ -337,7 +343,7 @@ func Test_createReference(t *testing.T) {
 				issue:      pullIssue,
 				actionType: activities_model.ActionReopenPullRequest,
 			},
-			prefix: fmt.Sprintf("%s/pulls/%d/reopen/", issue.Repo.FullName(), issue.Index),
+			prefix: fmt.Sprintf("<%s/pulls/%d/reopen/", issue.Repo.FullName(), issue.Index),
 		},
 		{
 			name: "Merge Pull",
@@ -345,7 +351,7 @@ func Test_createReference(t *testing.T) {
 				issue:      pullIssue,
 				actionType: activities_model.ActionMergePullRequest,
 			},
-			prefix: fmt.Sprintf("%s/pulls/%d/merge/", issue.Repo.FullName(), issue.Index),
+			prefix: fmt.Sprintf("<%s/pulls/%d/merge/", issue.Repo.FullName(), issue.Index),
 		},
 		{
 			name: "Ready Pull",
@@ -353,7 +359,7 @@ func Test_createReference(t *testing.T) {
 				issue:      pullIssue,
 				actionType: activities_model.ActionPullRequestReadyForReview,
 			},
-			prefix: fmt.Sprintf("%s/pulls/%d/ready/", issue.Repo.FullName(), issue.Index),
+			prefix: fmt.Sprintf("<%s/pulls/%d/ready/", issue.Repo.FullName(), issue.Index),
 		},
 	}
 	for _, tt := range tests {
@@ -362,9 +368,6 @@ func Test_createReference(t *testing.T) {
 			if !strings.HasPrefix(got, tt.prefix) {
 				t.Errorf("createReference() = %v, want %v", got, tt.prefix)
 			}
-			if !strings.HasSuffix(got, tt.suffix) {
-				t.Errorf("createReference() = %v, want %v", got, tt.prefix)
-			}
 		})
 	}
 }

services/mailer/mailer.go

@@ -36,6 +36,7 @@ type Message struct {
 	FromAddress     string
 	FromDisplayName string
 	To              []string
+	ReplyTo         string
 	Subject         string
 	Date            time.Time
 	Body            string
@@ -47,6 +48,9 @@ func (m *Message) ToMessage() *gomail.Message {
 	msg := gomail.NewMessage()
 	msg.SetAddressHeader("From", m.FromAddress, m.FromDisplayName)
 	msg.SetHeader("To", m.To...)
+	if m.ReplyTo != "" {
+		msg.SetHeader("Reply-To", m.ReplyTo)
+	}
 	for header := range m.Headers {
 		msg.SetHeader(header, m.Headers[header]...)
 	}

services/mailer/token/token.go

@@ -0,0 +1,128 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package token
+
+import (
+	"context"
+	crypto_hmac "crypto/hmac"
+	"crypto/sha256"
+	"encoding/base32"
+	"fmt"
+	"time"
+
+	user_model "code.gitea.io/gitea/models/user"
+	"code.gitea.io/gitea/modules/util"
+)
+
+// A token is a verifiable container describing an action.
+//
+// A token has a dynamic length depending on the contained data and has the following structure:
+// | Token Version | User ID | HMAC | Payload |
+//
+// The payload is verifiable by the generated HMAC using the user secret. It contains:
+// | Timestamp | Action/Handler Type | Action/Handler Data |
+
+const (
+	tokenVersion1        byte = 1
+	tokenLifetimeInYears int  = 1
+)
+
+type HandlerType byte
+
+const (
+	UnknownHandlerType HandlerType = iota
+	ReplyHandlerType
+	UnsubscribeHandlerType
+)
+
+var encodingWithoutPadding = base32.StdEncoding.WithPadding(base32.NoPadding)
+
+type ErrToken struct {
+	context string
+}
+
+func (err *ErrToken) Error() string {
+	return "invalid email token: " + err.context
+}
+
+func (err *ErrToken) Unwrap() error {
+	return util.ErrInvalidArgument
+}
+
+// CreateToken creates a token for the action/user tuple
+func CreateToken(ht HandlerType, user *user_model.User, data []byte) (string, error) {
+	payload, err := util.PackData(
+		time.Now().AddDate(tokenLifetimeInYears, 0, 0).Unix(),
+		ht,
+		data,
+	)
+	if err != nil {
+		return "", err
+	}
+
+	packagedData, err := util.PackData(
+		user.ID,
+		generateHmac([]byte(user.Rands), payload),
+		payload,
+	)
+	if err != nil {
+		return "", err
+	}
+
+	return encodingWithoutPadding.EncodeToString(append([]byte{tokenVersion1}, packagedData...)), nil
+}
+
+// ExtractToken extracts the action/user tuple from the token and verifies the content
+func ExtractToken(ctx context.Context, token string) (HandlerType, *user_model.User, []byte, error) {
+	data, err := encodingWithoutPadding.DecodeString(token)
+	if err != nil {
+		return UnknownHandlerType, nil, nil, err
+	}
+
+	if len(data) < 1 {
+		return UnknownHandlerType, nil, nil, &ErrToken{"no data"}
+	}
+
+	if data[0] != tokenVersion1 {
+		return UnknownHandlerType, nil, nil, &ErrToken{fmt.Sprintf("unsupported token version: %v", data[0])}
+	}
+
+	var userID int64
+	var hmac []byte
+	var payload []byte
+	if err := util.UnpackData(data[1:], &userID, &hmac, &payload); err != nil {
+		return UnknownHandlerType, nil, nil, err
+	}
+
+	user, err := user_model.GetUserByID(ctx, userID)
+	if err != nil {
+		return UnknownHandlerType, nil, nil, err
+	}
+
+	if !crypto_hmac.Equal(hmac, generateHmac([]byte(user.Rands), payload)) {
+		return UnknownHandlerType, nil, nil, &ErrToken{"verification failed"}
+	}
+
+	var expiresUnix int64
+	var handlerType HandlerType
+	var innerPayload []byte
+	if err := util.UnpackData(payload, &expiresUnix, &handlerType, &innerPayload); err != nil {
+		return UnknownHandlerType, nil, nil, err
+	}
+
+	if time.Unix(expiresUnix, 0).Before(time.Now()) {
+		return UnknownHandlerType, nil, nil, &ErrToken{"token expired"}
+	}
+
+	return handlerType, user, innerPayload, nil
+}
+
+// generateHmac creates a trunkated HMAC for the given payload
+func generateHmac(secret, payload []byte) []byte {
+	mac := crypto_hmac.New(sha256.New, secret)
+	mac.Write(payload)
+	hmac := mac.Sum(nil)
+
+	return hmac[:10] // RFC2104 recommends not using less then 80 bits
+}