Restrict repository indexing by glob match (#7767)

* Restrict repository indexing by file extension

* Use REPO_EXTENSIONS_LIST_INCLUDE instead of REPO_EXTENSIONS_LIST_EXCLUDE and have a more flexible extension pattern

* Corrected to pass lint gosimple

* Add wildcard support to REPO_INDEXER_EXTENSIONS

* This reverts commit 72a650c8e4.

* Add wildcard support to REPO_INDEXER_EXTENSIONS (no make vendor)

* Simplify isIndexable() for better clarity

* Add gobwas/glob to vendors

* manually set appengine new release

* Implement better REPO_INDEXER_INCLUDE and REPO_INDEXER_EXCLUDE

* Add unit and integration tests

* Update app.ini.sample and reword config-cheat-sheet

* Add doc page and correct app.ini.sample

* Some polish on the doc

* Simplify code as suggested by @lafriks

models/fixtures/repo_unit.yml

@@ -402,4 +402,39 @@
   repo_id: 11
   type: 1
   config: "{}"
-  created_unix: 946684810
+  created_unix: 946684810
+
+-
+  id: 59
+  repo_id: 42
+  type: 1
+  config: "{}"
+  created_unix: 946684810
+
+-
+  id: 60
+  repo_id: 42
+  type: 4
+  config: "{}"
+  created_unix: 946684810
+
+-
+  id: 61
+  repo_id: 42
+  type: 5
+  config: "{}"
+  created_unix: 946684810
+
+-
+  id: 62
+  repo_id: 42
+  type: 2
+  config: "{\"EnableTimetracker\":true,\"AllowOnlyContributorsToTrackTime\":true}"
+  created_unix: 946684810
+
+-
+  id: 63
+  repo_id: 42
+  type: 3
+  config: "{\"IgnoreWhitespaceConflicts\":false,\"AllowMerge\":true,\"AllowRebase\":true,\"AllowRebaseMerge\":true,\"AllowSquash\":true}"
+  created_unix: 946684810

models/fixtures/repository.yml

@@ -497,4 +497,15 @@
   num_stars: 0
   num_forks: 0
   num_issues: 0
+  is_mirror: false
+
+-
+  id: 42
+  owner_id: 2
+  lower_name: glob
+  name: glob
+  is_private: false
+  num_stars: 0
+  num_forks: 0
+  num_issues: 0
   is_mirror: false

models/fixtures/user.yml

@@ -30,7 +30,7 @@
   is_admin: false
   avatar: avatar2
   avatar_email: user2@example.com
-  num_repos: 8
+  num_repos: 9
   num_stars: 2
   num_followers: 2
   num_following: 1

models/repo_indexer.go

@@ -234,7 +234,21 @@ func addDelete(filename string, repo *Repository, batch rupture.FlushingBatch) e
 }
 
 func isIndexable(entry *git.TreeEntry) bool {
-	return entry.IsRegular() || entry.IsExecutable()
+	if !entry.IsRegular() && !entry.IsExecutable() {
+		return false
+	}
+	name := strings.ToLower(entry.Name())
+	for _, g := range setting.Indexer.ExcludePatterns {
+		if g.Match(name) {
+			return false
+		}
+	}
+	for _, g := range setting.Indexer.IncludePatterns {
+		if g.Match(name) {
+			return true
+		}
+	}
+	return len(setting.Indexer.IncludePatterns) == 0
 }
 
 // parseGitLsTreeOutput parses the output of a `git ls-tree -r --full-name` command

models/repo_list_test.go

@@ -175,10 +175,10 @@ func TestSearchRepository(t *testing.T) {
 			count: 14},
 		{name: "AllPublic/PublicRepositoriesOfUserIncludingCollaborative",
 			opts:  &SearchRepoOptions{Page: 1, PageSize: 10, OwnerID: 15, AllPublic: true},
-			count: 21},
+			count: 22},
 		{name: "AllPublic/PublicAndPrivateRepositoriesOfUserIncludingCollaborative",
 			opts:  &SearchRepoOptions{Page: 1, PageSize: 10, OwnerID: 15, Private: true, AllPublic: true},
-			count: 27},
+			count: 28},
 		{name: "AllPublic/PublicAndPrivateRepositoriesOfUserIncludingCollaborativeByName",
 			opts:  &SearchRepoOptions{Keyword: "test", Page: 1, PageSize: 10, OwnerID: 15, Private: true, AllPublic: true},
 			count: 15},
@@ -187,7 +187,7 @@ func TestSearchRepository(t *testing.T) {
 			count: 13},
 		{name: "AllPublic/PublicRepositoriesOfOrganization",
 			opts:  &SearchRepoOptions{Page: 1, PageSize: 10, OwnerID: 17, AllPublic: true, Collaborate: util.OptionalBoolFalse},
-			count: 21},
+			count: 22},
 	}
 
 	for _, testCase := range testCases {