From 52c2e82813c2519cd08c9ecd66ef7b0815682770 Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Fri, 10 Jun 2022 07:39:53 +0200 Subject: [PATCH] Custom regexp external issues (#17624) * Implement custom regular expression for external issue tracking. Signed-off-by: Alexander Beyn * Fix syntax/style * Update repo.go * Set metas['regexp'] * gofmt * fix some tests * fix more tests * refactor frontend * use LRU cache for regexp * Update modules/markup/html_internal_test.go Co-authored-by: Alexander Beyn Co-authored-by: techknowlogick Co-authored-by: Lauris BH Co-authored-by: 6543 <6543@obermui.de> Co-authored-by: wxiaoguang --- models/repo/repo.go | 3 ++ models/repo/repo_unit.go | 7 ++-- models/repo_test.go | 3 ++ modules/markup/html.go | 40 +++++++++++++++------ modules/markup/html_internal_test.go | 52 +++++++++++++++++++++++++--- modules/references/references.go | 20 ++++++++++- modules/regexplru/regexplru.go | 45 ++++++++++++++++++++++++ modules/regexplru/regexplru_test.go | 27 +++++++++++++++ options/locale/locale_en-US.ini | 3 ++ routers/web/repo/setting.go | 7 ++-- services/forms/repo_form.go | 1 + templates/repo/settings/options.tmpl | 19 +++++++--- web_src/js/features/repo-legacy.js | 5 +++ 13 files changed, 206 insertions(+), 26 deletions(-) create mode 100644 modules/regexplru/regexplru.go create mode 100644 modules/regexplru/regexplru_test.go diff --git a/models/repo/repo.go b/models/repo/repo.go index 3fd6b94eb1..57d85435eb 100644 --- a/models/repo/repo.go +++ b/models/repo/repo.go @@ -414,6 +414,9 @@ func (repo *Repository) ComposeMetas() map[string]string { switch unit.ExternalTrackerConfig().ExternalTrackerStyle { case markup.IssueNameStyleAlphanumeric: metas["style"] = markup.IssueNameStyleAlphanumeric + case markup.IssueNameStyleRegexp: + metas["style"] = markup.IssueNameStyleRegexp + metas["regexp"] = unit.ExternalTrackerConfig().ExternalTrackerRegexpPattern default: metas["style"] = markup.IssueNameStyleNumeric } diff --git a/models/repo/repo_unit.go b/models/repo/repo_unit.go index 8c17d6138c..da3e19dece 100644 --- a/models/repo/repo_unit.go +++ b/models/repo/repo_unit.go @@ -76,9 +76,10 @@ func (cfg *ExternalWikiConfig) ToDB() ([]byte, error) { // ExternalTrackerConfig describes external tracker config type ExternalTrackerConfig struct { - ExternalTrackerURL string - ExternalTrackerFormat string - ExternalTrackerStyle string + ExternalTrackerURL string + ExternalTrackerFormat string + ExternalTrackerStyle string + ExternalTrackerRegexpPattern string } // FromDB fills up a ExternalTrackerConfig from serialized format. diff --git a/models/repo_test.go b/models/repo_test.go index c9e66398d1..f554ff16a6 100644 --- a/models/repo_test.go +++ b/models/repo_test.go @@ -74,6 +74,9 @@ func TestMetas(t *testing.T) { externalTracker.ExternalTrackerConfig().ExternalTrackerStyle = markup.IssueNameStyleNumeric testSuccess(markup.IssueNameStyleNumeric) + externalTracker.ExternalTrackerConfig().ExternalTrackerStyle = markup.IssueNameStyleRegexp + testSuccess(markup.IssueNameStyleRegexp) + repo, err := repo_model.GetRepositoryByID(3) assert.NoError(t, err) diff --git a/modules/markup/html.go b/modules/markup/html.go index c5d36e701f..69d9ba3ef2 100644 --- a/modules/markup/html.go +++ b/modules/markup/html.go @@ -20,6 +20,7 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/markup/common" "code.gitea.io/gitea/modules/references" + "code.gitea.io/gitea/modules/regexplru" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/templates/vars" "code.gitea.io/gitea/modules/util" @@ -33,6 +34,7 @@ import ( const ( IssueNameStyleNumeric = "numeric" IssueNameStyleAlphanumeric = "alphanumeric" + IssueNameStyleRegexp = "regexp" ) var ( @@ -815,19 +817,35 @@ func issueIndexPatternProcessor(ctx *RenderContext, node *html.Node) { ) next := node.NextSibling + for node != nil && node != next { - _, exttrack := ctx.Metas["format"] - alphanum := ctx.Metas["style"] == IssueNameStyleAlphanumeric + _, hasExtTrackFormat := ctx.Metas["format"] // Repos with external issue trackers might still need to reference local PRs // We need to concern with the first one that shows up in the text, whichever it is - found, ref = references.FindRenderizableReferenceNumeric(node.Data, exttrack && alphanum) - if exttrack && alphanum { - if found2, ref2 := references.FindRenderizableReferenceAlphanumeric(node.Data); found2 { - if !found || ref2.RefLocation.Start < ref.RefLocation.Start { - found = true - ref = ref2 - } + isNumericStyle := ctx.Metas["style"] == "" || ctx.Metas["style"] == IssueNameStyleNumeric + foundNumeric, refNumeric := references.FindRenderizableReferenceNumeric(node.Data, hasExtTrackFormat && !isNumericStyle) + + switch ctx.Metas["style"] { + case "", IssueNameStyleNumeric: + found, ref = foundNumeric, refNumeric + case IssueNameStyleAlphanumeric: + found, ref = references.FindRenderizableReferenceAlphanumeric(node.Data) + case IssueNameStyleRegexp: + pattern, err := regexplru.GetCompiled(ctx.Metas["regexp"]) + if err != nil { + return + } + found, ref = references.FindRenderizableReferenceRegexp(node.Data, pattern) + } + + // Repos with external issue trackers might still need to reference local PRs + // We need to concern with the first one that shows up in the text, whichever it is + if hasExtTrackFormat && !isNumericStyle { + // If numeric (PR) was found, and it was BEFORE the non-numeric pattern, use that + if foundNumeric && refNumeric.RefLocation.Start < ref.RefLocation.Start { + found = foundNumeric + ref = refNumeric } } if !found { @@ -836,7 +854,7 @@ func issueIndexPatternProcessor(ctx *RenderContext, node *html.Node) { var link *html.Node reftext := node.Data[ref.RefLocation.Start:ref.RefLocation.End] - if exttrack && !ref.IsPull { + if hasExtTrackFormat && !ref.IsPull { ctx.Metas["index"] = ref.Issue res, err := vars.Expand(ctx.Metas["format"], ctx.Metas) @@ -869,7 +887,7 @@ func issueIndexPatternProcessor(ctx *RenderContext, node *html.Node) { // Decorate action keywords if actionable var keyword *html.Node - if references.IsXrefActionable(ref, exttrack, alphanum) { + if references.IsXrefActionable(ref, hasExtTrackFormat) { keyword = createKeyword(node.Data[ref.ActionLocation.Start:ref.ActionLocation.End]) } else { keyword = &html.Node{ diff --git a/modules/markup/html_internal_test.go b/modules/markup/html_internal_test.go index f0eb3253e1..25b0f7b7a5 100644 --- a/modules/markup/html_internal_test.go +++ b/modules/markup/html_internal_test.go @@ -21,8 +21,8 @@ const ( TestRepoURL = TestAppURL + TestOrgRepo + "/" ) -// alphanumLink an HTML link to an alphanumeric-style issue -func alphanumIssueLink(baseURL, class, name string) string { +// externalIssueLink an HTML link to an alphanumeric-style issue +func externalIssueLink(baseURL, class, name string) string { return link(util.URLJoin(baseURL, name), class, name) } @@ -54,6 +54,13 @@ var alphanumericMetas = map[string]string{ "style": IssueNameStyleAlphanumeric, } +var regexpMetas = map[string]string{ + "format": "https://someurl.com/{user}/{repo}/{index}", + "user": "someUser", + "repo": "someRepo", + "style": IssueNameStyleRegexp, +} + // these values should match the TestOrgRepo const above var localMetas = map[string]string{ "user": "gogits", @@ -184,7 +191,7 @@ func TestRender_IssueIndexPattern4(t *testing.T) { test := func(s, expectedFmt string, names ...string) { links := make([]interface{}, len(names)) for i, name := range names { - links[i] = alphanumIssueLink("https://someurl.com/someUser/someRepo/", "ref-issue ref-external-issue", name) + links[i] = externalIssueLink("https://someurl.com/someUser/someRepo/", "ref-issue ref-external-issue", name) } expected := fmt.Sprintf(expectedFmt, links...) testRenderIssueIndexPattern(t, s, expected, &RenderContext{Metas: alphanumericMetas}) @@ -194,6 +201,43 @@ func TestRender_IssueIndexPattern4(t *testing.T) { test("test issue ABCDEFGHIJ-1234567890", "test issue %s", "ABCDEFGHIJ-1234567890") } +func TestRender_IssueIndexPattern5(t *testing.T) { + setting.AppURL = TestAppURL + + // regexp: render inputs without valid mentions + test := func(s, expectedFmt, pattern string, ids, names []string) { + metas := regexpMetas + metas["regexp"] = pattern + links := make([]interface{}, len(ids)) + for i, id := range ids { + links[i] = link(util.URLJoin("https://someurl.com/someUser/someRepo/", id), "ref-issue ref-external-issue", names[i]) + } + + expected := fmt.Sprintf(expectedFmt, links...) + testRenderIssueIndexPattern(t, s, expected, &RenderContext{Metas: metas}) + } + + test("abc ISSUE-123 def", "abc %s def", + "ISSUE-(\\d+)", + []string{"123"}, + []string{"ISSUE-123"}, + ) + + test("abc (ISSUE 123) def", "abc %s def", + "\\(ISSUE (\\d+)\\)", + []string{"123"}, + []string{"(ISSUE 123)"}, + ) + + test("abc ISSUE-123 def", "abc %s def", + "(ISSUE-(\\d+))", + []string{"ISSUE-123"}, + []string{"ISSUE-123"}, + ) + + testRenderIssueIndexPattern(t, "will not match", "will not match", &RenderContext{Metas: regexpMetas}) +} + func testRenderIssueIndexPattern(t *testing.T, input, expected string, ctx *RenderContext) { if ctx.URLPrefix == "" { ctx.URLPrefix = TestAppURL @@ -202,7 +246,7 @@ func testRenderIssueIndexPattern(t *testing.T, input, expected string, ctx *Rend var buf strings.Builder err := postProcess(ctx, []processor{issueIndexPatternProcessor}, strings.NewReader(input), &buf) assert.NoError(t, err) - assert.Equal(t, expected, buf.String()) + assert.Equal(t, expected, buf.String(), "input=%q", input) } func TestRender_AutoLink(t *testing.T) { diff --git a/modules/references/references.go b/modules/references/references.go index 630e621043..7f5086d093 100644 --- a/modules/references/references.go +++ b/modules/references/references.go @@ -351,6 +351,24 @@ func FindRenderizableReferenceNumeric(content string, prOnly bool) (bool, *Rende } } +// FindRenderizableReferenceRegexp returns the first regexp unvalidated references found in a string. +func FindRenderizableReferenceRegexp(content string, pattern *regexp.Regexp) (bool, *RenderizableReference) { + match := pattern.FindStringSubmatchIndex(content) + if len(match) < 4 { + return false, nil + } + + action, location := findActionKeywords([]byte(content), match[2]) + + return true, &RenderizableReference{ + Issue: content[match[2]:match[3]], + RefLocation: &RefSpan{Start: match[0], End: match[1]}, + Action: action, + ActionLocation: location, + IsPull: false, + } +} + // FindRenderizableReferenceAlphanumeric returns the first alphanumeric unvalidated references found in a string. func FindRenderizableReferenceAlphanumeric(content string) (bool, *RenderizableReference) { match := issueAlphanumericPattern.FindStringSubmatchIndex(content) @@ -547,7 +565,7 @@ func findActionKeywords(content []byte, start int) (XRefAction, *RefSpan) { } // IsXrefActionable returns true if the xref action is actionable (i.e. produces a result when resolved) -func IsXrefActionable(ref *RenderizableReference, extTracker, alphaNum bool) bool { +func IsXrefActionable(ref *RenderizableReference, extTracker bool) bool { if extTracker { // External issues cannot be automatically closed return false diff --git a/modules/regexplru/regexplru.go b/modules/regexplru/regexplru.go new file mode 100644 index 0000000000..97c7cff4c1 --- /dev/null +++ b/modules/regexplru/regexplru.go @@ -0,0 +1,45 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package regexplru + +import ( + "regexp" + + "code.gitea.io/gitea/modules/log" + + lru "github.com/hashicorp/golang-lru" +) + +var lruCache *lru.Cache + +func init() { + var err error + lruCache, err = lru.New(1000) + if err != nil { + log.Fatal("failed to new LRU cache, err: %v", err) + } +} + +// GetCompiled works like regexp.Compile, the compiled expr or error is stored in LRU cache +func GetCompiled(expr string) (r *regexp.Regexp, err error) { + v, ok := lruCache.Get(expr) + if !ok { + r, err = regexp.Compile(expr) + if err != nil { + lruCache.Add(expr, err) + return nil, err + } + lruCache.Add(expr, r) + } else { + r, ok = v.(*regexp.Regexp) + if !ok { + if err, ok = v.(error); ok { + return nil, err + } + panic("impossible") + } + } + return r, nil +} diff --git a/modules/regexplru/regexplru_test.go b/modules/regexplru/regexplru_test.go new file mode 100644 index 0000000000..041f0dcfb9 --- /dev/null +++ b/modules/regexplru/regexplru_test.go @@ -0,0 +1,27 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package regexplru + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestRegexpLru(t *testing.T) { + r, err := GetCompiled("a") + assert.NoError(t, err) + assert.True(t, r.MatchString("a")) + + r, err = GetCompiled("a") + assert.NoError(t, err) + assert.True(t, r.MatchString("a")) + + assert.EqualValues(t, 1, lruCache.Len()) + + _, err = GetCompiled("(") + assert.Error(t, err) + assert.EqualValues(t, 2, lruCache.Len()) +} diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index b9ba6e1136..c4ad714717 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -1811,6 +1811,9 @@ settings.tracker_url_format_error = The external issue tracker URL format is not settings.tracker_issue_style = External Issue Tracker Number Format settings.tracker_issue_style.numeric = Numeric settings.tracker_issue_style.alphanumeric = Alphanumeric +settings.tracker_issue_style.regexp = Regular Expression +settings.tracker_issue_style.regexp_pattern = Regular Expression Pattern +settings.tracker_issue_style.regexp_pattern_desc = The first captured group will be used in place of {index}. settings.tracker_url_format_desc = Use the placeholders {user}, {repo} and {index} for the username, repository name and issue index. settings.enable_timetracker = Enable Time Tracking settings.allow_only_contributors_to_track_time = Let Only Contributors Track Time diff --git a/routers/web/repo/setting.go b/routers/web/repo/setting.go index 1a7a41ae91..f49ef6e85d 100644 --- a/routers/web/repo/setting.go +++ b/routers/web/repo/setting.go @@ -434,9 +434,10 @@ func SettingsPost(ctx *context.Context) { RepoID: repo.ID, Type: unit_model.TypeExternalTracker, Config: &repo_model.ExternalTrackerConfig{ - ExternalTrackerURL: form.ExternalTrackerURL, - ExternalTrackerFormat: form.TrackerURLFormat, - ExternalTrackerStyle: form.TrackerIssueStyle, + ExternalTrackerURL: form.ExternalTrackerURL, + ExternalTrackerFormat: form.TrackerURLFormat, + ExternalTrackerStyle: form.TrackerIssueStyle, + ExternalTrackerRegexpPattern: form.ExternalTrackerRegexpPattern, }, }) deleteUnitTypes = append(deleteUnitTypes, unit_model.TypeIssues) diff --git a/services/forms/repo_form.go b/services/forms/repo_form.go index 2bcb91f8c3..738a77d2bb 100644 --- a/services/forms/repo_form.go +++ b/services/forms/repo_form.go @@ -141,6 +141,7 @@ type RepoSettingForm struct { ExternalTrackerURL string TrackerURLFormat string TrackerIssueStyle string + ExternalTrackerRegexpPattern string EnableCloseIssuesViaCommitInAnyBranch bool EnableProjects bool EnablePackages bool diff --git a/templates/repo/settings/options.tmpl b/templates/repo/settings/options.tmpl index f44d9c98ad..67a98aff43 100644 --- a/templates/repo/settings/options.tmpl +++ b/templates/repo/settings/options.tmpl @@ -361,16 +361,27 @@
{{$externalTracker := (.Repository.MustGetUnit $.UnitTypeExternalTracker)}} {{$externalTrackerStyle := $externalTracker.ExternalTrackerConfig.ExternalTrackerStyle}} - - + +
- - + +
+
+
+ + +
+
+ +
+ + +

{{.i18n.Tr "repo.settings.tracker_issue_style.regexp_pattern_desc" | Str2html}}

diff --git a/web_src/js/features/repo-legacy.js b/web_src/js/features/repo-legacy.js index 53471b30cf..6cdde6a1e4 100644 --- a/web_src/js/features/repo-legacy.js +++ b/web_src/js/features/repo-legacy.js @@ -462,6 +462,11 @@ export function initRepository() { if (typeof $(this).data('context') !== 'undefined') $($(this).data('context')).addClass('disabled'); } }); + const $trackerIssueStyleRadios = $('.js-tracker-issue-style'); + $trackerIssueStyleRadios.on('change input', () => { + const checkedVal = $trackerIssueStyleRadios.filter(':checked').val(); + $('#tracker-issue-style-regex-box').toggleClass('disabled', checkedVal !== 'regexp'); + }); } // Labels