From cbf7a0ee531d43ba848d20d6815e2fd5281b05db Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sat, 27 Mar 2021 10:19:39 +0800 Subject: [PATCH 01/26] Refactor renders --- contrib/pr/checkout.go | 2 +- models/issue_comment.go | 10 +- models/repo.go | 5 +- modules/csv/csv.go | 26 +- modules/markup/csv/csv.go | 91 ++++--- modules/markup/csv/csv_test.go | 10 +- modules/markup/external/external.go | 54 ++--- modules/markup/html.go | 225 ++++++++---------- modules/markup/html_internal_test.go | 57 +++-- modules/markup/html_test.go | 64 +++-- modules/markup/markdown/markdown.go | 79 +++--- modules/markup/markdown/markdown_test.go | 62 +++-- modules/markup/orgmode/orgmode.go | 60 ++--- modules/markup/orgmode/orgmode_test.go | 11 +- modules/markup/renderer.go | 157 ++++++++++++ .../{markup_test.go => renderer_test.go} | 0 modules/notification/mail/mail.go | 8 +- modules/setting/markup.go | 12 +- modules/templates/helper.go | 29 ++- modules/util/util.go | 13 + routers/api/v1/misc/markdown.go | 36 +-- routers/init.go | 2 +- routers/org/home.go | 11 +- routers/repo/compare.go | 3 +- routers/repo/issue.go | 55 ++++- routers/repo/milestone.go | 19 +- routers/repo/projects.go | 19 +- routers/repo/release.go | 19 +- routers/repo/view.go | 36 ++- routers/repo/wiki.go | 31 ++- routers/user/home.go | 11 +- routers/user/profile.go | 11 +- services/mailer/mail.go | 32 ++- services/mailer/mail_issue.go | 6 +- services/mailer/mail_release.go | 11 +- services/mailer/mail_test.go | 9 +- 36 files changed, 880 insertions(+), 406 deletions(-) create mode 100644 modules/markup/renderer.go rename modules/markup/{markup_test.go => renderer_test.go} (100%) diff --git a/contrib/pr/checkout.go b/contrib/pr/checkout.go index 63eca484a51d4..9ee692fd35b1a 100644 --- a/contrib/pr/checkout.go +++ b/contrib/pr/checkout.go @@ -114,7 +114,7 @@ func runPR() { log.Printf("[PR] Setting up router\n") //routers.GlobalInit() - external.RegisterParsers() + external.RegisterRenderers() markup.Init() c := routes.NormalRoutes() diff --git a/models/issue_comment.go b/models/issue_comment.go index 53d4d638c4aad..26bf122dc9835 100644 --- a/models/issue_comment.go +++ b/models/issue_comment.go @@ -16,6 +16,7 @@ import ( "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/markup/markdown" "code.gitea.io/gitea/modules/references" "code.gitea.io/gitea/modules/structs" @@ -1178,8 +1179,13 @@ func findCodeComments(e Engine, opts FindCommentsOptions, issue *Issue, currentU return nil, err } - comment.RenderedContent = string(markdown.Render([]byte(comment.Content), issue.Repo.Link(), - issue.Repo.ComposeMetas())) + var err error + if comment.RenderedContent, err = markdown.RenderString(&markup.RenderContext{ + URLPrefix: issue.Repo.Link(), + Metas: issue.Repo.ComposeMetas(), + }, comment.Content); err != nil { + return nil, err + } } return comments[:n], nil } diff --git a/models/repo.go b/models/repo.go index bdb84ee00da55..fc673cace8b57 100644 --- a/models/repo.go +++ b/models/repo.go @@ -863,7 +863,10 @@ func (repo *Repository) getUsersWithAccessMode(e Engine, mode AccessMode) (_ []* // DescriptionHTML does special handles to description and return HTML string. func (repo *Repository) DescriptionHTML() template.HTML { - desc, err := markup.RenderDescriptionHTML([]byte(repo.Description), repo.HTMLURL(), repo.ComposeMetas()) + desc, err := markup.RenderDescriptionHTML(&markup.RenderContext{ + URLPrefix: repo.HTMLURL(), + Metas: repo.ComposeMetas(), + }, repo.Description) if err != nil { log.Error("Failed to render description for %s (ID: %d): %v", repo.Name, repo.ID, err) return template.HTML(markup.Sanitize(repo.Description)) diff --git a/modules/csv/csv.go b/modules/csv/csv.go index 1aa78fdeec76a..826cbbaba6a65 100644 --- a/modules/csv/csv.go +++ b/modules/csv/csv.go @@ -7,7 +7,9 @@ package csv import ( "bytes" "encoding/csv" + stdcsv "encoding/csv" "errors" + "io" "regexp" "strings" @@ -18,17 +20,31 @@ import ( var quoteRegexp = regexp.MustCompile(`["'][\s\S]+?["']`) // CreateReader creates a csv.Reader with the given delimiter. -func CreateReader(rawBytes []byte, delimiter rune) *csv.Reader { - rd := csv.NewReader(bytes.NewReader(rawBytes)) +func CreateReader(input io.Reader, delimiter rune) *stdcsv.Reader { + rd := stdcsv.NewReader(input) rd.Comma = delimiter rd.TrimLeadingSpace = true return rd } // CreateReaderAndGuessDelimiter tries to guess the field delimiter from the content and creates a csv.Reader. -func CreateReaderAndGuessDelimiter(rawBytes []byte) *csv.Reader { - delimiter := guessDelimiter(rawBytes) - return CreateReader(rawBytes, delimiter) +func CreateReaderAndGuessDelimiter(rd io.Reader) (*stdcsv.Reader, error) { + var data = make([]byte, 1e4) + size, err := rd.Read(data) + if err != nil { + return nil, err + } + + delimiter := guessDelimiter(data[:size]) + + var newInput io.Reader + if size < 1e4 { + newInput = bytes.NewReader(data) + } else { + newInput = io.MultiReader(bytes.NewReader(data), rd) + } + + return CreateReader(newInput, delimiter), nil } // guessDelimiter scores the input CSV data against delimiters, and returns the best match. diff --git a/modules/markup/csv/csv.go b/modules/markup/csv/csv.go index 68c89166b5b06..603504570c761 100644 --- a/modules/markup/csv/csv.go +++ b/modules/markup/csv/csv.go @@ -5,6 +5,7 @@ package markup import ( + "bufio" "bytes" "html" "io" @@ -16,52 +17,83 @@ import ( ) func init() { - markup.RegisterParser(Parser{}) + markup.RegisterRenderer(Renderer{}) + } -// Parser implements markup.Parser for csv files -type Parser struct { +// Renderer implements markup.Renderer for orgmode +type Renderer struct { } -// Name implements markup.Parser -func (Parser) Name() string { +// Name implements markup.Renderer +func (Renderer) Name() string { return "csv" } -// NeedPostProcess implements markup.Parser -func (Parser) NeedPostProcess() bool { return false } +// NeedPostProcess implements markup.Renderer +func (Renderer) NeedPostProcess() bool { return false } -// Extensions implements markup.Parser -func (Parser) Extensions() []string { +// Extensions implements markup.Renderer +func (Renderer) Extensions() []string { return []string{".csv", ".tsv"} } +func writeField(w io.Writer, element, class, field string) error { + if _, err := io.WriteString(w, "<"); err != nil { + return err + } + if _, err := io.WriteString(w, element); err != nil { + return err + } + if len(class) > 0 { + if _, err := io.WriteString(w, " class=\""); err != nil { + return err + } + if _, err := io.WriteString(w, class); err != nil { + return err + } + if _, err := io.WriteString(w, "\""); err != nil { + return err + } + } + if _, err := io.WriteString(w, ">"); err != nil { + return err + } + if _, err := io.WriteString(w, html.EscapeString(field)); err != nil { + return err + } + if _, err := io.WriteString(w, "") + return err +} + // Render implements markup.Parser -func (Parser) Render(rawBytes []byte, urlPrefix string, metas map[string]string, isWiki bool) []byte { - var tmpBlock bytes.Buffer +func (Renderer) Render(ctx *markup.RenderContext, input io.Reader, output io.Writer) error { + var tmpBlock = bufio.NewWriter(output) + + rawBytes, err := io.ReadAll(input) + if err != nil { + return err + } if setting.UI.CSV.MaxFileSize != 0 && setting.UI.CSV.MaxFileSize < int64(len(rawBytes)) { tmpBlock.WriteString("
")
 		tmpBlock.WriteString(html.EscapeString(string(rawBytes)))
 		tmpBlock.WriteString("
") - return tmpBlock.Bytes() + return nil } - rd := csv.CreateReaderAndGuessDelimiter(rawBytes) + rd, err := csv.CreateReaderAndGuessDelimiter(bytes.NewReader(rawBytes)) + if err != nil { + if err == io.EOF { - writeField := func(element, class, field string) { - tmpBlock.WriteString("<") - tmpBlock.WriteString(element) - if len(class) > 0 { - tmpBlock.WriteString(" class=\"") - tmpBlock.WriteString(class) - tmpBlock.WriteString("\"") } - tmpBlock.WriteString(">") - tmpBlock.WriteString(html.EscapeString(field)) - tmpBlock.WriteString("") + return err } tmpBlock.WriteString(``) @@ -79,15 +111,14 @@ func (Parser) Render(rawBytes []byte, urlPrefix string, metas map[string]string, if row == 1 { element = "th" } - writeField(element, "line-num", strconv.Itoa(row)) + writeField(tmpBlock, element, "line-num", strconv.Itoa(row)) for _, field := range fields { - writeField(element, "", field) + writeField(tmpBlock, element, "", field) } tmpBlock.WriteString("") row++ } - tmpBlock.WriteString("
") - - return tmpBlock.Bytes() + _, err = tmpBlock.WriteString("") + return err } diff --git a/modules/markup/csv/csv_test.go b/modules/markup/csv/csv_test.go index 5438ebdf5ca2a..48e3ce572514e 100644 --- a/modules/markup/csv/csv_test.go +++ b/modules/markup/csv/csv_test.go @@ -5,13 +5,15 @@ package markup import ( + "strings" "testing" + "code.gitea.io/gitea/modules/markup" "github.com/stretchr/testify/assert" ) func TestRenderCSV(t *testing.T) { - var parser Parser + var render Renderer var kases = map[string]string{ "a": "
1a
", "1,2": "
112
", @@ -20,7 +22,9 @@ func TestRenderCSV(t *testing.T) { } for k, v := range kases { - res := parser.Render([]byte(k), "", nil, false) - assert.EqualValues(t, v, string(res)) + var buf strings.Builder + err := render.Render(&markup.RenderContext{}, strings.NewReader(k), &buf) + assert.NoError(t, err) + assert.EqualValues(t, v, buf.String()) } } diff --git a/modules/markup/external/external.go b/modules/markup/external/external.go index 6e7e59970dbf0..93207bda5bdc8 100644 --- a/modules/markup/external/external.go +++ b/modules/markup/external/external.go @@ -5,7 +5,7 @@ package external import ( - "bytes" + "fmt" "io" "io/ioutil" "os" @@ -19,22 +19,22 @@ import ( "code.gitea.io/gitea/modules/util" ) -// RegisterParsers registers all supported third part parsers according settings -func RegisterParsers() { - for _, parser := range setting.ExternalMarkupParsers { - if parser.Enabled && parser.Command != "" && len(parser.FileExtensions) > 0 { - markup.RegisterParser(&Parser{parser}) +// RegisterRenderers registers all supported third part renderers according settings +func RegisterRenderers() { + for _, renderer := range setting.ExternalMarkupRenderers { + if renderer.Enabled && renderer.Command != "" && len(renderer.FileExtensions) > 0 { + markup.RegisterRenderer(&Renderer{renderer}) } } } -// Parser implements markup.Parser for external tools -type Parser struct { - setting.MarkupParser +// Renderer implements markup.Renderer for external tools +type Renderer struct { + setting.MarkupRenderer } // Name returns the external tool name -func (p *Parser) Name() string { +func (p *Renderer) Name() string { return p.MarkupName } @@ -44,7 +44,7 @@ func (p *Parser) NeedPostProcess() bool { } // Extensions returns the supported extensions of the tool -func (p *Parser) Extensions() []string { +func (p *Renderer) Extensions() []string { return p.FileExtensions } @@ -56,14 +56,10 @@ func envMark(envName string) string { } // Render renders the data of the document to HTML via the external tool. -func (p *Parser) Render(rawBytes []byte, urlPrefix string, metas map[string]string, isWiki bool) []byte { +func (p *Renderer) Render(ctx *markup.RenderContext, input io.Reader, output io.Writer) error { var ( - bs []byte - buf = bytes.NewBuffer(bs) - rd = bytes.NewReader(rawBytes) - urlRawPrefix = strings.Replace(urlPrefix, "/src/", "/raw/", 1) - - command = strings.NewReplacer(envMark("GITEA_PREFIX_SRC"), urlPrefix, + urlRawPrefix = strings.Replace(ctx.URLPrefix, "/src/", "/raw/", 1) + command = strings.NewReplacer(envMark("GITEA_PREFIX_SRC"), ctx.URLPrefix, envMark("GITEA_PREFIX_RAW"), urlRawPrefix).Replace(p.Command) commands = strings.Fields(command) args = commands[1:] @@ -73,8 +69,7 @@ func (p *Parser) Render(rawBytes []byte, urlPrefix string, metas map[string]stri // write to temp file f, err := ioutil.TempFile("", "gitea_input") if err != nil { - log.Error("%s create temp file when rendering %s failed: %v", p.Name(), p.Command, err) - return []byte("") + return fmt.Errorf("%s create temp file when rendering %s failed: %v", p.Name(), p.Command, err) } tmpPath := f.Name() defer func() { @@ -83,17 +78,15 @@ func (p *Parser) Render(rawBytes []byte, urlPrefix string, metas map[string]stri } }() - _, err = io.Copy(f, rd) + _, err = io.Copy(f, input) if err != nil { f.Close() - log.Error("%s write data to temp file when rendering %s failed: %v", p.Name(), p.Command, err) - return []byte("") + return fmt.Errorf("%s write data to temp file when rendering %s failed: %v", p.Name(), p.Command, err) } err = f.Close() if err != nil { - log.Error("%s close temp file when rendering %s failed: %v", p.Name(), p.Command, err) - return []byte("") + return fmt.Errorf("%s close temp file when rendering %s failed: %v", p.Name(), p.Command, err) } args = append(args, f.Name()) } @@ -101,16 +94,15 @@ func (p *Parser) Render(rawBytes []byte, urlPrefix string, metas map[string]stri cmd := exec.Command(commands[0], args...) cmd.Env = append( os.Environ(), - "GITEA_PREFIX_SRC="+urlPrefix, + "GITEA_PREFIX_SRC="+ctx.URLPrefix, "GITEA_PREFIX_RAW="+urlRawPrefix, ) if !p.IsInputFile { - cmd.Stdin = rd + cmd.Stdin = input } - cmd.Stdout = buf + cmd.Stdout = output if err := cmd.Run(); err != nil { - log.Error("%s render run command %s %v failed: %v", p.Name(), commands[0], args, err) - return []byte("") + return fmt.Errorf("%s render run command %s %v failed: %v", p.Name(), commands[0], args, err) } - return buf.Bytes() + return nil } diff --git a/modules/markup/html.go b/modules/markup/html.go index bec9ba2fb49f6..44b461854c7b5 100644 --- a/modules/markup/html.go +++ b/modules/markup/html.go @@ -7,6 +7,8 @@ package markup import ( "bytes" "fmt" + "io" + "io/ioutil" "net/url" "path" "path/filepath" @@ -144,7 +146,7 @@ func (p *postProcessError) Error() string { return "PostProcess: " + p.context + ", " + p.err.Error() } -type processor func(ctx *postProcessCtx, node *html.Node) +type processor func(ctx *RenderContext, node *html.Node) var defaultProcessors = []processor{ fullIssuePatternProcessor, @@ -159,34 +161,17 @@ var defaultProcessors = []processor{ emojiShortCodeProcessor, } -type postProcessCtx struct { - metas map[string]string - urlPrefix string - isWikiMarkdown bool - - // processors used by this context. - procs []processor -} - // PostProcess does the final required transformations to the passed raw HTML // data, and ensures its validity. Transformations include: replacing links and // emails with HTML links, parsing shortlinks in the format of [[Link]], like // MediaWiki, linking issues in the format #ID, and mentions in the format // @user, and others. func PostProcess( - rawHTML []byte, - urlPrefix string, - metas map[string]string, - isWikiMarkdown bool, -) ([]byte, error) { - // create the context from the parameters - ctx := &postProcessCtx{ - metas: metas, - urlPrefix: urlPrefix, - isWikiMarkdown: isWikiMarkdown, - procs: defaultProcessors, - } - return ctx.postProcess(rawHTML) + ctx *RenderContext, + input io.Reader, + output io.Writer, +) error { + return postProcess(ctx, defaultProcessors, input, output) } var commitMessageProcessors = []processor{ @@ -205,23 +190,18 @@ var commitMessageProcessors = []processor{ // the shortLinkProcessor and will add a defaultLinkProcessor if defaultLink is // set, which changes every text node into a link to the passed default link. func RenderCommitMessage( - rawHTML []byte, - urlPrefix, defaultLink string, - metas map[string]string, -) ([]byte, error) { - ctx := &postProcessCtx{ - metas: metas, - urlPrefix: urlPrefix, - procs: commitMessageProcessors, - } - if defaultLink != "" { + ctx *RenderContext, + content string, +) (string, error) { + var procs = commitMessageProcessors + if ctx.DefaultLink != "" { // we don't have to fear data races, because being // commitMessageProcessors of fixed len and cap, every time we append // something to it the slice is realloc+copied, so append always // generates the slice ex-novo. - ctx.procs = append(ctx.procs, genDefaultLinkProcessor(defaultLink)) + procs = append(procs, genDefaultLinkProcessor(ctx.DefaultLink)) } - return ctx.postProcess(rawHTML) + return renderProcessString(ctx, procs, content) } var commitMessageSubjectProcessors = []processor{ @@ -245,83 +225,72 @@ var emojiProcessors = []processor{ // emailAddressProcessor, will add a defaultLinkProcessor if defaultLink is set, // which changes every text node into a link to the passed default link. func RenderCommitMessageSubject( - rawHTML []byte, - urlPrefix, defaultLink string, - metas map[string]string, -) ([]byte, error) { - ctx := &postProcessCtx{ - metas: metas, - urlPrefix: urlPrefix, - procs: commitMessageSubjectProcessors, - } - if defaultLink != "" { + ctx *RenderContext, + content string, +) (string, error) { + var procs = commitMessageSubjectProcessors + if ctx.DefaultLink != "" { // we don't have to fear data races, because being // commitMessageSubjectProcessors of fixed len and cap, every time we // append something to it the slice is realloc+copied, so append always // generates the slice ex-novo. - ctx.procs = append(ctx.procs, genDefaultLinkProcessor(defaultLink)) + procs = append(procs, genDefaultLinkProcessor(ctx.DefaultLink)) } - return ctx.postProcess(rawHTML) + return renderProcessString(ctx, procs, content) } // RenderIssueTitle to process title on individual issue/pull page func RenderIssueTitle( - rawHTML []byte, - urlPrefix string, - metas map[string]string, -) ([]byte, error) { - ctx := &postProcessCtx{ - metas: metas, - urlPrefix: urlPrefix, - procs: []processor{ - issueIndexPatternProcessor, - sha1CurrentPatternProcessor, - emojiShortCodeProcessor, - emojiProcessor, - }, + ctx *RenderContext, + title string, +) (string, error) { + return renderProcessString(ctx, []processor{ + issueIndexPatternProcessor, + sha1CurrentPatternProcessor, + emojiShortCodeProcessor, + emojiProcessor, + }, title) +} + +func renderProcessString(ctx *RenderContext, procs []processor, content string) (string, error) { + var buf strings.Builder + if err := postProcess(ctx, procs, strings.NewReader(content), &buf); err != nil { + return "", err } - return ctx.postProcess(rawHTML) + return buf.String(), nil } // RenderDescriptionHTML will use similar logic as PostProcess, but will // use a single special linkProcessor. func RenderDescriptionHTML( - rawHTML []byte, - urlPrefix string, - metas map[string]string, -) ([]byte, error) { - ctx := &postProcessCtx{ - metas: metas, - urlPrefix: urlPrefix, - procs: []processor{ - descriptionLinkProcessor, - emojiShortCodeProcessor, - emojiProcessor, - }, - } - return ctx.postProcess(rawHTML) + ctx *RenderContext, + content string, +) (string, error) { + return renderProcessString(ctx, []processor{ + descriptionLinkProcessor, + emojiShortCodeProcessor, + emojiProcessor, + }, content) } // RenderEmoji for when we want to just process emoji and shortcodes // in various places it isn't already run through the normal markdown procesor func RenderEmoji( - rawHTML []byte, -) ([]byte, error) { - ctx := &postProcessCtx{ - procs: emojiProcessors, - } - return ctx.postProcess(rawHTML) + content string, +) (string, error) { + return renderProcessString(&RenderContext{}, emojiProcessors, content) } var tagCleaner = regexp.MustCompile(`<((?:/?\w+/\w+)|(?:/[\w ]+/)|(/?[hH][tT][mM][lL]\b)|(/?[hH][eE][aA][dD]\b))`) var nulCleaner = strings.NewReplacer("\000", "") -func (ctx *postProcessCtx) postProcess(rawHTML []byte) ([]byte, error) { - if ctx.procs == nil { - ctx.procs = defaultProcessors +func postProcess(ctx *RenderContext, procs []processor, input io.Reader, output io.Writer) error { + // FIXME: don't read all content to memory + rawHTML, err := ioutil.ReadAll(input) + if err != nil { + return err } - // give a generous extra 50 bytes res := bytes.NewBuffer(make([]byte, 0, len(rawHTML)+50)) // prepend "" _, _ = res.WriteString("") @@ -335,11 +304,11 @@ func (ctx *postProcessCtx) postProcess(rawHTML []byte) ([]byte, error) { // parse the HTML nodes, err := html.ParseFragment(res, nil) if err != nil { - return nil, &postProcessError{"invalid HTML", err} + return &postProcessError{"invalid HTML", err} } for _, node := range nodes { - ctx.visitNode(node, true) + visitNode(ctx, procs, node, true) } newNodes := make([]*html.Node, 0, len(nodes)) @@ -375,15 +344,16 @@ func (ctx *postProcessCtx) postProcess(rawHTML []byte) ([]byte, error) { for _, node := range nodes { err = html.Render(res, node) if err != nil { - return nil, &postProcessError{"error rendering processed HTML", err} + return &postProcessError{"error rendering processed HTML", err} } } // Everything done successfully, return parsed data. - return res.Bytes(), nil + _, err = io.Copy(output, res) + return err } -func (ctx *postProcessCtx) visitNode(node *html.Node, visitText bool) { +func visitNode(ctx *RenderContext, procs []processor, node *html.Node, visitText bool) { // Add user-content- to IDs if they don't already have them for idx, attr := range node.Attr { if attr.Key == "id" && !(strings.HasPrefix(attr.Val, "user-content-") || blackfridayExtRegex.MatchString(attr.Val)) { @@ -399,7 +369,7 @@ func (ctx *postProcessCtx) visitNode(node *html.Node, visitText bool) { switch node.Type { case html.TextNode: if visitText { - ctx.textNode(node) + textNode(ctx, procs, node) } case html.ElementNode: if node.Data == "img" { @@ -410,8 +380,8 @@ func (ctx *postProcessCtx) visitNode(node *html.Node, visitText bool) { } link := []byte(attr.Val) if len(link) > 0 && !IsLink(link) { - prefix := ctx.urlPrefix - if ctx.isWikiMarkdown { + prefix := ctx.URLPrefix + if ctx.IsWiki { prefix = util.URLJoin(prefix, "wiki", "raw") } prefix = strings.Replace(prefix, "/src/", "/media/", 1) @@ -449,7 +419,7 @@ func (ctx *postProcessCtx) visitNode(node *html.Node, visitText bool) { } } for n := node.FirstChild; n != nil; n = n.NextSibling { - ctx.visitNode(n, visitText) + visitNode(ctx, procs, n, visitText) } } // ignore everything else @@ -457,8 +427,8 @@ func (ctx *postProcessCtx) visitNode(node *html.Node, visitText bool) { // textNode runs the passed node through various processors, in order to handle // all kinds of special links handled by the post-processing. -func (ctx *postProcessCtx) textNode(node *html.Node) { - for _, processor := range ctx.procs { +func textNode(ctx *RenderContext, procs []processor, node *html.Node) { + for _, processor := range procs { processor(ctx, node) } } @@ -609,7 +579,7 @@ func replaceContentList(node *html.Node, i, j int, newNodes []*html.Node) { } } -func mentionProcessor(ctx *postProcessCtx, node *html.Node) { +func mentionProcessor(ctx *RenderContext, node *html.Node) { // We replace only the first mention; other mentions will be addressed later found, loc := references.FindFirstMentionBytes([]byte(node.Data)) if !found { @@ -617,26 +587,26 @@ func mentionProcessor(ctx *postProcessCtx, node *html.Node) { } mention := node.Data[loc.Start:loc.End] var teams string - teams, ok := ctx.metas["teams"] + teams, ok := ctx.Metas["teams"] // FIXME: util.URLJoin may not be necessary here: // - setting.AppURL is defined to have a terminal '/' so unless mention[1:] // is an AppSubURL link we can probably fallback to concatenation. // team mention should follow @orgName/teamName style if ok && strings.Contains(mention, "/") { mentionOrgAndTeam := strings.Split(mention, "/") - if mentionOrgAndTeam[0][1:] == ctx.metas["org"] && strings.Contains(teams, ","+strings.ToLower(mentionOrgAndTeam[1])+",") { - replaceContent(node, loc.Start, loc.End, createLink(util.URLJoin(setting.AppURL, "org", ctx.metas["org"], "teams", mentionOrgAndTeam[1]), mention, "mention")) + if mentionOrgAndTeam[0][1:] == ctx.Metas["org"] && strings.Contains(teams, ","+strings.ToLower(mentionOrgAndTeam[1])+",") { + replaceContent(node, loc.Start, loc.End, createLink(util.URLJoin(setting.AppURL, "org", ctx.Metas["org"], "teams", mentionOrgAndTeam[1]), mention, "mention")) } return } replaceContent(node, loc.Start, loc.End, createLink(util.URLJoin(setting.AppURL, mention[1:]), mention, "mention")) } -func shortLinkProcessor(ctx *postProcessCtx, node *html.Node) { +func shortLinkProcessor(ctx *RenderContext, node *html.Node) { shortLinkProcessorFull(ctx, node, false) } -func shortLinkProcessorFull(ctx *postProcessCtx, node *html.Node, noLink bool) { +func shortLinkProcessorFull(ctx *RenderContext, node *html.Node, noLink bool) { m := shortLinkPattern.FindStringSubmatchIndex(node.Data) if m == nil { return @@ -741,13 +711,13 @@ func shortLinkProcessorFull(ctx *postProcessCtx, node *html.Node, noLink bool) { link = url.PathEscape(link) } } - urlPrefix := ctx.urlPrefix + urlPrefix := ctx.URLPrefix if image { if !absoluteLink { if IsSameDomain(urlPrefix) { urlPrefix = strings.Replace(urlPrefix, "/src/", "/raw/", 1) } - if ctx.isWikiMarkdown { + if ctx.IsWiki { link = util.URLJoin("wiki", "raw", link) } link = util.URLJoin(urlPrefix, link) @@ -778,7 +748,7 @@ func shortLinkProcessorFull(ctx *postProcessCtx, node *html.Node, noLink bool) { } } else { if !absoluteLink { - if ctx.isWikiMarkdown { + if ctx.IsWiki { link = util.URLJoin("wiki", link) } link = util.URLJoin(urlPrefix, link) @@ -794,8 +764,8 @@ func shortLinkProcessorFull(ctx *postProcessCtx, node *html.Node, noLink bool) { replaceContent(node, m[0], m[1], linkNode) } -func fullIssuePatternProcessor(ctx *postProcessCtx, node *html.Node) { - if ctx.metas == nil { +func fullIssuePatternProcessor(ctx *RenderContext, node *html.Node) { + if ctx.Metas == nil { return } m := getIssueFullPattern().FindStringSubmatchIndex(node.Data) @@ -811,7 +781,7 @@ func fullIssuePatternProcessor(ctx *postProcessCtx, node *html.Node) { matchOrg := linkParts[len(linkParts)-4] matchRepo := linkParts[len(linkParts)-3] - if matchOrg == ctx.metas["user"] && matchRepo == ctx.metas["repo"] { + if matchOrg == ctx.Metas["user"] && matchRepo == ctx.Metas["repo"] { // TODO if m[4]:m[5] is not nil, then link is to a comment, // and we should indicate that in the text somehow replaceContent(node, m[0], m[1], createLink(link, id, "ref-issue")) @@ -822,8 +792,8 @@ func fullIssuePatternProcessor(ctx *postProcessCtx, node *html.Node) { } } -func issueIndexPatternProcessor(ctx *postProcessCtx, node *html.Node) { - if ctx.metas == nil { +func issueIndexPatternProcessor(ctx *RenderContext, node *html.Node) { + if ctx.Metas == nil { return } @@ -832,8 +802,8 @@ func issueIndexPatternProcessor(ctx *postProcessCtx, node *html.Node) { ref *references.RenderizableReference ) - _, exttrack := ctx.metas["format"] - alphanum := ctx.metas["style"] == IssueNameStyleAlphanumeric + _, exttrack := ctx.Metas["format"] + alphanum := ctx.Metas["style"] == IssueNameStyleAlphanumeric // Repos with external issue trackers might still need to reference local PRs // We need to concern with the first one that shows up in the text, whichever it is @@ -853,8 +823,8 @@ func issueIndexPatternProcessor(ctx *postProcessCtx, node *html.Node) { var link *html.Node reftext := node.Data[ref.RefLocation.Start:ref.RefLocation.End] if exttrack && !ref.IsPull { - ctx.metas["index"] = ref.Issue - link = createLink(com.Expand(ctx.metas["format"], ctx.metas), reftext, "ref-issue") + ctx.Metas["index"] = ref.Issue + link = createLink(com.Expand(ctx.Metas["format"], ctx.Metas), reftext, "ref-issue") } else { // Path determines the type of link that will be rendered. It's unknown at this point whether // the linked item is actually a PR or an issue. Luckily it's of no real consequence because @@ -864,7 +834,7 @@ func issueIndexPatternProcessor(ctx *postProcessCtx, node *html.Node) { path = "pulls" } if ref.Owner == "" { - link = createLink(util.URLJoin(setting.AppURL, ctx.metas["user"], ctx.metas["repo"], path, ref.Issue), reftext, "ref-issue") + link = createLink(util.URLJoin(setting.AppURL, ctx.Metas["user"], ctx.Metas["repo"], path, ref.Issue), reftext, "ref-issue") } else { link = createLink(util.URLJoin(setting.AppURL, ref.Owner, ref.Name, path, ref.Issue), reftext, "ref-issue") } @@ -893,8 +863,8 @@ func issueIndexPatternProcessor(ctx *postProcessCtx, node *html.Node) { } // fullSha1PatternProcessor renders SHA containing URLs -func fullSha1PatternProcessor(ctx *postProcessCtx, node *html.Node) { - if ctx.metas == nil { +func fullSha1PatternProcessor(ctx *RenderContext, node *html.Node) { + if ctx.Metas == nil { return } m := anySHA1Pattern.FindStringSubmatchIndex(node.Data) @@ -944,8 +914,7 @@ func fullSha1PatternProcessor(ctx *postProcessCtx, node *html.Node) { } // emojiShortCodeProcessor for rendering text like :smile: into emoji -func emojiShortCodeProcessor(ctx *postProcessCtx, node *html.Node) { - +func emojiShortCodeProcessor(ctx *RenderContext, node *html.Node) { m := EmojiShortCodeRegex.FindStringSubmatchIndex(node.Data) if m == nil { return @@ -968,7 +937,7 @@ func emojiShortCodeProcessor(ctx *postProcessCtx, node *html.Node) { } // emoji processor to match emoji and add emoji class -func emojiProcessor(ctx *postProcessCtx, node *html.Node) { +func emojiProcessor(ctx *RenderContext, node *html.Node) { m := emoji.FindEmojiSubmatchIndex(node.Data) if m == nil { return @@ -983,8 +952,8 @@ func emojiProcessor(ctx *postProcessCtx, node *html.Node) { // sha1CurrentPatternProcessor renders SHA1 strings to corresponding links that // are assumed to be in the same repository. -func sha1CurrentPatternProcessor(ctx *postProcessCtx, node *html.Node) { - if ctx.metas == nil || ctx.metas["user"] == "" || ctx.metas["repo"] == "" || ctx.metas["repoPath"] == "" { +func sha1CurrentPatternProcessor(ctx *RenderContext, node *html.Node) { + if ctx.Metas == nil || ctx.Metas["user"] == "" || ctx.Metas["repo"] == "" || ctx.Metas["repoPath"] == "" { return } m := sha1CurrentPattern.FindStringSubmatchIndex(node.Data) @@ -1000,7 +969,7 @@ func sha1CurrentPatternProcessor(ctx *postProcessCtx, node *html.Node) { // as used by git and github for linking and thus we have to do similar. // Because of this, we check to make sure that a matched hash is actually // a commit in the repository before making it a link. - if _, err := git.NewCommand("rev-parse", "--verify", hash).RunInDirBytes(ctx.metas["repoPath"]); err != nil { + if _, err := git.NewCommand("rev-parse", "--verify", hash).RunInDirBytes(ctx.Metas["repoPath"]); err != nil { if !strings.Contains(err.Error(), "fatal: Needed a single revision") { log.Debug("sha1CurrentPatternProcessor git rev-parse: %v", err) } @@ -1008,11 +977,11 @@ func sha1CurrentPatternProcessor(ctx *postProcessCtx, node *html.Node) { } replaceContent(node, m[2], m[3], - createCodeLink(util.URLJoin(setting.AppURL, ctx.metas["user"], ctx.metas["repo"], "commit", hash), base.ShortSha(hash), "commit")) + createCodeLink(util.URLJoin(setting.AppURL, ctx.Metas["user"], ctx.Metas["repo"], "commit", hash), base.ShortSha(hash), "commit")) } // emailAddressProcessor replaces raw email addresses with a mailto: link. -func emailAddressProcessor(ctx *postProcessCtx, node *html.Node) { +func emailAddressProcessor(ctx *RenderContext, node *html.Node) { m := emailRegex.FindStringSubmatchIndex(node.Data) if m == nil { return @@ -1023,7 +992,7 @@ func emailAddressProcessor(ctx *postProcessCtx, node *html.Node) { // linkProcessor creates links for any HTTP or HTTPS URL not captured by // markdown. -func linkProcessor(ctx *postProcessCtx, node *html.Node) { +func linkProcessor(ctx *RenderContext, node *html.Node) { m := common.LinkRegex.FindStringIndex(node.Data) if m == nil { return @@ -1033,7 +1002,7 @@ func linkProcessor(ctx *postProcessCtx, node *html.Node) { } func genDefaultLinkProcessor(defaultLink string) processor { - return func(ctx *postProcessCtx, node *html.Node) { + return func(ctx *RenderContext, node *html.Node) { ch := &html.Node{ Parent: node, Type: html.TextNode, @@ -1052,7 +1021,7 @@ func genDefaultLinkProcessor(defaultLink string) processor { } // descriptionLinkProcessor creates links for DescriptionHTML -func descriptionLinkProcessor(ctx *postProcessCtx, node *html.Node) { +func descriptionLinkProcessor(ctx *RenderContext, node *html.Node) { m := common.LinkRegex.FindStringIndex(node.Data) if m == nil { return diff --git a/modules/markup/html_internal_test.go b/modules/markup/html_internal_test.go index 7e4bb6f22f31e..dc76f5b435ad2 100644 --- a/modules/markup/html_internal_test.go +++ b/modules/markup/html_internal_test.go @@ -62,7 +62,7 @@ func TestRender_IssueIndexPattern(t *testing.T) { // numeric: render inputs without valid mentions test := func(s string) { testRenderIssueIndexPattern(t, s, s, nil) - testRenderIssueIndexPattern(t, s, s, &postProcessCtx{metas: numericMetas}) + testRenderIssueIndexPattern(t, s, s, &RenderContext{Metas: numericMetas}) } // should not render anything when there are no mentions @@ -109,13 +109,13 @@ func TestRender_IssueIndexPattern2(t *testing.T) { links[i] = numericIssueLink(util.URLJoin(setting.AppSubURL, path), "ref-issue", index, marker) } expectedNil := fmt.Sprintf(expectedFmt, links...) - testRenderIssueIndexPattern(t, s, expectedNil, &postProcessCtx{metas: localMetas}) + testRenderIssueIndexPattern(t, s, expectedNil, &RenderContext{Metas: localMetas}) for i, index := range indices { links[i] = numericIssueLink(prefix, "ref-issue", index, marker) } expectedNum := fmt.Sprintf(expectedFmt, links...) - testRenderIssueIndexPattern(t, s, expectedNum, &postProcessCtx{metas: numericMetas}) + testRenderIssueIndexPattern(t, s, expectedNum, &RenderContext{Metas: numericMetas}) } // should render freestanding mentions @@ -150,7 +150,7 @@ func TestRender_IssueIndexPattern3(t *testing.T) { // alphanumeric: render inputs without valid mentions test := func(s string) { - testRenderIssueIndexPattern(t, s, s, &postProcessCtx{metas: alphanumericMetas}) + testRenderIssueIndexPattern(t, s, s, &RenderContext{Metas: alphanumericMetas}) } test("") test("this is a test") @@ -181,25 +181,22 @@ func TestRender_IssueIndexPattern4(t *testing.T) { links[i] = alphanumIssueLink("https://someurl.com/someUser/someRepo/", "ref-issue", name) } expected := fmt.Sprintf(expectedFmt, links...) - testRenderIssueIndexPattern(t, s, expected, &postProcessCtx{metas: alphanumericMetas}) + testRenderIssueIndexPattern(t, s, expected, &RenderContext{Metas: alphanumericMetas}) } test("OTT-1234 test", "%s test", "OTT-1234") test("test T-12 issue", "test %s issue", "T-12") test("test issue ABCDEFGHIJ-1234567890", "test issue %s", "ABCDEFGHIJ-1234567890") } -func testRenderIssueIndexPattern(t *testing.T, input, expected string, ctx *postProcessCtx) { - if ctx == nil { - ctx = new(postProcessCtx) - } - ctx.procs = []processor{issueIndexPatternProcessor} - if ctx.urlPrefix == "" { - ctx.urlPrefix = AppSubURL +func testRenderIssueIndexPattern(t *testing.T, input, expected string, ctx *RenderContext) { + if ctx.URLPrefix == "" { + ctx.URLPrefix = AppSubURL } - res, err := ctx.postProcess([]byte(input)) + var buf strings.Builder + err := postProcess(ctx, []processor{issueIndexPatternProcessor}, strings.NewReader(input), &buf) assert.NoError(t, err) - assert.Equal(t, expected, string(res)) + assert.Equal(t, expected, buf.String()) } func TestRender_AutoLink(t *testing.T) { @@ -207,12 +204,22 @@ func TestRender_AutoLink(t *testing.T) { setting.AppSubURL = AppSubURL test := func(input, expected string) { - buffer, err := PostProcess([]byte(input), setting.AppSubURL, localMetas, false) + var buffer strings.Builder + err := PostProcess(&RenderContext{ + URLPrefix: setting.AppSubURL, + Metas: localMetas, + }, strings.NewReader(input), &buffer) assert.Equal(t, err, nil) - assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(string(buffer))) - buffer, err = PostProcess([]byte(input), setting.AppSubURL, localMetas, true) + assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer.String())) + + buffer.Reset() + err = PostProcess(&RenderContext{ + URLPrefix: setting.AppSubURL, + Metas: localMetas, + IsWiki: true, + }, strings.NewReader(input), &buffer) assert.Equal(t, err, nil) - assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(string(buffer))) + assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer.String())) } // render valid issue URLs @@ -235,15 +242,13 @@ func TestRender_FullIssueURLs(t *testing.T) { setting.AppSubURL = AppSubURL test := func(input, expected string) { - ctx := new(postProcessCtx) - ctx.procs = []processor{fullIssuePatternProcessor} - if ctx.urlPrefix == "" { - ctx.urlPrefix = AppSubURL - } - ctx.metas = localMetas - result, err := ctx.postProcess([]byte(input)) + var result strings.Builder + err := postProcess(&RenderContext{ + URLPrefix: AppSubURL, + Metas: localMetas, + }, []processor{fullIssuePatternProcessor}, strings.NewReader(input), &result) assert.NoError(t, err) - assert.Equal(t, expected, string(result)) + assert.Equal(t, expected, result.String()) } test("Here is a link https://git.osgeo.org/gogs/postgis/postgis/pulls/6", "Here is a link https://git.osgeo.org/gogs/postgis/postgis/pulls/6") diff --git a/modules/markup/html_test.go b/modules/markup/html_test.go index 1e39be401ba0a..02ca0dfe629b4 100644 --- a/modules/markup/html_test.go +++ b/modules/markup/html_test.go @@ -9,6 +9,7 @@ import ( "testing" "code.gitea.io/gitea/modules/emoji" + "code.gitea.io/gitea/modules/markup" . "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/markup/markdown" "code.gitea.io/gitea/modules/setting" @@ -28,7 +29,12 @@ func TestRender_Commits(t *testing.T) { setting.AppSubURL = AppSubURL test := func(input, expected string) { - buffer := RenderString(".md", input, setting.AppSubURL, localMetas) + buffer, err := RenderString(&markup.RenderContext{ + Filename: ".md", + URLPrefix: setting.AppSubURL, + Metas: localMetas, + }, input) + assert.NoError(t, err) assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer)) } @@ -59,7 +65,12 @@ func TestRender_CrossReferences(t *testing.T) { setting.AppSubURL = AppSubURL test := func(input, expected string) { - buffer := RenderString("a.md", input, setting.AppSubURL, localMetas) + buffer, err := RenderString(&markup.RenderContext{ + Filename: "a.md", + URLPrefix: setting.AppSubURL, + Metas: localMetas, + }, input) + assert.NoError(t, err) assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer)) } @@ -91,7 +102,11 @@ func TestRender_links(t *testing.T) { setting.AppSubURL = AppSubURL test := func(input, expected string) { - buffer := RenderString("a.md", input, setting.AppSubURL, nil) + buffer, err := RenderString(&markup.RenderContext{ + Filename: "a.md", + URLPrefix: setting.AppSubURL, + }, input) + assert.NoError(t, err) assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer)) } // Text that should be turned into URL @@ -187,8 +202,12 @@ func TestRender_email(t *testing.T) { setting.AppSubURL = AppSubURL test := func(input, expected string) { - buffer := RenderString("a.md", input, setting.AppSubURL, nil) - assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer)) + res, err := RenderString(&markup.RenderContext{ + Filename: "a.md", + URLPrefix: setting.AppSubURL, + }, input) + assert.NoError(t, err) + assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(res)) } // Text that should be turned into email link @@ -242,7 +261,11 @@ func TestRender_emoji(t *testing.T) { test := func(input, expected string) { expected = strings.ReplaceAll(expected, "&", "&") - buffer := RenderString("a.md", input, setting.AppSubURL, nil) + buffer, err := RenderString(&markup.RenderContext{ + Filename: "a.md", + URLPrefix: setting.AppSubURL, + }, input) + assert.NoError(t, err) assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer)) } @@ -291,9 +314,16 @@ func TestRender_ShortLinks(t *testing.T) { tree := util.URLJoin(AppSubURL, "src", "master") test := func(input, expected, expectedWiki string) { - buffer := markdown.RenderString(input, tree, nil) + buffer, err := markdown.RenderString(&markup.RenderContext{ + URLPrefix: tree, + }, input) + assert.NoError(t, err) assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer)) - buffer = markdown.RenderWiki([]byte(input), setting.AppSubURL, localMetas) + buffer, err = markdown.RenderString(&markup.RenderContext{ + URLPrefix: setting.AppSubURL, + Metas: localMetas, + }, input) + assert.NoError(t, err) assert.Equal(t, strings.TrimSpace(expectedWiki), strings.TrimSpace(buffer)) } @@ -395,16 +425,22 @@ func Test_ParseClusterFuzz(t *testing.T) { data := "
https://google.com/

` @@ -74,7 +82,10 @@ func TestRender_Images(t *testing.T) { setting.AppSubURL = AppSubURL test := func(input, expected string) { - buffer := RenderString(input, setting.AppSubURL, nil) + buffer, err := RenderString(&markup.RenderContext{ + URLPrefix: setting.AppSubURL, + }, input) + assert.NoError(t, err) assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer)) } @@ -261,7 +272,12 @@ func TestTotal_RenderWiki(t *testing.T) { answers := testAnswers(util.URLJoin(AppSubURL, "wiki/"), util.URLJoin(AppSubURL, "wiki", "raw/")) for i := 0; i < len(sameCases); i++ { - line := RenderWiki([]byte(sameCases[i]), AppSubURL, localMetas) + line, err := RenderString(&markup.RenderContext{ + URLPrefix: AppSubURL, + Metas: localMetas, + IsWiki: true, + }, sameCases[i]) + assert.NoError(t, err) assert.Equal(t, answers[i], line) } @@ -279,7 +295,10 @@ func TestTotal_RenderWiki(t *testing.T) { } for i := 0; i < len(testCases); i += 2 { - line := RenderWiki([]byte(testCases[i]), AppSubURL, nil) + line, err := RenderString(&markup.RenderContext{ + URLPrefix: AppSubURL, + }, testCases[i]) + assert.NoError(t, err) assert.Equal(t, testCases[i+1], line) } } @@ -288,30 +307,39 @@ func TestTotal_RenderString(t *testing.T) { answers := testAnswers(util.URLJoin(AppSubURL, "src", "master/"), util.URLJoin(AppSubURL, "raw", "master/")) for i := 0; i < len(sameCases); i++ { - line := RenderString(sameCases[i], util.URLJoin(AppSubURL, "src", "master/"), localMetas) + line, err := RenderString(&markup.RenderContext{ + URLPrefix: util.URLJoin(AppSubURL, "src", "master/"), + Metas: localMetas, + }, sameCases[i]) + assert.NoError(t, err) assert.Equal(t, answers[i], line) } testCases := []string{} for i := 0; i < len(testCases); i += 2 { - line := RenderString(testCases[i], AppSubURL, nil) + line, err := RenderString(&markup.RenderContext{ + URLPrefix: AppSubURL, + }, testCases[i]) + assert.NoError(t, err) assert.Equal(t, testCases[i+1], line) } } func TestRender_RenderParagraphs(t *testing.T) { test := func(t *testing.T, str string, cnt int) { - unix := []byte(str) - res := string(RenderRaw(unix, "", false)) + res, err := RenderString(&markup.RenderContext{}, str) + assert.NoError(t, err) assert.Equal(t, strings.Count(res, "image1
image2

` - res := string(RenderRaw([]byte(testcase), "", false)) + res, err := RenderString(&markup.RenderContext{}, testcase) + assert.NoError(t, err) assert.Equal(t, expected, res) } diff --git a/modules/markup/orgmode/orgmode.go b/modules/markup/orgmode/orgmode.go index b445b76956866..9bef57bd7f36e 100644 --- a/modules/markup/orgmode/orgmode.go +++ b/modules/markup/orgmode/orgmode.go @@ -8,9 +8,9 @@ import ( "bytes" "fmt" "html" + "io" "strings" - "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/util" @@ -18,58 +18,62 @@ import ( ) func init() { - markup.RegisterParser(Parser{}) + markup.RegisterRenderer(Renderer{}) } -// Parser implements markup.Parser for orgmode -type Parser struct { +// Renderer implements markup.Renderer for orgmode +type Renderer struct { } -// Name implements markup.Parser -func (Parser) Name() string { +// Name implements markup.Renderer +func (Renderer) Name() string { return "orgmode" } -// NeedPostProcess implements markup.Parser -func (Parser) NeedPostProcess() bool { return true } +// NeedPostProcess implements markup.Renderer +func (Renderer) NeedPostProcess() bool { return true } -// Extensions implements markup.Parser -func (Parser) Extensions() []string { +// Extensions implements markup.Renderer +func (Renderer) Extensions() []string { return []string{".org"} } // Render renders orgmode rawbytes to HTML -func Render(rawBytes []byte, urlPrefix string, metas map[string]string, isWiki bool) []byte { +func Render(ctx *markup.RenderContext, input io.Reader, output io.Writer) error { htmlWriter := org.NewHTMLWriter() - renderer := &Renderer{ + w := &Writer{ HTMLWriter: htmlWriter, - URLPrefix: urlPrefix, - IsWiki: isWiki, + URLPrefix: ctx.URLPrefix, + IsWiki: ctx.IsWiki, } - htmlWriter.ExtendingWriter = renderer + htmlWriter.ExtendingWriter = w - res, err := org.New().Silent().Parse(bytes.NewReader(rawBytes), "").Write(renderer) + res, err := org.New().Silent().Parse(input, "").Write(w) if err != nil { - log.Error("Panic in orgmode.Render: %v Just returning the rawBytes", err) - return rawBytes + return fmt.Errorf("orgmode.Render failed: %v", err) } - return []byte(res) + _, err = io.Copy(output, strings.NewReader(res)) + return err } -// RenderString reners orgmode string to HTML string -func RenderString(rawContent string, urlPrefix string, metas map[string]string, isWiki bool) string { - return string(Render([]byte(rawContent), urlPrefix, metas, isWiki)) +// RenderString renders Markdown string to HTML with all specific handling stuff and return string +func RenderString(ctx *markup.RenderContext, content string) (string, error) { + var buf strings.Builder + if err := Render(ctx, strings.NewReader(content), &buf); err != nil { + return "", err + } + return buf.String(), nil } -// Render reners orgmode string to HTML string -func (Parser) Render(rawBytes []byte, urlPrefix string, metas map[string]string, isWiki bool) []byte { - return Render(rawBytes, urlPrefix, metas, isWiki) +// Render renders orgmode string to HTML string +func (Renderer) Render(ctx *markup.RenderContext, input io.Reader, output io.Writer) error { + return Render(ctx, input, output) } -// Renderer implements org.Writer -type Renderer struct { +// Writer implements org.Writer +type Writer struct { *org.HTMLWriter URLPrefix string IsWiki bool @@ -78,7 +82,7 @@ type Renderer struct { var byteMailto = []byte("mailto:") // WriteRegularLink renders images, links or videos -func (r *Renderer) WriteRegularLink(l org.RegularLink) { +func (r *Writer) WriteRegularLink(l org.RegularLink) { link := []byte(html.EscapeString(l.URL)) if l.Protocol == "file" { link = link[len("file:"):] diff --git a/modules/markup/orgmode/orgmode_test.go b/modules/markup/orgmode/orgmode_test.go index 020a3f592ad85..da89326e9e13e 100644 --- a/modules/markup/orgmode/orgmode_test.go +++ b/modules/markup/orgmode/orgmode_test.go @@ -8,6 +8,7 @@ import ( "strings" "testing" + "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" @@ -23,7 +24,10 @@ func TestRender_StandardLinks(t *testing.T) { setting.AppSubURL = AppSubURL test := func(input, expected string) { - buffer := RenderString(input, setting.AppSubURL, nil, false) + buffer, err := RenderString(&markup.RenderContext{ + URLPrefix: setting.AppSubURL, + }, input) + assert.NoError(t, err) assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer)) } @@ -40,7 +44,10 @@ func TestRender_Images(t *testing.T) { setting.AppSubURL = AppSubURL test := func(input, expected string) { - buffer := RenderString(input, setting.AppSubURL, nil, false) + buffer, err := RenderString(&markup.RenderContext{ + URLPrefix: setting.AppSubURL, + }, input) + assert.NoError(t, err) assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer)) } diff --git a/modules/markup/renderer.go b/modules/markup/renderer.go new file mode 100644 index 0000000000000..bbbca310c0c55 --- /dev/null +++ b/modules/markup/renderer.go @@ -0,0 +1,157 @@ +// Copyright 2017 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package markup + +import ( + "context" + "errors" + "fmt" + "io" + "path/filepath" + "strings" + + "code.gitea.io/gitea/modules/setting" +) + +// Init initialize regexps for markdown parsing +func Init() { + getIssueFullPattern() + NewSanitizer() + if len(setting.Markdown.CustomURLSchemes) > 0 { + CustomLinkURLSchemes(setting.Markdown.CustomURLSchemes) + } + + // since setting maybe changed extensions, this will reload all renderer extensions mapping + extRenderers = make(map[string]Renderer) + for _, renderer := range renderers { + for _, ext := range renderer.Extensions() { + extRenderers[strings.ToLower(ext)] = renderer + } + } +} + +// RenderContext represents a render context +type RenderContext struct { + Ctx context.Context + Filename string + Type string + IsWiki bool + URLPrefix string + Metas map[string]string + DefaultLink string +} + +// Renderer defines an interface for rendering markup file to HTML +type Renderer interface { + Name() string // markup format name + Extensions() []string + Render(ctx *RenderContext, input io.Reader, output io.Writer) error +} + +var ( + extRenderers = make(map[string]Renderer) + renderers = make(map[string]Renderer) +) + +// RegisterRenderer registers a new markup file renderer +func RegisterRenderer(renderer Renderer) { + renderers[renderer.Name()] = renderer + for _, ext := range renderer.Extensions() { + extRenderers[strings.ToLower(ext)] = renderer + } +} + +// GetRendererByFileName get renderer by filename +func GetRendererByFileName(filename string) Renderer { + extension := strings.ToLower(filepath.Ext(filename)) + return extRenderers[extension] +} + +// GetRendererByType returns a renderer according type +func GetRendererByType(tp string) Renderer { + return renderers[tp] +} + +// Render renders markup file to HTML with all specific handling stuff. +func Render(ctx *RenderContext, input io.Reader, output io.Writer) error { + if ctx.Type != "" { + return renderByType(ctx, input, output) + } else if ctx.Filename != "" { + return renderFile(ctx, input, output) + } + return errors.New("Render options both filename and type missing") +} + +// RenderString renders Markup string to HTML with all specific handling stuff and return string +func RenderString(ctx *RenderContext, content string) (string, error) { + var buf strings.Builder + if err := Render(ctx, strings.NewReader(content), &buf); err != nil { + return "", err + } + return buf.String(), nil +} + +func render(ctx *RenderContext, parser Renderer, input io.Reader, output io.Writer) error { + var buf1 strings.Builder + if err := parser.Render(ctx, input, &buf1); err != nil { + return err + } + + var buf2 strings.Builder + if err := PostProcess(ctx, strings.NewReader(buf1.String()), &buf2); err != nil { + return fmt.Errorf("PostProcess: %v", err) + } + buf := SanitizeReader(strings.NewReader(buf2.String())) + _, err := io.Copy(output, buf) + return err +} + +func renderByType(ctx *RenderContext, input io.Reader, output io.Writer) error { + if parser, ok := renderers[ctx.Type]; ok { + return render(ctx, parser, input, output) + } + return nil +} + +func renderFile(ctx *RenderContext, input io.Reader, output io.Writer) error { + extension := strings.ToLower(filepath.Ext(ctx.Filename)) + if parser, ok := extRenderers[extension]; ok { + return render(ctx, parser, input, output) + } + return nil +} + +// Type returns if markup format via the filename +func Type(filename string) string { + if parser := GetRendererByFileName(filename); parser != nil { + return parser.Name() + } + return "" +} + +// IsMarkupFile reports whether file is a markup type file +func IsMarkupFile(name, markup string) bool { + if parser := GetRendererByFileName(name); parser != nil { + return parser.Name() == markup + } + return false +} + +// IsReadmeFile reports whether name looks like a README file +// based on its name. If an extension is provided, it will strictly +// match that extension. +// Note that the '.' should be provided in ext, e.g ".md" +func IsReadmeFile(name string, ext ...string) bool { + name = strings.ToLower(name) + if len(ext) > 0 { + return name == "readme"+ext[0] + } + if len(name) < 6 { + return false + } else if len(name) == 6 { + return name == "readme" + } + return name[:7] == "readme." +} diff --git a/modules/markup/markup_test.go b/modules/markup/renderer_test.go similarity index 100% rename from modules/markup/markup_test.go rename to modules/markup/renderer_test.go diff --git a/modules/notification/mail/mail.go b/modules/notification/mail/mail.go index 9c000da0f6c71..bd02a6c252d4e 100644 --- a/modules/notification/mail/mail.go +++ b/modules/notification/mail/mail.go @@ -104,14 +104,18 @@ func (m *mailNotifier) NotifyIssueChangeAssignee(doer *models.User, issue *model // mail only sent to added assignees and not self-assignee if !removed && doer.ID != assignee.ID && assignee.EmailNotifications() == models.EmailNotificationsEnabled { ct := fmt.Sprintf("Assigned #%d.", issue.Index) - mailer.SendIssueAssignedMail(issue, doer, ct, comment, []*models.User{assignee}) + if err := mailer.SendIssueAssignedMail(issue, doer, ct, comment, []*models.User{assignee}); err != nil { + log.Error("SendIssueAssignedMail faile: %v", err) + } } } func (m *mailNotifier) NotifyPullReviewRequest(doer *models.User, issue *models.Issue, reviewer *models.User, isRequest bool, comment *models.Comment) { if isRequest && doer.ID != reviewer.ID && reviewer.EmailNotifications() == models.EmailNotificationsEnabled { ct := fmt.Sprintf("Requested to review %s.", issue.HTMLURL()) - mailer.SendIssueAssignedMail(issue, doer, ct, comment, []*models.User{reviewer}) + if err := mailer.SendIssueAssignedMail(issue, doer, ct, comment, []*models.User{reviewer}); err != nil { + log.Error("SendIssueAssignedMail faile: %v", err) + } } } diff --git a/modules/setting/markup.go b/modules/setting/markup.go index 36cba68262d52..f0849a863a56f 100644 --- a/modules/setting/markup.go +++ b/modules/setting/markup.go @@ -13,14 +13,14 @@ import ( "gopkg.in/ini.v1" ) -// ExternalMarkupParsers represents the external markup parsers +// ExternalMarkupRenderers represents the external markup renderers var ( - ExternalMarkupParsers []MarkupParser - ExternalSanitizerRules []MarkupSanitizerRule + ExternalMarkupRenderers []MarkupRenderer + ExternalSanitizerRules []MarkupSanitizerRule ) -// MarkupParser defines the external parser configured in ini -type MarkupParser struct { +// MarkupRenderer defines the external parser configured in ini +type MarkupRenderer struct { Enabled bool MarkupName string Command string @@ -124,7 +124,7 @@ func newMarkupRenderer(name string, sec *ini.Section) { return } - ExternalMarkupParsers = append(ExternalMarkupParsers, MarkupParser{ + ExternalMarkupRenderers = append(ExternalMarkupRenderers, MarkupRenderer{ Enabled: sec.Key("ENABLED").MustBool(false), MarkupName: name, FileExtensions: exts, diff --git a/modules/templates/helper.go b/modules/templates/helper.go index 7e33f262094ee..7b175bfab3e76 100644 --- a/modules/templates/helper.go +++ b/modules/templates/helper.go @@ -665,7 +665,11 @@ func RenderCommitMessageLink(msg, urlPrefix, urlDefault string, metas map[string cleanMsg := template.HTMLEscapeString(msg) // we can safely assume that it will not return any error, since there // shouldn't be any special HTML. - fullMessage, err := markup.RenderCommitMessage([]byte(cleanMsg), urlPrefix, urlDefault, metas) + fullMessage, err := markup.RenderCommitMessage(&markup.RenderContext{ + URLPrefix: urlPrefix, + DefaultLink: urlDefault, + Metas: metas, + }, cleanMsg) if err != nil { log.Error("RenderCommitMessage: %v", err) return "" @@ -692,7 +696,11 @@ func RenderCommitMessageLinkSubject(msg, urlPrefix, urlDefault string, metas map // we can safely assume that it will not return any error, since there // shouldn't be any special HTML. - renderedMessage, err := markup.RenderCommitMessageSubject([]byte(template.HTMLEscapeString(msgLine)), urlPrefix, urlDefault, metas) + renderedMessage, err := markup.RenderCommitMessageSubject(&markup.RenderContext{ + URLPrefix: urlPrefix, + DefaultLink: urlDefault, + Metas: metas, + }, template.HTMLEscapeString(msgLine)) if err != nil { log.Error("RenderCommitMessageSubject: %v", err) return template.HTML("") @@ -714,7 +722,10 @@ func RenderCommitBody(msg, urlPrefix string, metas map[string]string) template.H return template.HTML("") } - renderedMessage, err := markup.RenderCommitMessage([]byte(template.HTMLEscapeString(msgLine)), urlPrefix, "", metas) + renderedMessage, err := markup.RenderCommitMessage(&markup.RenderContext{ + URLPrefix: urlPrefix, + Metas: metas, + }, template.HTMLEscapeString(msgLine)) if err != nil { log.Error("RenderCommitMessage: %v", err) return "" @@ -724,7 +735,10 @@ func RenderCommitBody(msg, urlPrefix string, metas map[string]string) template.H // RenderIssueTitle renders issue/pull title with defined post processors func RenderIssueTitle(text, urlPrefix string, metas map[string]string) template.HTML { - renderedText, err := markup.RenderIssueTitle([]byte(template.HTMLEscapeString(text)), urlPrefix, metas) + renderedText, err := markup.RenderIssueTitle(&markup.RenderContext{ + URLPrefix: urlPrefix, + Metas: metas, + }, template.HTMLEscapeString(text)) if err != nil { log.Error("RenderIssueTitle: %v", err) return template.HTML("") @@ -734,7 +748,7 @@ func RenderIssueTitle(text, urlPrefix string, metas map[string]string) template. // RenderEmoji renders html text with emoji post processors func RenderEmoji(text string) template.HTML { - renderedText, err := markup.RenderEmoji([]byte(template.HTMLEscapeString(text))) + renderedText, err := markup.RenderEmoji(template.HTMLEscapeString(text)) if err != nil { log.Error("RenderEmoji: %v", err) return template.HTML("") @@ -758,7 +772,10 @@ func ReactionToEmoji(reaction string) template.HTML { // RenderNote renders the contents of a git-notes file as a commit message. func RenderNote(msg, urlPrefix string, metas map[string]string) template.HTML { cleanMsg := template.HTMLEscapeString(msg) - fullMessage, err := markup.RenderCommitMessage([]byte(cleanMsg), urlPrefix, "", metas) + fullMessage, err := markup.RenderCommitMessage(&markup.RenderContext{ + URLPrefix: urlPrefix, + Metas: metas, + }, cleanMsg) if err != nil { log.Error("RenderNote: %v", err) return "" diff --git a/modules/util/util.go b/modules/util/util.go index 9de1710ac7bad..0072d0e7be600 100644 --- a/modules/util/util.go +++ b/modules/util/util.go @@ -5,8 +5,10 @@ package util import ( + "bufio" "bytes" "errors" + "io" "strings" ) @@ -66,6 +68,17 @@ func IsEmptyString(s string) bool { return len(strings.TrimSpace(s)) == 0 } +// NormalizeEOLReader will convert Windows (CRLF) and Mac (CR) EOLs to UNIX (LF) from a reader +func NormalizeEOLReader(rd io.Reader) []byte { + scanner := bufio.NewScanner(rd) + var buf bytes.Buffer + for scanner.Scan() { + buf.Write(scanner.Bytes()) + buf.WriteByte('\n') + } + return buf.Bytes() +} + // NormalizeEOL will convert Windows (CRLF) and Mac (CR) EOLs to UNIX (LF) func NormalizeEOL(input []byte) []byte { var right, left, pos int diff --git a/routers/api/v1/misc/markdown.go b/routers/api/v1/misc/markdown.go index 571818530981f..87e65003f0062 100644 --- a/routers/api/v1/misc/markdown.go +++ b/routers/api/v1/misc/markdown.go @@ -5,11 +5,11 @@ package misc import ( - "io/ioutil" "net/http" "strings" "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/markup/markdown" "code.gitea.io/gitea/modules/setting" api "code.gitea.io/gitea/modules/structs" @@ -55,7 +55,6 @@ func Markdown(ctx *context.APIContext) { case "comment": fallthrough case "gfm": - md := []byte(form.Text) urlPrefix := form.Context meta := map[string]string{} if !strings.HasPrefix(setting.AppSubURL+"/", urlPrefix) { @@ -77,22 +76,17 @@ func Markdown(ctx *context.APIContext) { if form.Mode == "gfm" { meta["mode"] = "document" } - if form.Wiki { - _, err := ctx.Write([]byte(markdown.RenderWiki(md, urlPrefix, meta))) - if err != nil { - ctx.InternalServerError(err) - return - } - } else { - _, err := ctx.Write(markdown.Render(md, urlPrefix, meta)) - if err != nil { - ctx.InternalServerError(err) - return - } + + if err := markdown.Render(&markup.RenderContext{ + URLPrefix: urlPrefix, + Metas: meta, + IsWiki: form.Wiki, + }, strings.NewReader(form.Text), ctx.Resp); err != nil { + ctx.InternalServerError(err) + return } default: - _, err := ctx.Write(markdown.RenderRaw([]byte(form.Text), "", false)) - if err != nil { + if err := markdown.Render(&markup.RenderContext{}, strings.NewReader(form.Text), ctx.Resp); err != nil { ctx.InternalServerError(err) return } @@ -120,14 +114,8 @@ func MarkdownRaw(ctx *context.APIContext) { // "$ref": "#/responses/MarkdownRender" // "422": // "$ref": "#/responses/validationError" - - body, err := ioutil.ReadAll(ctx.Req.Body) - if err != nil { - ctx.Error(http.StatusUnprocessableEntity, "", err) - return - } - _, err = ctx.Write(markdown.RenderRaw(body, "", false)) - if err != nil { + defer ctx.Req.Body.Close() + if err := markdown.Render(&markup.RenderContext{}, ctx.Req.Body, ctx.Resp); err != nil { ctx.InternalServerError(err) return } diff --git a/routers/init.go b/routers/init.go index f5dbfc87d2721..220d87a29da87 100644 --- a/routers/init.go +++ b/routers/init.go @@ -143,7 +143,7 @@ func GlobalInit(ctx context.Context) { NewServices() highlight.NewContext() - external.RegisterParsers() + external.RegisterRenderers() markup.Init() if setting.EnableSQLite3 { diff --git a/routers/org/home.go b/routers/org/home.go index 9a40d8be6ab4a..d84ae870ab6db 100644 --- a/routers/org/home.go +++ b/routers/org/home.go @@ -11,6 +11,7 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/markup/markdown" "code.gitea.io/gitea/modules/setting" ) @@ -37,7 +38,15 @@ func Home(ctx *context.Context) { ctx.Data["PageIsUserProfile"] = true ctx.Data["Title"] = org.DisplayName() if len(org.Description) != 0 { - ctx.Data["RenderedDescription"] = string(markdown.Render([]byte(org.Description), ctx.Repo.RepoLink, map[string]string{"mode": "document"})) + desc, err := markdown.RenderString(&markup.RenderContext{ + URLPrefix: ctx.Repo.RepoLink, + Metas: map[string]string{"mode": "document"}, + }, org.Description) + if err != nil { + ctx.ServerError("RenderString", err) + return + } + ctx.Data["RenderedDescription"] = desc } var orderBy models.SearchOrderBy diff --git a/routers/repo/compare.go b/routers/repo/compare.go index 7046f3ecdb601..f7b9d2122cfdc 100644 --- a/routers/repo/compare.go +++ b/routers/repo/compare.go @@ -6,6 +6,7 @@ package repo import ( "bufio" + "bytes" "encoding/csv" "errors" "fmt" @@ -124,7 +125,7 @@ func setCsvCompareContext(ctx *context.Context) { b = charset.ToUTF8WithFallback(b) - return csv_module.CreateReaderAndGuessDelimiter(b), nil + return csv_module.CreateReaderAndGuessDelimiter(bytes.NewReader(b)) } baseReader, err := csvReaderFromCommit(baseCommit) diff --git a/routers/repo/issue.go b/routers/repo/issue.go index 7471bb65a4ef0..12726cd22c9a0 100644 --- a/routers/repo/issue.go +++ b/routers/repo/issue.go @@ -1131,8 +1131,14 @@ func ViewIssue(ctx *context.Context) { } ctx.Data["IssueWatch"] = iw - issue.RenderedContent = string(markdown.Render([]byte(issue.Content), ctx.Repo.RepoLink, - ctx.Repo.Repository.ComposeMetas())) + issue.RenderedContent, err = markdown.RenderString(&markup.RenderContext{ + URLPrefix: ctx.Repo.RepoLink, + Metas: ctx.Repo.Repository.ComposeMetas(), + }, issue.Content) + if err != nil { + ctx.ServerError("RenderString", err) + return + } repo := ctx.Repo.Repository @@ -1289,9 +1295,14 @@ func ViewIssue(ctx *context.Context) { return } - comment.RenderedContent = string(markdown.Render([]byte(comment.Content), ctx.Repo.RepoLink, - ctx.Repo.Repository.ComposeMetas())) - + comment.RenderedContent, err = markdown.RenderString(&markup.RenderContext{ + URLPrefix: ctx.Repo.RepoLink, + Metas: ctx.Repo.Repository.ComposeMetas(), + }, comment.Content) + if err != nil { + ctx.ServerError("RenderString", err) + return + } // Check tag. tag, ok = marked[comment.PosterID] if ok { @@ -1359,8 +1370,14 @@ func ViewIssue(ctx *context.Context) { } } } else if comment.Type == models.CommentTypeCode || comment.Type == models.CommentTypeReview || comment.Type == models.CommentTypeDismissReview { - comment.RenderedContent = string(markdown.Render([]byte(comment.Content), ctx.Repo.RepoLink, - ctx.Repo.Repository.ComposeMetas())) + comment.RenderedContent, err = markdown.RenderString(&markup.RenderContext{ + URLPrefix: ctx.Repo.RepoLink, + Metas: ctx.Repo.Repository.ComposeMetas(), + }, comment.Content) + if err != nil { + ctx.ServerError("RenderString", err) + return + } if err = comment.LoadReview(); err != nil && !models.IsErrReviewNotExist(err) { ctx.ServerError("LoadReview", err) return @@ -1708,10 +1725,20 @@ func UpdateIssueContent(ctx *context.Context) { files := ctx.QueryStrings("files[]") if err := updateAttachments(issue, files); err != nil { ctx.ServerError("UpdateAttachments", err) + return + } + + content, err := markdown.RenderString(&markup.RenderContext{ + URLPrefix: ctx.Query("context"), + Metas: ctx.Repo.Repository.ComposeMetas(), + }, issue.Content) + if err != nil { + ctx.ServerError("RenderString", err) + return } ctx.JSON(http.StatusOK, map[string]interface{}{ - "content": string(markdown.Render([]byte(issue.Content), ctx.Query("context"), ctx.Repo.Repository.ComposeMetas())), + "content": content, "attachments": attachmentsHTML(ctx, issue.Attachments, issue.Content), }) } @@ -2125,10 +2152,20 @@ func UpdateCommentContent(ctx *context.Context) { files := ctx.QueryStrings("files[]") if err := updateAttachments(comment, files); err != nil { ctx.ServerError("UpdateAttachments", err) + return + } + + content, err := markdown.RenderString(&markup.RenderContext{ + URLPrefix: ctx.Query("context"), + Metas: ctx.Repo.Repository.ComposeMetas(), + }, comment.Content) + if err != nil { + ctx.ServerError("RenderString", err) + return } ctx.JSON(http.StatusOK, map[string]interface{}{ - "content": string(markdown.Render([]byte(comment.Content), ctx.Query("context"), ctx.Repo.Repository.ComposeMetas())), + "content": content, "attachments": attachmentsHTML(ctx, comment.Attachments, comment.Content), }) } diff --git a/routers/repo/milestone.go b/routers/repo/milestone.go index 5a9d2351bcfb5..bb6b310cbe8d4 100644 --- a/routers/repo/milestone.go +++ b/routers/repo/milestone.go @@ -12,6 +12,7 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/markup/markdown" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/structs" @@ -84,7 +85,14 @@ func Milestones(ctx *context.Context) { } } for _, m := range miles { - m.RenderedContent = string(markdown.Render([]byte(m.Content), ctx.Repo.RepoLink, ctx.Repo.Repository.ComposeMetas())) + m.RenderedContent, err = markdown.RenderString(&markup.RenderContext{ + URLPrefix: ctx.Repo.RepoLink, + Metas: ctx.Repo.Repository.ComposeMetas(), + }, m.Content) + if err != nil { + ctx.ServerError("RenderString", err) + return + } } ctx.Data["Milestones"] = miles @@ -269,7 +277,14 @@ func MilestoneIssuesAndPulls(ctx *context.Context) { return } - milestone.RenderedContent = string(markdown.Render([]byte(milestone.Content), ctx.Repo.RepoLink, ctx.Repo.Repository.ComposeMetas())) + milestone.RenderedContent, err = markdown.RenderString(&markup.RenderContext{ + URLPrefix: ctx.Repo.RepoLink, + Metas: ctx.Repo.Repository.ComposeMetas(), + }, milestone.Content) + if err != nil { + ctx.ServerError("RenderString", err) + return + } ctx.Data["Title"] = milestone.Name ctx.Data["Milestone"] = milestone diff --git a/routers/repo/projects.go b/routers/repo/projects.go index 96ef2c6c0c024..eb0719995cb55 100644 --- a/routers/repo/projects.go +++ b/routers/repo/projects.go @@ -12,6 +12,7 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/markup/markdown" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" @@ -77,7 +78,14 @@ func Projects(ctx *context.Context) { } for i := range projects { - projects[i].RenderedContent = string(markdown.Render([]byte(projects[i].Description), ctx.Repo.RepoLink, ctx.Repo.Repository.ComposeMetas())) + projects[i].RenderedContent, err = markdown.RenderString(&markup.RenderContext{ + URLPrefix: ctx.Repo.RepoLink, + Metas: ctx.Repo.Repository.ComposeMetas(), + }, projects[i].Description) + if err != nil { + ctx.ServerError("RenderString", err) + return + } } ctx.Data["Projects"] = projects @@ -311,7 +319,14 @@ func ViewProject(ctx *context.Context) { } ctx.Data["LinkedPRs"] = linkedPrsMap - project.RenderedContent = string(markdown.Render([]byte(project.Description), ctx.Repo.RepoLink, ctx.Repo.Repository.ComposeMetas())) + project.RenderedContent, err = markdown.RenderString(&markup.RenderContext{ + URLPrefix: ctx.Repo.RepoLink, + Metas: ctx.Repo.Repository.ComposeMetas(), + }, project.Description) + if err != nil { + ctx.ServerError("RenderString", err) + return + } ctx.Data["CanWriteProjects"] = ctx.Repo.Permission.CanWrite(models.UnitTypeProjects) ctx.Data["Project"] = project diff --git a/routers/repo/release.go b/routers/repo/release.go index 2ebb69b6ab083..abce3e9ac1a28 100644 --- a/routers/repo/release.go +++ b/routers/repo/release.go @@ -15,6 +15,7 @@ import ( "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/convert" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/markup/markdown" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/upload" @@ -132,7 +133,14 @@ func releasesOrTags(ctx *context.Context, isTagList bool) { ctx.ServerError("calReleaseNumCommitsBehind", err) return } - r.Note = markdown.RenderString(r.Note, ctx.Repo.RepoLink, ctx.Repo.Repository.ComposeMetas()) + r.Note, err = markdown.RenderString(&markup.RenderContext{ + URLPrefix: ctx.Repo.RepoLink, + Metas: ctx.Repo.Repository.ComposeMetas(), + }, r.Note) + if err != nil { + ctx.ServerError("RenderString", err) + return + } } ctx.Data["Releases"] = releases @@ -182,7 +190,14 @@ func SingleRelease(ctx *context.Context) { ctx.ServerError("calReleaseNumCommitsBehind", err) return } - release.Note = markdown.RenderString(release.Note, ctx.Repo.RepoLink, ctx.Repo.Repository.ComposeMetas()) + release.Note, err = markdown.RenderString(&markup.RenderContext{ + URLPrefix: ctx.Repo.RepoLink, + Metas: ctx.Repo.Repository.ComposeMetas(), + }, release.Note) + if err != nil { + ctx.ServerError("RenderString", err) + return + } ctx.Data["Releases"] = []*models.Release{release} ctx.HTML(http.StatusOK, tplReleases) diff --git a/routers/repo/view.go b/routers/repo/view.go index a03fd58c8aae9..d8e9c89a5931e 100644 --- a/routers/repo/view.go +++ b/routers/repo/view.go @@ -330,7 +330,17 @@ func renderDirectory(ctx *context.Context, treeLink string) { if markupType := markup.Type(readmeFile.name); markupType != "" { ctx.Data["IsMarkup"] = true ctx.Data["MarkupType"] = string(markupType) - ctx.Data["FileContent"] = string(markup.Render(readmeFile.name, buf, readmeTreelink, ctx.Repo.Repository.ComposeDocumentMetas())) + var result strings.Builder + err := markup.Render(&markup.RenderContext{ + Filename: readmeFile.name, + URLPrefix: readmeTreelink, + Metas: ctx.Repo.Repository.ComposeDocumentMetas(), + }, bytes.NewReader(buf), &result) + if err != nil { + ctx.ServerError("Render", err) + return + } + ctx.Data["FileContent"] = result.String() } else { ctx.Data["IsRenderedHTML"] = true ctx.Data["FileContent"] = strings.ReplaceAll( @@ -488,7 +498,17 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st if markupType := markup.Type(blob.Name()); markupType != "" { ctx.Data["IsMarkup"] = true ctx.Data["MarkupType"] = markupType - ctx.Data["FileContent"] = string(markup.Render(blob.Name(), buf, path.Dir(treeLink), ctx.Repo.Repository.ComposeDocumentMetas())) + var result strings.Builder + err := markup.Render(&markup.RenderContext{ + Filename: blob.Name(), + URLPrefix: path.Dir(treeLink), + Metas: ctx.Repo.Repository.ComposeDocumentMetas(), + }, bytes.NewReader(buf), &result) + if err != nil { + ctx.ServerError("Render", err) + return + } + ctx.Data["FileContent"] = result.String() } else if readmeExist { ctx.Data["IsRenderedHTML"] = true ctx.Data["FileContent"] = strings.ReplaceAll( @@ -536,7 +556,17 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st buf = append(buf, d...) ctx.Data["IsMarkup"] = true ctx.Data["MarkupType"] = markupType - ctx.Data["FileContent"] = string(markup.Render(blob.Name(), buf, path.Dir(treeLink), ctx.Repo.Repository.ComposeDocumentMetas())) + var result strings.Builder + err := markup.Render(&markup.RenderContext{ + Filename: blob.Name(), + URLPrefix: path.Dir(treeLink), + Metas: ctx.Repo.Repository.ComposeDocumentMetas(), + }, bytes.NewReader(buf), &result) + if err != nil { + ctx.ServerError("Render", err) + return + } + ctx.Data["FileContent"] = result.String() } } diff --git a/routers/repo/wiki.go b/routers/repo/wiki.go index 290e2e8bb294a..1bdd06dce57d7 100644 --- a/routers/repo/wiki.go +++ b/routers/repo/wiki.go @@ -6,6 +6,7 @@ package repo import ( + "bytes" "fmt" "io/ioutil" "net/http" @@ -211,12 +212,34 @@ func renderViewPage(ctx *context.Context) (*git.Repository, *git.TreeEntry) { return nil, nil } - metas := ctx.Repo.Repository.ComposeDocumentMetas() - ctx.Data["content"] = markdown.RenderWiki(data, ctx.Repo.RepoLink, metas) + var rctx = &markup.RenderContext{ + URLPrefix: ctx.Repo.RepoLink, + Metas: ctx.Repo.Repository.ComposeDocumentMetas(), + IsWiki: true, + } + + var buf strings.Builder + if err := markdown.Render(rctx, bytes.NewReader(data), &buf); err != nil { + ctx.ServerError("Render", err) + return nil, nil + } + ctx.Data["content"] = buf.String() + + buf.Reset() + if err := markdown.Render(rctx, bytes.NewReader(sidebarContent), &buf); err != nil { + ctx.ServerError("Render", err) + return nil, nil + } ctx.Data["sidebarPresent"] = sidebarContent != nil - ctx.Data["sidebarContent"] = markdown.RenderWiki(sidebarContent, ctx.Repo.RepoLink, metas) + ctx.Data["sidebarContent"] = buf.String() + + buf.Reset() + if err := markdown.Render(rctx, bytes.NewReader(footerContent), &buf); err != nil { + ctx.ServerError("Render", err) + return nil, nil + } ctx.Data["footerPresent"] = footerContent != nil - ctx.Data["footerContent"] = markdown.RenderWiki(footerContent, ctx.Repo.RepoLink, metas) + ctx.Data["footerContent"] = buf.String() // get commit count - wiki revisions commitsCount, _ := wikiRepo.FileCommitsCount("master", pageFilename) diff --git a/routers/user/home.go b/routers/user/home.go index 584bc019fab74..acf73f82fe362 100644 --- a/routers/user/home.go +++ b/routers/user/home.go @@ -19,6 +19,7 @@ import ( "code.gitea.io/gitea/modules/context" issue_indexer "code.gitea.io/gitea/modules/indexer/issues" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/markup/markdown" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" @@ -267,7 +268,15 @@ func Milestones(ctx *context.Context) { continue } - milestones[i].RenderedContent = string(markdown.Render([]byte(milestones[i].Content), milestones[i].Repo.Link(), milestones[i].Repo.ComposeMetas())) + milestones[i].RenderedContent, err = markdown.RenderString(&markup.RenderContext{ + URLPrefix: milestones[i].Repo.Link(), + Metas: milestones[i].Repo.ComposeMetas(), + }, milestones[i].Content) + if err != nil { + ctx.ServerError("RenderString", err) + return + } + if milestones[i].Repo.IsTimetrackerEnabled() { err := milestones[i].LoadTotalTrackedTime() if err != nil { diff --git a/routers/user/profile.go b/routers/user/profile.go index c24614b108fc2..bb4c0cd5b166b 100644 --- a/routers/user/profile.go +++ b/routers/user/profile.go @@ -13,6 +13,7 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/markup/markdown" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" @@ -110,7 +111,15 @@ func Profile(ctx *context.Context) { } if len(ctxUser.Description) != 0 { - ctx.Data["RenderedDescription"] = string(markdown.Render([]byte(ctxUser.Description), ctx.Repo.RepoLink, map[string]string{"mode": "document"})) + content, err := markdown.RenderString(&markup.RenderContext{ + URLPrefix: ctx.Repo.RepoLink, + Metas: map[string]string{"mode": "document"}, + }, ctxUser.Description) + if err != nil { + ctx.ServerError("RenderString", err) + return + } + ctx.Data["RenderedDescription"] = content } showPrivate := ctx.IsSigned && (ctx.User.IsAdmin || ctx.User.ID == ctxUser.ID) diff --git a/services/mailer/mail.go b/services/mailer/mail.go index c50795968aa64..f22140c9f762e 100644 --- a/services/mailer/mail.go +++ b/services/mailer/mail.go @@ -174,8 +174,7 @@ func SendCollaboratorMail(u, doer *models.User, repo *models.Repository) { SendAsync(msg) } -func composeIssueCommentMessages(ctx *mailCommentContext, lang string, tos []string, fromMention bool, info string) []*Message { - +func composeIssueCommentMessages(ctx *mailCommentContext, lang string, tos []string, fromMention bool, info string) ([]*Message, error) { var ( subject string link string @@ -199,7 +198,14 @@ func composeIssueCommentMessages(ctx *mailCommentContext, lang string, tos []str } // This is the body of the new issue or comment, not the mail body - body := string(markup.RenderByType(markdown.MarkupName, []byte(ctx.Content), ctx.Issue.Repo.HTMLURL(), ctx.Issue.Repo.ComposeMetas())) + body, err := markdown.RenderString(&markup.RenderContext{ + URLPrefix: ctx.Issue.Repo.HTMLURL(), + Metas: ctx.Issue.Repo.ComposeMetas(), + }, ctx.Content) + if err != nil { + return nil, err + } + actType, actName, tplName := actionToTemplate(ctx.Issue, ctx.ActionType, commentType, reviewType) if actName != "new" { @@ -240,14 +246,13 @@ func composeIssueCommentMessages(ctx *mailCommentContext, lang string, tos []str // TODO: i18n templates? if err := subjectTemplates.ExecuteTemplate(&mailSubject, string(tplName), mailMeta); err == nil { subject = sanitizeSubject(mailSubject.String()) + if subject == "" { + subject = fallback + } } else { log.Error("ExecuteTemplate [%s]: %v", tplName+"/subject", err) } - if subject == "" { - subject = fallback - } - subject = emoji.ReplaceAliases(subject) mailMeta["Subject"] = subject @@ -275,7 +280,7 @@ func composeIssueCommentMessages(ctx *mailCommentContext, lang string, tos []str msgs = append(msgs, msg) } - return msgs + return msgs, nil } func sanitizeSubject(subject string) string { @@ -288,21 +293,26 @@ func sanitizeSubject(subject string) string { } // SendIssueAssignedMail composes and sends issue assigned email -func SendIssueAssignedMail(issue *models.Issue, doer *models.User, content string, comment *models.Comment, recipients []*models.User) { +func SendIssueAssignedMail(issue *models.Issue, doer *models.User, content string, comment *models.Comment, recipients []*models.User) error { langMap := make(map[string][]string) for _, user := range recipients { langMap[user.Language] = append(langMap[user.Language], user.Email) } for lang, tos := range langMap { - SendAsyncs(composeIssueCommentMessages(&mailCommentContext{ + msgs, err := composeIssueCommentMessages(&mailCommentContext{ Issue: issue, Doer: doer, ActionType: models.ActionType(0), Content: content, Comment: comment, - }, lang, tos, false, "issue assigned")) + }, lang, tos, false, "issue assigned") + if err != nil { + return err + } + SendAsyncs(msgs) } + return nil } // actionToTemplate returns the type and name of the action facing the user diff --git a/services/mailer/mail_issue.go b/services/mailer/mail_issue.go index 9786a06f62f3b..bb541d27a091a 100644 --- a/services/mailer/mail_issue.go +++ b/services/mailer/mail_issue.go @@ -146,7 +146,11 @@ func mailIssueCommentBatch(ctx *mailCommentContext, users []*models.User, visite // working backwards from the last (possibly) incomplete batch. If len(receivers) can be 0 this // starting condition will need to be changed slightly for i := ((len(receivers) - 1) / MailBatchSize) * MailBatchSize; i >= 0; i -= MailBatchSize { - SendAsyncs(composeIssueCommentMessages(ctx, lang, receivers[i:], fromMention, "issue comments")) + msgs, err := composeIssueCommentMessages(ctx, lang, receivers[i:], fromMention, "issue comments") + if err != nil { + return err + } + SendAsyncs(msgs) receivers = receivers[:i] } } diff --git a/services/mailer/mail_release.go b/services/mailer/mail_release.go index 22efe2f0464b3..1e12fe13acdec 100644 --- a/services/mailer/mail_release.go +++ b/services/mailer/mail_release.go @@ -10,6 +10,7 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/markup/markdown" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/translation" @@ -48,7 +49,15 @@ func MailNewRelease(rel *models.Release) { func mailNewRelease(lang string, tos []string, rel *models.Release) { locale := translation.NewLocale(lang) - rel.RenderedNote = markdown.RenderString(rel.Note, rel.Repo.Link(), rel.Repo.ComposeMetas()) + var err error + rel.RenderedNote, err = markdown.RenderString(&markup.RenderContext{ + URLPrefix: rel.Repo.Link(), + Metas: rel.Repo.ComposeMetas(), + }, rel.Note) + if err != nil { + log.Error("markdown.RenderString(%d): %v", rel.RepoID, err) + return + } subject := locale.Tr("mail.release.new.subject", rel.TagName, rel.Repo.FullName()) mailMeta := map[string]interface{}{ diff --git a/services/mailer/mail_test.go b/services/mailer/mail_test.go index 9eef084408dff..a22e7401a23e7 100644 --- a/services/mailer/mail_test.go +++ b/services/mailer/mail_test.go @@ -58,8 +58,9 @@ func TestComposeIssueCommentMessage(t *testing.T) { InitMailRender(stpl, btpl) tos := []string{"test@gitea.com", "test2@gitea.com"} - msgs := composeIssueCommentMessages(&mailCommentContext{Issue: issue, Doer: doer, ActionType: models.ActionCommentIssue, + msgs, err := composeIssueCommentMessages(&mailCommentContext{Issue: issue, Doer: doer, ActionType: models.ActionCommentIssue, Content: "test body", Comment: comment}, "en-US", tos, false, "issue comment") + assert.NoError(t, err) assert.Len(t, msgs, 2) gomailMsg := msgs[0].ToMessage() mailto := gomailMsg.GetHeader("To") @@ -92,8 +93,9 @@ func TestComposeIssueMessage(t *testing.T) { InitMailRender(stpl, btpl) tos := []string{"test@gitea.com", "test2@gitea.com"} - msgs := composeIssueCommentMessages(&mailCommentContext{Issue: issue, Doer: doer, ActionType: models.ActionCreateIssue, + msgs, err := composeIssueCommentMessages(&mailCommentContext{Issue: issue, Doer: doer, ActionType: models.ActionCreateIssue, Content: "test body"}, "en-US", tos, false, "issue create") + assert.Error(t, err) assert.Len(t, msgs, 2) gomailMsg := msgs[0].ToMessage() @@ -218,7 +220,8 @@ func TestTemplateServices(t *testing.T) { } func testComposeIssueCommentMessage(t *testing.T, ctx *mailCommentContext, tos []string, fromMention bool, info string) *Message { - msgs := composeIssueCommentMessages(ctx, "en-US", tos, fromMention, info) + msgs, err := composeIssueCommentMessages(ctx, "en-US", tos, fromMention, info) + assert.NoError(t, err) assert.Len(t, msgs, 1) return msgs[0] } From c8f03a6e1f23d5688ad13a019a30190864af5827 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sat, 27 Mar 2021 11:16:48 +0800 Subject: [PATCH 02/26] Some performance optimization --- modules/charset/charset.go | 35 +++++++++++++++++++++++++++++++++++ routers/repo/view.go | 12 +++++------- 2 files changed, 40 insertions(+), 7 deletions(-) diff --git a/modules/charset/charset.go b/modules/charset/charset.go index a7e427db99eb6..e29fa4eabf7f5 100644 --- a/modules/charset/charset.go +++ b/modules/charset/charset.go @@ -7,6 +7,7 @@ package charset import ( "bytes" "fmt" + "io" "strings" "unicode/utf8" @@ -47,6 +48,40 @@ func ToUTF8WithErr(content []byte) (string, error) { return string(result), err } +// ToUTF8WithFallback detects the encoding of content and coverts to UTF-8 reader if possible +func ToUTF8WithFallbackReader(rd io.Reader) io.Reader { + var buf = make([]byte, 2048) + _, err := rd.Read(buf) + if err != nil { + return rd + } + + charsetLabel, err := DetectEncoding(buf) + if err != nil || charsetLabel == "UTF-8" { + return io.MultiReader(bytes.NewReader(RemoveBOMIfPresent(buf)), rd) + } + + encoding, _ := charset.Lookup(charsetLabel) + if encoding == nil { + return io.MultiReader(bytes.NewReader(buf), rd) + } + + // If there is an error, we concatenate the nicely decoded part and the + // original left over. This way we won't lose data. + var w = bytes.NewBuffer(buf) + _, err = io.Copy(w, rd) + if err != nil { + return io.MultiReader(bytes.NewReader(buf), rd) + } + content := w.Bytes() + result, n, err := transform.Bytes(encoding.NewDecoder(), content) + if err != nil { + return bytes.NewReader(append(result, content[n:]...)) + } + + return bytes.NewReader(RemoveBOMIfPresent(result)) +} + // ToUTF8WithFallback detects the encoding of content and coverts to UTF-8 if possible func ToUTF8WithFallback(content []byte) []byte { charsetLabel, err := DetectEncoding(content) diff --git a/routers/repo/view.go b/routers/repo/view.go index d8e9c89a5931e..79850312ca785 100644 --- a/routers/repo/view.go +++ b/routers/repo/view.go @@ -491,8 +491,7 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st break } - d, _ := ioutil.ReadAll(dataRc) - buf = charset.ToUTF8WithFallback(append(buf, d...)) + rd := charset.ToUTF8WithFallbackReader(io.MultiReader(bytes.NewReader(buf), dataRc)) readmeExist := markup.IsReadmeFile(blob.Name()) ctx.Data["ReadmeExist"] = readmeExist if markupType := markup.Type(blob.Name()); markupType != "" { @@ -503,7 +502,7 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st Filename: blob.Name(), URLPrefix: path.Dir(treeLink), Metas: ctx.Repo.Repository.ComposeDocumentMetas(), - }, bytes.NewReader(buf), &result) + }, rd, &result) if err != nil { ctx.ServerError("Render", err) return @@ -515,7 +514,7 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st gotemplate.HTMLEscapeString(string(buf)), "\n", `
`, ) } else { - buf = charset.ToUTF8WithFallback(buf) + buf, _ := ioutil.ReadAll(rd) lineNums := linesBytesCount(buf) ctx.Data["NumLines"] = strconv.Itoa(lineNums) ctx.Data["NumLinesSet"] = true @@ -552,8 +551,7 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st } if markupType := markup.Type(blob.Name()); markupType != "" { - d, _ := ioutil.ReadAll(dataRc) - buf = append(buf, d...) + rd := io.MultiReader(bytes.NewReader(buf), dataRc) ctx.Data["IsMarkup"] = true ctx.Data["MarkupType"] = markupType var result strings.Builder @@ -561,7 +559,7 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st Filename: blob.Name(), URLPrefix: path.Dir(treeLink), Metas: ctx.Repo.Repository.ComposeDocumentMetas(), - }, bytes.NewReader(buf), &result) + }, rd, &result) if err != nil { ctx.ServerError("Render", err) return From 4d1e3b8b6bd632c8182927b59a6fc80fcb2fe638 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sat, 27 Mar 2021 12:49:35 +0800 Subject: [PATCH 03/26] Fix comment --- modules/charset/charset.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/charset/charset.go b/modules/charset/charset.go index e29fa4eabf7f5..e79839c98bdcd 100644 --- a/modules/charset/charset.go +++ b/modules/charset/charset.go @@ -48,7 +48,7 @@ func ToUTF8WithErr(content []byte) (string, error) { return string(result), err } -// ToUTF8WithFallback detects the encoding of content and coverts to UTF-8 reader if possible +// ToUTF8WithFallbackReader detects the encoding of content and coverts to UTF-8 reader if possible func ToUTF8WithFallbackReader(rd io.Reader) io.Reader { var buf = make([]byte, 2048) _, err := rd.Read(buf) From 379182e43366149ef9cce3fc8d661f107ba225b4 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sat, 27 Mar 2021 13:06:29 +0800 Subject: [PATCH 04/26] Transform reader --- modules/charset/charset.go | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/modules/charset/charset.go b/modules/charset/charset.go index e79839c98bdcd..02bf68f91b424 100644 --- a/modules/charset/charset.go +++ b/modules/charset/charset.go @@ -66,20 +66,13 @@ func ToUTF8WithFallbackReader(rd io.Reader) io.Reader { return io.MultiReader(bytes.NewReader(buf), rd) } - // If there is an error, we concatenate the nicely decoded part and the - // original left over. This way we won't lose data. - var w = bytes.NewBuffer(buf) - _, err = io.Copy(w, rd) - if err != nil { - return io.MultiReader(bytes.NewReader(buf), rd) - } - content := w.Bytes() - result, n, err := transform.Bytes(encoding.NewDecoder(), content) - if err != nil { - return bytes.NewReader(append(result, content[n:]...)) - } - - return bytes.NewReader(RemoveBOMIfPresent(result)) + return transform.NewReader( + io.MultiReader( + bytes.NewReader(RemoveBOMIfPresent(buf)), + rd, + ), + encoding.NewDecoder(), + ) } // ToUTF8WithFallback detects the encoding of content and coverts to UTF-8 if possible From e7cd922b7940cf681fd44b4ab5e1c4fbb48ec364 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sat, 27 Mar 2021 13:38:33 +0800 Subject: [PATCH 05/26] Fix csv test --- modules/markup/csv/csv.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/modules/markup/csv/csv.go b/modules/markup/csv/csv.go index 603504570c761..e1b76e2d029b6 100644 --- a/modules/markup/csv/csv.go +++ b/modules/markup/csv/csv.go @@ -119,6 +119,8 @@ func (Renderer) Render(ctx *markup.RenderContext, input io.Reader, output io.Wri row++ } - _, err = tmpBlock.WriteString("") - return err + if _, err = tmpBlock.WriteString(""); err != nil { + return err + } + return tmpBlock.Flush() } From 914bf1c04ba84c47f00234d43989e0dbbec64f4e Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sat, 27 Mar 2021 21:37:32 +0800 Subject: [PATCH 06/26] Fix test --- modules/markup/markdown/markdown_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/markup/markdown/markdown_test.go b/modules/markup/markdown/markdown_test.go index 8dbc14dc5ad7f..df5cbb073cbdc 100644 --- a/modules/markup/markdown/markdown_test.go +++ b/modules/markup/markdown/markdown_test.go @@ -297,6 +297,7 @@ func TestTotal_RenderWiki(t *testing.T) { for i := 0; i < len(testCases); i += 2 { line, err := RenderString(&markup.RenderContext{ URLPrefix: AppSubURL, + IsWiki: true, }, testCases[i]) assert.NoError(t, err) assert.Equal(t, testCases[i+1], line) From 8483a5d9688e549c446b420a3b65cc13b02203a8 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 28 Mar 2021 12:15:16 +0800 Subject: [PATCH 07/26] Fix tests --- modules/markup/html_internal_test.go | 2 +- modules/markup/html_test.go | 1 + modules/markup/markdown/markdown.go | 23 +++++++++++++++++++++-- modules/markup/markdown/markdown_test.go | 16 ++++++++-------- routers/api/v1/misc/markdown.go | 6 ++++-- 5 files changed, 35 insertions(+), 13 deletions(-) diff --git a/modules/markup/html_internal_test.go b/modules/markup/html_internal_test.go index dc76f5b435ad2..330750a47a7b0 100644 --- a/modules/markup/html_internal_test.go +++ b/modules/markup/html_internal_test.go @@ -61,7 +61,7 @@ var localMetas = map[string]string{ func TestRender_IssueIndexPattern(t *testing.T) { // numeric: render inputs without valid mentions test := func(s string) { - testRenderIssueIndexPattern(t, s, s, nil) + testRenderIssueIndexPattern(t, s, s, &RenderContext{}) testRenderIssueIndexPattern(t, s, s, &RenderContext{Metas: numericMetas}) } diff --git a/modules/markup/html_test.go b/modules/markup/html_test.go index 02ca0dfe629b4..b23257d65e920 100644 --- a/modules/markup/html_test.go +++ b/modules/markup/html_test.go @@ -322,6 +322,7 @@ func TestRender_ShortLinks(t *testing.T) { buffer, err = markdown.RenderString(&markup.RenderContext{ URLPrefix: setting.AppSubURL, Metas: localMetas, + IsWiki: true, }, input) assert.NoError(t, err) assert.Equal(t, strings.TrimSpace(expectedWiki), strings.TrimSpace(buffer)) diff --git a/modules/markup/markdown/markdown.go b/modules/markup/markdown/markdown.go index a79934a3d3704..4e5270d7dea1c 100644 --- a/modules/markup/markdown/markdown.go +++ b/modules/markup/markdown/markdown.go @@ -8,6 +8,7 @@ package markdown import ( "fmt" "io" + "io/ioutil" "strings" "sync" @@ -15,7 +16,6 @@ import ( "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/markup/common" "code.gitea.io/gitea/modules/setting" - giteautil "code.gitea.io/gitea/modules/util" chromahtml "github.com/alecthomas/chroma/formatters/html" "github.com/yuin/goldmark" @@ -185,7 +185,12 @@ func actualRender(ctx *markup.RenderContext, input io.Reader, output io.Writer) }() pc := NewGiteaParseContext(ctx) - if err := converter.Convert(giteautil.NormalizeEOLReader(input), lw, parser.WithContext(pc)); err != nil { + buf, err := ioutil.ReadAll(input) + if err != nil { + log.Error("Unable to ReadAll: %v", err) + return + } + if err := converter.Convert(buf, lw, parser.WithContext(pc)); err != nil { log.Error("Unable to render: %v", err) _ = lw.CloseWithError(err) return @@ -264,6 +269,20 @@ func RenderString(ctx *markup.RenderContext, content string) (string, error) { return buf.String(), nil } +// RenderRaw renders Markdown to HTML without handling special links. +func RenderRaw(ctx *markup.RenderContext, input io.Reader, output io.Writer) error { + return render(ctx, input, output) +} + +// RenderRawString renders Markdown to HTML without handling special links and return string +func RenderRawString(ctx *markup.RenderContext, content string) (string, error) { + var buf strings.Builder + if err := RenderRaw(ctx, strings.NewReader(content), &buf); err != nil { + return "", err + } + return buf.String(), nil +} + // IsMarkdownFile reports whether name looks like a Markdown file // based on its extension. func IsMarkdownFile(name string) bool { diff --git a/modules/markup/markdown/markdown_test.go b/modules/markup/markdown/markdown_test.go index df5cbb073cbdc..5997dbccdcf95 100644 --- a/modules/markup/markdown/markdown_test.go +++ b/modules/markup/markdown/markdown_test.go @@ -329,19 +329,19 @@ func TestTotal_RenderString(t *testing.T) { func TestRender_RenderParagraphs(t *testing.T) { test := func(t *testing.T, str string, cnt int) { - res, err := RenderString(&markup.RenderContext{}, str) + res, err := RenderRawString(&markup.RenderContext{}, str) assert.NoError(t, err) - assert.Equal(t, strings.Count(res, "image1
image2

` - res, err := RenderString(&markup.RenderContext{}, testcase) + res, err := RenderRawString(&markup.RenderContext{}, testcase) assert.NoError(t, err) assert.Equal(t, expected, res) diff --git a/routers/api/v1/misc/markdown.go b/routers/api/v1/misc/markdown.go index 87e65003f0062..f1007b7ee2db2 100644 --- a/routers/api/v1/misc/markdown.go +++ b/routers/api/v1/misc/markdown.go @@ -86,7 +86,9 @@ func Markdown(ctx *context.APIContext) { return } default: - if err := markdown.Render(&markup.RenderContext{}, strings.NewReader(form.Text), ctx.Resp); err != nil { + if err := markdown.RenderRaw(&markup.RenderContext{ + URLPrefix: form.Context, + }, strings.NewReader(form.Text), ctx.Resp); err != nil { ctx.InternalServerError(err) return } @@ -115,7 +117,7 @@ func MarkdownRaw(ctx *context.APIContext) { // "422": // "$ref": "#/responses/validationError" defer ctx.Req.Body.Close() - if err := markdown.Render(&markup.RenderContext{}, ctx.Req.Body, ctx.Resp); err != nil { + if err := markdown.RenderRaw(&markup.RenderContext{}, ctx.Req.Body, ctx.Resp); err != nil { ctx.InternalServerError(err) return } From f144bb940551cad36a1c331cab46df4190eb5226 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 28 Mar 2021 17:35:59 +0800 Subject: [PATCH 08/26] Improve optimaziation --- modules/markup/html.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/modules/markup/html.go b/modules/markup/html.go index 44b461854c7b5..593fdcd04bee8 100644 --- a/modules/markup/html.go +++ b/modules/markup/html.go @@ -342,15 +342,12 @@ func postProcess(ctx *RenderContext, procs []processor, input io.Reader, output res.Reset() // Render everything to buf. for _, node := range nodes { - err = html.Render(res, node) + err = html.Render(output, node) if err != nil { return &postProcessError{"error rendering processed HTML", err} } } - - // Everything done successfully, return parsed data. - _, err = io.Copy(output, res) - return err + return nil } func visitNode(ctx *RenderContext, procs []processor, node *html.Node, visitText bool) { From 31ab960f12badbc81e5b746ee4282ca73fbed69e Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 28 Mar 2021 19:58:00 +0800 Subject: [PATCH 09/26] Fix test --- modules/markup/markdown/markdown.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/markup/markdown/markdown.go b/modules/markup/markdown/markdown.go index 4e5270d7dea1c..7f412f71c53eb 100644 --- a/modules/markup/markdown/markdown.go +++ b/modules/markup/markdown/markdown.go @@ -16,6 +16,7 @@ import ( "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/markup/common" "code.gitea.io/gitea/modules/setting" + giteautil "code.gitea.io/gitea/modules/util" chromahtml "github.com/alecthomas/chroma/formatters/html" "github.com/yuin/goldmark" @@ -190,7 +191,7 @@ func actualRender(ctx *markup.RenderContext, input io.Reader, output io.Writer) log.Error("Unable to ReadAll: %v", err) return } - if err := converter.Convert(buf, lw, parser.WithContext(pc)); err != nil { + if err := converter.Convert(giteautil.NormalizeEOL(buf), lw, parser.WithContext(pc)); err != nil { log.Error("Unable to render: %v", err) _ = lw.CloseWithError(err) return From 400a9f9452aa9e548497029487abd63643bb9754 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 28 Mar 2021 23:26:42 +0800 Subject: [PATCH 10/26] Fix test --- modules/markup/markdown/markdown.go | 4 +- modules/util/util.go | 88 ++++++++++++++++------------- 2 files changed, 50 insertions(+), 42 deletions(-) diff --git a/modules/markup/markdown/markdown.go b/modules/markup/markdown/markdown.go index 7f412f71c53eb..41356b18ff4e4 100644 --- a/modules/markup/markdown/markdown.go +++ b/modules/markup/markdown/markdown.go @@ -186,12 +186,12 @@ func actualRender(ctx *markup.RenderContext, input io.Reader, output io.Writer) }() pc := NewGiteaParseContext(ctx) - buf, err := ioutil.ReadAll(input) + buf, err := ioutil.ReadAll(giteautil.NormalizeEOLReader(input)) if err != nil { log.Error("Unable to ReadAll: %v", err) return } - if err := converter.Convert(giteautil.NormalizeEOL(buf), lw, parser.WithContext(pc)); err != nil { + if err := converter.Convert(buf, lw, parser.WithContext(pc)); err != nil { log.Error("Unable to render: %v", err) _ = lw.CloseWithError(err) return diff --git a/modules/util/util.go b/modules/util/util.go index 0072d0e7be600..8db3c4646d80a 100644 --- a/modules/util/util.go +++ b/modules/util/util.go @@ -5,10 +5,10 @@ package util import ( - "bufio" "bytes" "errors" "io" + "io/ioutil" "strings" ) @@ -68,51 +68,59 @@ func IsEmptyString(s string) bool { return len(strings.TrimSpace(s)) == 0 } -// NormalizeEOLReader will convert Windows (CRLF) and Mac (CR) EOLs to UNIX (LF) from a reader -func NormalizeEOLReader(rd io.Reader) []byte { - scanner := bufio.NewScanner(rd) - var buf bytes.Buffer - for scanner.Scan() { - buf.Write(scanner.Bytes()) - buf.WriteByte('\n') - } - return buf.Bytes() +type normalizeEOLReader struct { + rd io.Reader + isLastReturn bool } -// NormalizeEOL will convert Windows (CRLF) and Mac (CR) EOLs to UNIX (LF) -func NormalizeEOL(input []byte) []byte { - var right, left, pos int - if right = bytes.IndexByte(input, '\r'); right == -1 { - return input +func (r *normalizeEOLReader) Read(bs []byte) (int, error) { + var p = make([]byte, len(bs)) + n, err := r.rd.Read(p) + if err != nil { + return n, err } - length := len(input) - tmp := make([]byte, length) - - // We know that left < length because otherwise right would be -1 from IndexByte. - copy(tmp[pos:pos+right], input[left:left+right]) - pos += right - tmp[pos] = '\n' - left += right + 1 - pos++ - - for left < length { - if input[left] == '\n' { - left++ - } - right = bytes.IndexByte(input[left:], '\r') - if right == -1 { - copy(tmp[pos:], input[left:]) - pos += length - left - break + var j = 0 + for i, c := range p[:n] { + if i == 0 { + if c == '\n' && r.isLastReturn { + r.isLastReturn = false + continue + } + r.isLastReturn = false + } + if c == '\r' { + if i < n-1 { + if p[i+1] != '\n' { + bs[j] = '\n' + } else { + continue + } + } else { + r.isLastReturn = true + bs[j] = '\n' + } + } else { + bs[j] = c } - copy(tmp[pos:pos+right], input[left:left+right]) - pos += right - tmp[pos] = '\n' - left += right + 1 - pos++ + j++ } - return tmp[:pos] + + return j, nil +} + +// NormalizeEOLReader will convert Windows (CRLF) and Mac (CR) EOLs to UNIX (LF) from a reader +func NormalizeEOLReader(rd io.Reader) io.Reader { + return &normalizeEOLReader{ + rd: rd, + isLastReturn: false, + } +} + +// NormalizeEOL will convert Windows (CRLF) and Mac (CR) EOLs to UNIX (LF) +func NormalizeEOL(input []byte) []byte { + bs, _ := ioutil.ReadAll(NormalizeEOLReader(bytes.NewReader(input))) + return bs } // MergeInto merges pairs of values into a "dict" From aabc327c5190f2fae5947cc0f53b04c19d0beb25 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Mon, 29 Mar 2021 11:41:45 +0800 Subject: [PATCH 11/26] Detect file encoding with reader --- modules/charset/charset.go | 31 ++++++++----------------------- 1 file changed, 8 insertions(+), 23 deletions(-) diff --git a/modules/charset/charset.go b/modules/charset/charset.go index 02bf68f91b424..219e7e6584d9b 100644 --- a/modules/charset/charset.go +++ b/modules/charset/charset.go @@ -8,6 +8,7 @@ import ( "bytes" "fmt" "io" + "io/ioutil" "strings" "unicode/utf8" @@ -51,24 +52,24 @@ func ToUTF8WithErr(content []byte) (string, error) { // ToUTF8WithFallbackReader detects the encoding of content and coverts to UTF-8 reader if possible func ToUTF8WithFallbackReader(rd io.Reader) io.Reader { var buf = make([]byte, 2048) - _, err := rd.Read(buf) + n, err := rd.Read(buf) if err != nil { return rd } - charsetLabel, err := DetectEncoding(buf) + charsetLabel, err := DetectEncoding(buf[:n]) if err != nil || charsetLabel == "UTF-8" { - return io.MultiReader(bytes.NewReader(RemoveBOMIfPresent(buf)), rd) + return io.MultiReader(bytes.NewReader(RemoveBOMIfPresent(buf[:n])), rd) } encoding, _ := charset.Lookup(charsetLabel) if encoding == nil { - return io.MultiReader(bytes.NewReader(buf), rd) + return io.MultiReader(bytes.NewReader(buf[:n]), rd) } return transform.NewReader( io.MultiReader( - bytes.NewReader(RemoveBOMIfPresent(buf)), + bytes.NewReader(RemoveBOMIfPresent(buf[:n])), rd, ), encoding.NewDecoder(), @@ -77,24 +78,8 @@ func ToUTF8WithFallbackReader(rd io.Reader) io.Reader { // ToUTF8WithFallback detects the encoding of content and coverts to UTF-8 if possible func ToUTF8WithFallback(content []byte) []byte { - charsetLabel, err := DetectEncoding(content) - if err != nil || charsetLabel == "UTF-8" { - return RemoveBOMIfPresent(content) - } - - encoding, _ := charset.Lookup(charsetLabel) - if encoding == nil { - return content - } - - // If there is an error, we concatenate the nicely decoded part and the - // original left over. This way we won't lose data. - result, n, err := transform.Bytes(encoding.NewDecoder(), content) - if err != nil { - return append(result, content[n:]...) - } - - return RemoveBOMIfPresent(result) + bs, _ := ioutil.ReadAll(ToUTF8WithFallbackReader(bytes.NewReader(content))) + return bs } // ToUTF8 converts content to UTF8 encoding and ignore error From 26576f7502543c5b36fde0ff3eb762b131cab154 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Mon, 29 Mar 2021 23:52:52 +0800 Subject: [PATCH 12/26] Improve optimaziation --- modules/charset/charset.go | 52 +++++++++++++++++++------------------- routers/repo/lfs.go | 13 +++------- routers/repo/view.go | 13 +++++----- 3 files changed, 36 insertions(+), 42 deletions(-) diff --git a/modules/charset/charset.go b/modules/charset/charset.go index 219e7e6584d9b..3000864c2ea0e 100644 --- a/modules/charset/charset.go +++ b/modules/charset/charset.go @@ -23,32 +23,6 @@ import ( // UTF8BOM is the utf-8 byte-order marker var UTF8BOM = []byte{'\xef', '\xbb', '\xbf'} -// ToUTF8WithErr converts content to UTF8 encoding -func ToUTF8WithErr(content []byte) (string, error) { - charsetLabel, err := DetectEncoding(content) - if err != nil { - return "", err - } else if charsetLabel == "UTF-8" { - return string(RemoveBOMIfPresent(content)), nil - } - - encoding, _ := charset.Lookup(charsetLabel) - if encoding == nil { - return string(content), fmt.Errorf("Unknown encoding: %s", charsetLabel) - } - - // If there is an error, we concatenate the nicely decoded part and the - // original left over. This way we won't lose much data. - result, n, err := transform.Bytes(encoding.NewDecoder(), content) - if err != nil { - result = append(result, content[n:]...) - } - - result = RemoveBOMIfPresent(result) - - return string(result), err -} - // ToUTF8WithFallbackReader detects the encoding of content and coverts to UTF-8 reader if possible func ToUTF8WithFallbackReader(rd io.Reader) io.Reader { var buf = make([]byte, 2048) @@ -76,6 +50,32 @@ func ToUTF8WithFallbackReader(rd io.Reader) io.Reader { ) } +// ToUTF8WithErr converts content to UTF8 encoding +func ToUTF8WithErr(content []byte) (string, error) { + charsetLabel, err := DetectEncoding(content) + if err != nil { + return "", err + } else if charsetLabel == "UTF-8" { + return string(RemoveBOMIfPresent(content)), nil + } + + encoding, _ := charset.Lookup(charsetLabel) + if encoding == nil { + return string(content), fmt.Errorf("Unknown encoding: %s", charsetLabel) + } + + // If there is an error, we concatenate the nicely decoded part and the + // original left over. This way we won't lose much data. + result, n, err := transform.Bytes(encoding.NewDecoder(), content) + if err != nil { + result = append(result, content[n:]...) + } + + result = RemoveBOMIfPresent(result) + + return string(result), err +} + // ToUTF8WithFallback detects the encoding of content and coverts to UTF-8 if possible func ToUTF8WithFallback(content []byte) []byte { bs, _ := ioutil.ReadAll(ToUTF8WithFallbackReader(bytes.NewReader(content))) diff --git a/routers/repo/lfs.go b/routers/repo/lfs.go index 457ffb6aba5f8..3a7ce2e23bd51 100644 --- a/routers/repo/lfs.go +++ b/routers/repo/lfs.go @@ -296,20 +296,13 @@ func LFSFileGet(ctx *context.Context) { break } - d, _ := ioutil.ReadAll(dataRc) - buf = charset.ToUTF8WithFallback(append(buf, d...)) + buf := charset.ToUTF8WithFallbackReader(io.MultiReader(bytes.NewReader(buf), dataRc)) // Building code view blocks with line number on server side. - var fileContent string - if content, err := charset.ToUTF8WithErr(buf); err != nil { - log.Error("ToUTF8WithErr: %v", err) - fileContent = string(buf) - } else { - fileContent = content - } + fileContent, _ := ioutil.ReadAll(buf) var output bytes.Buffer - lines := strings.Split(fileContent, "\n") + lines := strings.Split(string(fileContent), "\n") //Remove blank line at the end of file if len(lines) > 0 && lines[len(lines)-1] == "" { lines = lines[:len(lines)-1] diff --git a/routers/repo/view.go b/routers/repo/view.go index 79850312ca785..ae27caf0044af 100644 --- a/routers/repo/view.go +++ b/routers/repo/view.go @@ -324,8 +324,7 @@ func renderDirectory(ctx *context.Context, treeLink string) { ctx.Data["IsTextFile"] = true ctx.Data["FileSize"] = fileSize } else { - d, _ := ioutil.ReadAll(dataRc) - buf = charset.ToUTF8WithFallback(append(buf, d...)) + rd := charset.ToUTF8WithFallbackReader(io.MultiReader(bytes.NewReader(buf), dataRc)) if markupType := markup.Type(readmeFile.name); markupType != "" { ctx.Data["IsMarkup"] = true @@ -335,12 +334,14 @@ func renderDirectory(ctx *context.Context, treeLink string) { Filename: readmeFile.name, URLPrefix: readmeTreelink, Metas: ctx.Repo.Repository.ComposeDocumentMetas(), - }, bytes.NewReader(buf), &result) + }, rd, &result) if err != nil { - ctx.ServerError("Render", err) - return + log.Error("Render failed: %v then fallback", err) + bs, _ := ioutil.ReadAll(rd) + ctx.Data["FileContent"] = string(bs) + } else { + ctx.Data["FileContent"] = result.String() } - ctx.Data["FileContent"] = result.String() } else { ctx.Data["IsRenderedHTML"] = true ctx.Data["FileContent"] = strings.ReplaceAll( From 0c6f41989dbd08599b15e6b374ce347b43c03c56 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Tue, 30 Mar 2021 19:26:00 +0800 Subject: [PATCH 13/26] reduce memory usage --- modules/markup/html.go | 8 +--- modules/markup/markdown/markdown.go | 1 + modules/markup/renderer.go | 74 +++++++++++++++++++++++------ 3 files changed, 61 insertions(+), 22 deletions(-) diff --git a/modules/markup/html.go b/modules/markup/html.go index 593fdcd04bee8..7c4c10ee22105 100644 --- a/modules/markup/html.go +++ b/modules/markup/html.go @@ -334,14 +334,8 @@ func postProcess(ctx *RenderContext, procs []processor, input io.Reader, output } } - nodes = newNodes - - // Create buffer in which the data will be placed again. We know that the - // length will be at least that of res; to spare a few alloc+copy, we - // reuse res, resetting its length to 0. - res.Reset() // Render everything to buf. - for _, node := range nodes { + for _, node := range newNodes { err = html.Render(output, node) if err != nil { return &postProcessError{"error rendering processed HTML", err} diff --git a/modules/markup/markdown/markdown.go b/modules/markup/markdown/markdown.go index 41356b18ff4e4..1fa3a0a7a3610 100644 --- a/modules/markup/markdown/markdown.go +++ b/modules/markup/markdown/markdown.go @@ -185,6 +185,7 @@ func actualRender(ctx *markup.RenderContext, input io.Reader, output io.Writer) _ = lw.CloseWithError(fmt.Errorf("%v", err)) }() + // FIXME: Don't read all to memory, but goldmark doesn't support pc := NewGiteaParseContext(ctx) buf, err := ioutil.ReadAll(giteautil.NormalizeEOLReader(input)) if err != nil { diff --git a/modules/markup/renderer.go b/modules/markup/renderer.go index bbbca310c0c55..7cc81574ba0e1 100644 --- a/modules/markup/renderer.go +++ b/modules/markup/renderer.go @@ -11,6 +11,7 @@ import ( "io" "path/filepath" "strings" + "sync" "code.gitea.io/gitea/modules/setting" ) @@ -94,33 +95,76 @@ func RenderString(ctx *RenderContext, content string) (string, error) { } func render(ctx *RenderContext, parser Renderer, input io.Reader, output io.Writer) error { - var buf1 strings.Builder - if err := parser.Render(ctx, input, &buf1); err != nil { - return err + var wg sync.WaitGroup + var err error + pr, pw := io.Pipe() + defer func() { + _ = pr.Close() + _ = pw.Close() + }() + + pr2, pw2 := io.Pipe() + defer func() { + _ = pr2.Close() + _ = pw2.Close() + }() + + wg.Add(1) + go func() { + buf := SanitizeReader(pr2) + _, err = io.Copy(output, buf) + _ = pr2.Close() + wg.Done() + }() + + wg.Add(1) + go func() { + err = PostProcess(ctx, pr, pw2) + _ = pr.Close() + _ = pw2.Close() + wg.Done() + }() + + if err1 := parser.Render(ctx, input, pw); err1 != nil { + return err1 } + _ = pw.Close() - var buf2 strings.Builder - if err := PostProcess(ctx, strings.NewReader(buf1.String()), &buf2); err != nil { - return fmt.Errorf("PostProcess: %v", err) - } - buf := SanitizeReader(strings.NewReader(buf2.String())) - _, err := io.Copy(output, buf) + wg.Wait() return err } +// ErrUnsupportedRenderType represents +type ErrUnsupportedRenderType struct { + Type string +} + +func (err ErrUnsupportedRenderType) Error() string { + return fmt.Sprintf("Unsupported render type: %s", err.Type) +} + func renderByType(ctx *RenderContext, input io.Reader, output io.Writer) error { - if parser, ok := renderers[ctx.Type]; ok { - return render(ctx, parser, input, output) + if renderer, ok := renderers[ctx.Type]; ok { + return render(ctx, renderer, input, output) } - return nil + return ErrUnsupportedRenderType{ctx.Type} +} + +// ErrUnsupportedRenderExtension represents the error when extension doesn't supported to render +type ErrUnsupportedRenderExtension struct { + Extension string +} + +func (err ErrUnsupportedRenderExtension) Error() string { + return fmt.Sprintf("Unsupported render extension: %s", err.Extension) } func renderFile(ctx *RenderContext, input io.Reader, output io.Writer) error { extension := strings.ToLower(filepath.Ext(ctx.Filename)) - if parser, ok := extRenderers[extension]; ok { - return render(ctx, parser, input, output) + if renderer, ok := extRenderers[extension]; ok { + return render(ctx, renderer, input, output) } - return nil + return ErrUnsupportedRenderExtension{extension} } // Type returns if markup format via the filename From ba32c297d6f3b52cb07207e428c35557c5c874cc Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Tue, 30 Mar 2021 21:24:27 +0800 Subject: [PATCH 14/26] improve code --- modules/csv/csv.go | 2 +- modules/csv/csv_test.go | 7 +++++-- routers/repo/compare.go | 11 +---------- services/gitdiff/csv_test.go | 10 ++++++++-- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/modules/csv/csv.go b/modules/csv/csv.go index 826cbbaba6a65..bf433f77d29ff 100644 --- a/modules/csv/csv.go +++ b/modules/csv/csv.go @@ -39,7 +39,7 @@ func CreateReaderAndGuessDelimiter(rd io.Reader) (*stdcsv.Reader, error) { var newInput io.Reader if size < 1e4 { - newInput = bytes.NewReader(data) + newInput = bytes.NewReader(data[:size]) } else { newInput = io.MultiReader(bytes.NewReader(data), rd) } diff --git a/modules/csv/csv_test.go b/modules/csv/csv_test.go index 3a7584e21d2e7..3cc09c40aa5b3 100644 --- a/modules/csv/csv_test.go +++ b/modules/csv/csv_test.go @@ -5,20 +5,23 @@ package csv import ( + "bytes" + "strings" "testing" "github.com/stretchr/testify/assert" ) func TestCreateReader(t *testing.T) { - rd := CreateReader([]byte{}, ',') + rd := CreateReader(bytes.NewReader([]byte{}), ',') assert.Equal(t, ',', rd.Comma) } func TestCreateReaderAndGuessDelimiter(t *testing.T) { input := "a;b;c\n1;2;3\n4;5;6" - rd := CreateReaderAndGuessDelimiter([]byte(input)) + rd, err := CreateReaderAndGuessDelimiter(strings.NewReader(input)) + assert.NoError(t, err) assert.Equal(t, ';', rd.Comma) } diff --git a/routers/repo/compare.go b/routers/repo/compare.go index f7b9d2122cfdc..a658374d9b127 100644 --- a/routers/repo/compare.go +++ b/routers/repo/compare.go @@ -6,12 +6,10 @@ package repo import ( "bufio" - "bytes" "encoding/csv" "errors" "fmt" "html" - "io/ioutil" "net/http" "path" "path/filepath" @@ -118,14 +116,7 @@ func setCsvCompareContext(ctx *context.Context) { } defer reader.Close() - b, err := ioutil.ReadAll(reader) - if err != nil { - return nil, err - } - - b = charset.ToUTF8WithFallback(b) - - return csv_module.CreateReaderAndGuessDelimiter(bytes.NewReader(b)) + return csv_module.CreateReaderAndGuessDelimiter(charset.ToUTF8WithFallbackReader(reader)) } baseReader, err := csvReaderFromCommit(baseCommit) diff --git a/services/gitdiff/csv_test.go b/services/gitdiff/csv_test.go index 17edea582c407..33fe9bb0cb85e 100644 --- a/services/gitdiff/csv_test.go +++ b/services/gitdiff/csv_test.go @@ -95,11 +95,17 @@ func TestCSVDiff(t *testing.T) { var baseReader *csv.Reader if len(c.base) > 0 { - baseReader = csv_module.CreateReaderAndGuessDelimiter([]byte(c.base)) + baseReader,err = csv_module.CreateReaderAndGuessDelimiter(strings.NewReader(c.base)) + if err != nil { + t.Errorf("CreateReaderAndGuessDelimiter failed: %s", err) + } } var headReader *csv.Reader if len(c.head) > 0 { - headReader = csv_module.CreateReaderAndGuessDelimiter([]byte(c.head)) + headReader, err = csv_module.CreateReaderAndGuessDelimiter(strings.NewReader(c.head)) + if err != nil { + t.Errorf("CreateReaderAndGuessDelimiter failed: %s", err) + } } result, err := CreateCsvDiff(diff.Files[0], baseReader, headReader) From f3a85fa65a98aa5ed4c034bb2e156bcf3bf36477 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Tue, 30 Mar 2021 21:47:12 +0800 Subject: [PATCH 15/26] fix build --- modules/markup/csv/csv.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/modules/markup/csv/csv.go b/modules/markup/csv/csv.go index e1b76e2d029b6..702e896a1026a 100644 --- a/modules/markup/csv/csv.go +++ b/modules/markup/csv/csv.go @@ -76,6 +76,7 @@ func writeField(w io.Writer, element, class, field string) error { func (Renderer) Render(ctx *markup.RenderContext, input io.Reader, output io.Writer) error { var tmpBlock = bufio.NewWriter(output) + // FIXME: don't read all to memory rawBytes, err := io.ReadAll(input) if err != nil { return err @@ -90,9 +91,6 @@ func (Renderer) Render(ctx *markup.RenderContext, input io.Reader, output io.Wri rd, err := csv.CreateReaderAndGuessDelimiter(bytes.NewReader(rawBytes)) if err != nil { - if err == io.EOF { - - } return err } From a0d7dbc92b9a149e7058ccfd3eea942f72531be0 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Wed, 31 Mar 2021 23:59:10 +0800 Subject: [PATCH 16/26] Fix test --- modules/markup/csv/csv.go | 32 +++++++++++++++++++++++--------- modules/markup/html_test.go | 15 +++++++-------- services/gitdiff/csv_test.go | 2 +- 3 files changed, 31 insertions(+), 18 deletions(-) diff --git a/modules/markup/csv/csv.go b/modules/markup/csv/csv.go index 702e896a1026a..04a57758fba3c 100644 --- a/modules/markup/csv/csv.go +++ b/modules/markup/csv/csv.go @@ -83,10 +83,14 @@ func (Renderer) Render(ctx *markup.RenderContext, input io.Reader, output io.Wri } if setting.UI.CSV.MaxFileSize != 0 && setting.UI.CSV.MaxFileSize < int64(len(rawBytes)) { - tmpBlock.WriteString("
")
-		tmpBlock.WriteString(html.EscapeString(string(rawBytes)))
-		tmpBlock.WriteString("
") - return nil + if _, err := tmpBlock.WriteString("
"); err != nil {
+			return err
+		}
+		if _, err := tmpBlock.WriteString(html.EscapeString(string(rawBytes))); err != nil {
+			return err
+		}
+		_, err = tmpBlock.WriteString("
") + return err } rd, err := csv.CreateReaderAndGuessDelimiter(bytes.NewReader(rawBytes)) @@ -94,7 +98,9 @@ func (Renderer) Render(ctx *markup.RenderContext, input io.Reader, output io.Wri return err } - tmpBlock.WriteString(``) + if _, err := tmpBlock.WriteString(`
`); err != nil { + return err + } row := 1 for { fields, err := rd.Read() @@ -104,16 +110,24 @@ func (Renderer) Render(ctx *markup.RenderContext, input io.Reader, output io.Wri if err != nil { continue } - tmpBlock.WriteString("") + if _, err := tmpBlock.WriteString(""); err != nil { + return err + } element := "td" if row == 1 { element = "th" } - writeField(tmpBlock, element, "line-num", strconv.Itoa(row)) + if err := writeField(tmpBlock, element, "line-num", strconv.Itoa(row)); err != nil { + return err + } for _, field := range fields { - writeField(tmpBlock, element, "", field) + if err := writeField(tmpBlock, element, "", field); err != nil { + return err + } + } + if _, err := tmpBlock.WriteString(""); err != nil { + return err } - tmpBlock.WriteString("") row++ } diff --git a/modules/markup/html_test.go b/modules/markup/html_test.go index b23257d65e920..3425c3d3a8872 100644 --- a/modules/markup/html_test.go +++ b/modules/markup/html_test.go @@ -9,7 +9,6 @@ import ( "testing" "code.gitea.io/gitea/modules/emoji" - "code.gitea.io/gitea/modules/markup" . "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/markup/markdown" "code.gitea.io/gitea/modules/setting" @@ -29,7 +28,7 @@ func TestRender_Commits(t *testing.T) { setting.AppSubURL = AppSubURL test := func(input, expected string) { - buffer, err := RenderString(&markup.RenderContext{ + buffer, err := RenderString(&RenderContext{ Filename: ".md", URLPrefix: setting.AppSubURL, Metas: localMetas, @@ -65,7 +64,7 @@ func TestRender_CrossReferences(t *testing.T) { setting.AppSubURL = AppSubURL test := func(input, expected string) { - buffer, err := RenderString(&markup.RenderContext{ + buffer, err := RenderString(&RenderContext{ Filename: "a.md", URLPrefix: setting.AppSubURL, Metas: localMetas, @@ -102,7 +101,7 @@ func TestRender_links(t *testing.T) { setting.AppSubURL = AppSubURL test := func(input, expected string) { - buffer, err := RenderString(&markup.RenderContext{ + buffer, err := RenderString(&RenderContext{ Filename: "a.md", URLPrefix: setting.AppSubURL, }, input) @@ -202,7 +201,7 @@ func TestRender_email(t *testing.T) { setting.AppSubURL = AppSubURL test := func(input, expected string) { - res, err := RenderString(&markup.RenderContext{ + res, err := RenderString(&RenderContext{ Filename: "a.md", URLPrefix: setting.AppSubURL, }, input) @@ -261,7 +260,7 @@ func TestRender_emoji(t *testing.T) { test := func(input, expected string) { expected = strings.ReplaceAll(expected, "&", "&") - buffer, err := RenderString(&markup.RenderContext{ + buffer, err := RenderString(&RenderContext{ Filename: "a.md", URLPrefix: setting.AppSubURL, }, input) @@ -314,12 +313,12 @@ func TestRender_ShortLinks(t *testing.T) { tree := util.URLJoin(AppSubURL, "src", "master") test := func(input, expected, expectedWiki string) { - buffer, err := markdown.RenderString(&markup.RenderContext{ + buffer, err := markdown.RenderString(&RenderContext{ URLPrefix: tree, }, input) assert.NoError(t, err) assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer)) - buffer, err = markdown.RenderString(&markup.RenderContext{ + buffer, err = markdown.RenderString(&RenderContext{ URLPrefix: setting.AppSubURL, Metas: localMetas, IsWiki: true, diff --git a/services/gitdiff/csv_test.go b/services/gitdiff/csv_test.go index 33fe9bb0cb85e..f3dc0c2a2c46d 100644 --- a/services/gitdiff/csv_test.go +++ b/services/gitdiff/csv_test.go @@ -95,7 +95,7 @@ func TestCSVDiff(t *testing.T) { var baseReader *csv.Reader if len(c.base) > 0 { - baseReader,err = csv_module.CreateReaderAndGuessDelimiter(strings.NewReader(c.base)) + baseReader, err = csv_module.CreateReaderAndGuessDelimiter(strings.NewReader(c.base)) if err != nil { t.Errorf("CreateReaderAndGuessDelimiter failed: %s", err) } From 933c84c521cf84e0e87eb0c6bdb9259ae345a050 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Thu, 1 Apr 2021 00:33:18 +0800 Subject: [PATCH 17/26] Fix for go1.15 --- modules/markup/csv/csv.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/markup/csv/csv.go b/modules/markup/csv/csv.go index 04a57758fba3c..07da35063e7ea 100644 --- a/modules/markup/csv/csv.go +++ b/modules/markup/csv/csv.go @@ -9,6 +9,7 @@ import ( "bytes" "html" "io" + "io/ioutil" "strconv" "code.gitea.io/gitea/modules/csv" @@ -77,7 +78,7 @@ func (Renderer) Render(ctx *markup.RenderContext, input io.Reader, output io.Wri var tmpBlock = bufio.NewWriter(output) // FIXME: don't read all to memory - rawBytes, err := io.ReadAll(input) + rawBytes, err := ioutil.ReadAll(input) if err != nil { return err } From e63e1194eccb916e9ad45f9b57ae922dea189bf1 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Thu, 1 Apr 2021 10:13:53 +0800 Subject: [PATCH 18/26] Fix render --- modules/markup/csv/csv.go | 1 - modules/markup/csv/csv_test.go | 1 + routers/repo/view.go | 4 +++- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/modules/markup/csv/csv.go b/modules/markup/csv/csv.go index 07da35063e7ea..d90910b9a2744 100644 --- a/modules/markup/csv/csv.go +++ b/modules/markup/csv/csv.go @@ -19,7 +19,6 @@ import ( func init() { markup.RegisterRenderer(Renderer{}) - } // Renderer implements markup.Renderer for orgmode diff --git a/modules/markup/csv/csv_test.go b/modules/markup/csv/csv_test.go index 48e3ce572514e..613762f86cee9 100644 --- a/modules/markup/csv/csv_test.go +++ b/modules/markup/csv/csv_test.go @@ -9,6 +9,7 @@ import ( "testing" "code.gitea.io/gitea/modules/markup" + "github.com/stretchr/testify/assert" ) diff --git a/routers/repo/view.go b/routers/repo/view.go index ae27caf0044af..10deb7065a33f 100644 --- a/routers/repo/view.go +++ b/routers/repo/view.go @@ -338,7 +338,9 @@ func renderDirectory(ctx *context.Context, treeLink string) { if err != nil { log.Error("Render failed: %v then fallback", err) bs, _ := ioutil.ReadAll(rd) - ctx.Data["FileContent"] = string(bs) + ctx.Data["FileContent"] = strings.ReplaceAll( + gotemplate.HTMLEscapeString(string(bs)), "\n", `
`, + ) } else { ctx.Data["FileContent"] = result.String() } From 1eb31b0d664ccf1825d72e36147d5ef050ed6719 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Thu, 1 Apr 2021 14:13:27 +0800 Subject: [PATCH 19/26] Fix comment --- modules/markup/csv/csv.go | 2 +- modules/markup/orgmode/orgmode.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/markup/csv/csv.go b/modules/markup/csv/csv.go index d90910b9a2744..eab8f3b494f63 100644 --- a/modules/markup/csv/csv.go +++ b/modules/markup/csv/csv.go @@ -21,7 +21,7 @@ func init() { markup.RegisterRenderer(Renderer{}) } -// Renderer implements markup.Renderer for orgmode +// Renderer implements markup.Renderer for csv files type Renderer struct { } diff --git a/modules/markup/orgmode/orgmode.go b/modules/markup/orgmode/orgmode.go index 9bef57bd7f36e..96e67f90cfa2a 100644 --- a/modules/markup/orgmode/orgmode.go +++ b/modules/markup/orgmode/orgmode.go @@ -58,7 +58,7 @@ func Render(ctx *markup.RenderContext, input io.Reader, output io.Writer) error return err } -// RenderString renders Markdown string to HTML with all specific handling stuff and return string +// RenderString renders orgmode string to HTML string func RenderString(ctx *markup.RenderContext, content string) (string, error) { var buf strings.Builder if err := Render(ctx, strings.NewReader(content), &buf); err != nil { From 89b677308f4e1ffa506a4668f076adaf7688a7bc Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Tue, 13 Apr 2021 15:18:44 +0800 Subject: [PATCH 20/26] Fix lint --- modules/markup/csv/csv.go | 2 +- modules/markup/external/external.go | 6 +- modules/markup/markup.go | 143 ---------------------------- 3 files changed, 4 insertions(+), 147 deletions(-) delete mode 100644 modules/markup/markup.go diff --git a/modules/markup/csv/csv.go b/modules/markup/csv/csv.go index eab8f3b494f63..6572b0ee1e819 100644 --- a/modules/markup/csv/csv.go +++ b/modules/markup/csv/csv.go @@ -72,7 +72,7 @@ func writeField(w io.Writer, element, class, field string) error { return err } -// Render implements markup.Parser +// Render implements markup.Renderer func (Renderer) Render(ctx *markup.RenderContext, input io.Reader, output io.Writer) error { var tmpBlock = bufio.NewWriter(output) diff --git a/modules/markup/external/external.go b/modules/markup/external/external.go index 93207bda5bdc8..62814c9914b9e 100644 --- a/modules/markup/external/external.go +++ b/modules/markup/external/external.go @@ -38,9 +38,9 @@ func (p *Renderer) Name() string { return p.MarkupName } -// NeedPostProcess implements markup.Parser -func (p *Parser) NeedPostProcess() bool { - return p.MarkupParser.NeedPostProcess +// NeedPostProcess implements markup.Renderer +func (p *Renderer) NeedPostProcess() bool { + return p.MarkupRenderer.NeedPostProcess } // Extensions returns the supported extensions of the tool diff --git a/modules/markup/markup.go b/modules/markup/markup.go deleted file mode 100644 index bc357577758ac..0000000000000 --- a/modules/markup/markup.go +++ /dev/null @@ -1,143 +0,0 @@ -// Copyright 2017 The Gitea Authors. All rights reserved. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. - -package markup - -import ( - "path/filepath" - "strings" - - "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/setting" -) - -// Init initialize regexps for markdown parsing -func Init() { - getIssueFullPattern() - NewSanitizer() - if len(setting.Markdown.CustomURLSchemes) > 0 { - CustomLinkURLSchemes(setting.Markdown.CustomURLSchemes) - } - - // since setting maybe changed extensions, this will reload all parser extensions mapping - extParsers = make(map[string]Parser) - for _, parser := range parsers { - for _, ext := range parser.Extensions() { - extParsers[strings.ToLower(ext)] = parser - } - } -} - -// Parser defines an interface for parsering markup file to HTML -type Parser interface { - Name() string // markup format name - Extensions() []string - NeedPostProcess() bool - Render(rawBytes []byte, urlPrefix string, metas map[string]string, isWiki bool) []byte -} - -var ( - extParsers = make(map[string]Parser) - parsers = make(map[string]Parser) -) - -// RegisterParser registers a new markup file parser -func RegisterParser(parser Parser) { - parsers[parser.Name()] = parser - for _, ext := range parser.Extensions() { - extParsers[strings.ToLower(ext)] = parser - } -} - -// GetParserByFileName get parser by filename -func GetParserByFileName(filename string) Parser { - extension := strings.ToLower(filepath.Ext(filename)) - return extParsers[extension] -} - -// GetParserByType returns a parser according type -func GetParserByType(tp string) Parser { - return parsers[tp] -} - -// Render renders markup file to HTML with all specific handling stuff. -func Render(filename string, rawBytes []byte, urlPrefix string, metas map[string]string) []byte { - return renderFile(filename, rawBytes, urlPrefix, metas, false) -} - -// RenderByType renders markup to HTML with special links and returns string type. -func RenderByType(tp string, rawBytes []byte, urlPrefix string, metas map[string]string) []byte { - return renderByType(tp, rawBytes, urlPrefix, metas, false) -} - -// RenderString renders Markdown to HTML with special links and returns string type. -func RenderString(filename string, raw, urlPrefix string, metas map[string]string) string { - return string(renderFile(filename, []byte(raw), urlPrefix, metas, false)) -} - -// RenderWiki renders markdown wiki page to HTML and return HTML string -func RenderWiki(filename string, rawBytes []byte, urlPrefix string, metas map[string]string) string { - return string(renderFile(filename, rawBytes, urlPrefix, metas, true)) -} - -func render(parser Parser, rawBytes []byte, urlPrefix string, metas map[string]string, isWiki bool) []byte { - result := parser.Render(rawBytes, urlPrefix, metas, isWiki) - if parser.NeedPostProcess() { - var err error - // TODO: one day the error should be returned. - result, err = PostProcess(result, urlPrefix, metas, isWiki) - if err != nil { - log.Error("PostProcess: %v", err) - } - } - return SanitizeBytes(result) -} - -func renderByType(tp string, rawBytes []byte, urlPrefix string, metas map[string]string, isWiki bool) []byte { - if parser, ok := parsers[tp]; ok { - return render(parser, rawBytes, urlPrefix, metas, isWiki) - } - return nil -} - -func renderFile(filename string, rawBytes []byte, urlPrefix string, metas map[string]string, isWiki bool) []byte { - extension := strings.ToLower(filepath.Ext(filename)) - if parser, ok := extParsers[extension]; ok { - return render(parser, rawBytes, urlPrefix, metas, isWiki) - } - return nil -} - -// Type returns if markup format via the filename -func Type(filename string) string { - if parser := GetParserByFileName(filename); parser != nil { - return parser.Name() - } - return "" -} - -// IsMarkupFile reports whether file is a markup type file -func IsMarkupFile(name, markup string) bool { - if parser := GetParserByFileName(name); parser != nil { - return parser.Name() == markup - } - return false -} - -// IsReadmeFile reports whether name looks like a README file -// based on its name. If an extension is provided, it will strictly -// match that extension. -// Note that the '.' should be provided in ext, e.g ".md" -func IsReadmeFile(name string, ext ...string) bool { - name = strings.ToLower(name) - if len(ext) > 0 { - return name == "readme"+ext[0] - } - if len(name) < 6 { - return false - } else if len(name) == 6 { - return name == "readme" - } - return name[:7] == "readme." -} From 5305005a4cd65828686dac4d21ee583377dd4b48 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sat, 17 Apr 2021 17:17:15 +0800 Subject: [PATCH 21/26] Fix test --- services/mailer/mail_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/mailer/mail_test.go b/services/mailer/mail_test.go index a22e7401a23e7..813e51c0d215b 100644 --- a/services/mailer/mail_test.go +++ b/services/mailer/mail_test.go @@ -95,7 +95,7 @@ func TestComposeIssueMessage(t *testing.T) { tos := []string{"test@gitea.com", "test2@gitea.com"} msgs, err := composeIssueCommentMessages(&mailCommentContext{Issue: issue, Doer: doer, ActionType: models.ActionCreateIssue, Content: "test body"}, "en-US", tos, false, "issue create") - assert.Error(t, err) + assert.NoError(t, err) assert.Len(t, msgs, 2) gomailMsg := msgs[0].ToMessage() From a5a2e00a13312ca95718789e2a0c45e57e504053 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 18 Apr 2021 00:30:07 +0800 Subject: [PATCH 22/26] Don't use NormalEOF when unnecessary --- models/repo_generate.go | 9 +++++---- modules/markup/markdown/markdown.go | 5 ++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/models/repo_generate.go b/models/repo_generate.go index b0016494c459e..1cf73bc55e6ad 100644 --- a/models/repo_generate.go +++ b/models/repo_generate.go @@ -5,13 +5,14 @@ package models import ( + "bufio" + "bytes" "strconv" "strings" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/storage" - "code.gitea.io/gitea/modules/util" "github.com/gobwas/glob" ) @@ -49,9 +50,9 @@ func (gt GiteaTemplate) Globs() []glob.Glob { } gt.globs = make([]glob.Glob, 0) - lines := strings.Split(string(util.NormalizeEOL(gt.Content)), "\n") - for _, line := range lines { - line = strings.TrimSpace(line) + scanner := bufio.NewScanner(bytes.NewReader(gt.Content)) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) if line == "" || strings.HasPrefix(line, "#") { continue } diff --git a/modules/markup/markdown/markdown.go b/modules/markup/markdown/markdown.go index 1fa3a0a7a3610..2421b868b778c 100644 --- a/modules/markup/markdown/markdown.go +++ b/modules/markup/markdown/markdown.go @@ -16,7 +16,6 @@ import ( "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/markup/common" "code.gitea.io/gitea/modules/setting" - giteautil "code.gitea.io/gitea/modules/util" chromahtml "github.com/alecthomas/chroma/formatters/html" "github.com/yuin/goldmark" @@ -170,7 +169,7 @@ func actualRender(ctx *markup.RenderContext, input io.Reader, output io.Writer) limit: setting.UI.MaxDisplayFileSize * 3, } - // FIXME: should we include a timeout that closes the pipe to abort the parser and sanitizer if it takes too long? + // FIXME: should we include a timeout that closes the pipe to abort the renderer and sanitizer if it takes too long? go func() { defer func() { err := recover() @@ -187,7 +186,7 @@ func actualRender(ctx *markup.RenderContext, input io.Reader, output io.Writer) // FIXME: Don't read all to memory, but goldmark doesn't support pc := NewGiteaParseContext(ctx) - buf, err := ioutil.ReadAll(giteautil.NormalizeEOLReader(input)) + buf, err := ioutil.ReadAll(input) if err != nil { log.Error("Unable to ReadAll: %v", err) return From c732424353bd807ee53e6784fbb2c3fe9750c5c9 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 18 Apr 2021 00:44:30 +0800 Subject: [PATCH 23/26] revert change on util.go --- modules/util/util.go | 81 ++++++++++++++++---------------------------- 1 file changed, 30 insertions(+), 51 deletions(-) diff --git a/modules/util/util.go b/modules/util/util.go index 8db3c4646d80a..9de1710ac7bad 100644 --- a/modules/util/util.go +++ b/modules/util/util.go @@ -7,8 +7,6 @@ package util import ( "bytes" "errors" - "io" - "io/ioutil" "strings" ) @@ -68,59 +66,40 @@ func IsEmptyString(s string) bool { return len(strings.TrimSpace(s)) == 0 } -type normalizeEOLReader struct { - rd io.Reader - isLastReturn bool -} - -func (r *normalizeEOLReader) Read(bs []byte) (int, error) { - var p = make([]byte, len(bs)) - n, err := r.rd.Read(p) - if err != nil { - return n, err +// NormalizeEOL will convert Windows (CRLF) and Mac (CR) EOLs to UNIX (LF) +func NormalizeEOL(input []byte) []byte { + var right, left, pos int + if right = bytes.IndexByte(input, '\r'); right == -1 { + return input } - - var j = 0 - for i, c := range p[:n] { - if i == 0 { - if c == '\n' && r.isLastReturn { - r.isLastReturn = false - continue - } - r.isLastReturn = false - } - if c == '\r' { - if i < n-1 { - if p[i+1] != '\n' { - bs[j] = '\n' - } else { - continue - } - } else { - r.isLastReturn = true - bs[j] = '\n' - } - } else { - bs[j] = c + length := len(input) + tmp := make([]byte, length) + + // We know that left < length because otherwise right would be -1 from IndexByte. + copy(tmp[pos:pos+right], input[left:left+right]) + pos += right + tmp[pos] = '\n' + left += right + 1 + pos++ + + for left < length { + if input[left] == '\n' { + left++ } - j++ - } - - return j, nil -} -// NormalizeEOLReader will convert Windows (CRLF) and Mac (CR) EOLs to UNIX (LF) from a reader -func NormalizeEOLReader(rd io.Reader) io.Reader { - return &normalizeEOLReader{ - rd: rd, - isLastReturn: false, + right = bytes.IndexByte(input[left:], '\r') + if right == -1 { + copy(tmp[pos:], input[left:]) + pos += length - left + break + } + copy(tmp[pos:pos+right], input[left:left+right]) + pos += right + tmp[pos] = '\n' + left += right + 1 + pos++ } -} - -// NormalizeEOL will convert Windows (CRLF) and Mac (CR) EOLs to UNIX (LF) -func NormalizeEOL(input []byte) []byte { - bs, _ := ioutil.ReadAll(NormalizeEOLReader(bytes.NewReader(input))) - return bs + return tmp[:pos] } // MergeInto merges pairs of values into a "dict" From 26064e5addd26cfe19a662e79722a7c0f2e59853 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 18 Apr 2021 00:56:56 +0800 Subject: [PATCH 24/26] Apply suggestions from code review Co-authored-by: zeripath --- modules/notification/mail/mail.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/notification/mail/mail.go b/modules/notification/mail/mail.go index bd02a6c252d4e..eb45409faf277 100644 --- a/modules/notification/mail/mail.go +++ b/modules/notification/mail/mail.go @@ -105,7 +105,7 @@ func (m *mailNotifier) NotifyIssueChangeAssignee(doer *models.User, issue *model if !removed && doer.ID != assignee.ID && assignee.EmailNotifications() == models.EmailNotificationsEnabled { ct := fmt.Sprintf("Assigned #%d.", issue.Index) if err := mailer.SendIssueAssignedMail(issue, doer, ct, comment, []*models.User{assignee}); err != nil { - log.Error("SendIssueAssignedMail faile: %v", err) + log.Error("Error in SendIssueAssignedMail for issue[%d] to assignee[%d]: %v", issue.ID, assignee.ID, err) } } } @@ -114,7 +114,7 @@ func (m *mailNotifier) NotifyPullReviewRequest(doer *models.User, issue *models. if isRequest && doer.ID != reviewer.ID && reviewer.EmailNotifications() == models.EmailNotificationsEnabled { ct := fmt.Sprintf("Requested to review %s.", issue.HTMLURL()) if err := mailer.SendIssueAssignedMail(issue, doer, ct, comment, []*models.User{reviewer}); err != nil { - log.Error("SendIssueAssignedMail faile: %v", err) + log.Error("Error in SendIssueAssignedMail for issue[%d] to reviewer[%d]: %v", issue.ID, reviewer.ID, err) } } } From 7e7a16117da483e7cc19ef387aad350c59a5f275 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 18 Apr 2021 00:57:42 +0800 Subject: [PATCH 25/26] rename function --- modules/markup/markdown/markdown.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/markup/markdown/markdown.go b/modules/markup/markdown/markdown.go index 2421b868b778c..c5426578c8218 100644 --- a/modules/markup/markdown/markdown.go +++ b/modules/markup/markdown/markdown.go @@ -73,8 +73,8 @@ func (l *limitWriter) CloseWithError(err error) error { return l.w.CloseWithError(err) } -// NewGiteaParseContext creates a parser.Context with the gitea context set -func NewGiteaParseContext(ctx *markup.RenderContext) parser.Context { +// newParserContext creates a parser.Context with the render context set +func newParserContext(ctx *markup.RenderContext) parser.Context { pc := parser.NewContext(parser.WithIDs(newPrefixedIDs())) pc.Set(urlPrefixKey, ctx.URLPrefix) pc.Set(isWikiKey, ctx.IsWiki) @@ -185,7 +185,7 @@ func actualRender(ctx *markup.RenderContext, input io.Reader, output io.Writer) }() // FIXME: Don't read all to memory, but goldmark doesn't support - pc := NewGiteaParseContext(ctx) + pc := newParserContext(ctx) buf, err := ioutil.ReadAll(input) if err != nil { log.Error("Unable to ReadAll: %v", err) From 1eeb58712190d55020cec3ab63bc04e005de4182 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 18 Apr 2021 16:05:54 +0800 Subject: [PATCH 26/26] Take NormalEOF back --- modules/markup/markdown/markdown.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/markup/markdown/markdown.go b/modules/markup/markdown/markdown.go index c5426578c8218..87fae2a23b2fb 100644 --- a/modules/markup/markdown/markdown.go +++ b/modules/markup/markdown/markdown.go @@ -16,6 +16,7 @@ import ( "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/markup/common" "code.gitea.io/gitea/modules/setting" + giteautil "code.gitea.io/gitea/modules/util" chromahtml "github.com/alecthomas/chroma/formatters/html" "github.com/yuin/goldmark" @@ -191,7 +192,7 @@ func actualRender(ctx *markup.RenderContext, input io.Reader, output io.Writer) log.Error("Unable to ReadAll: %v", err) return } - if err := converter.Convert(buf, lw, parser.WithContext(pc)); err != nil { + if err := converter.Convert(giteautil.NormalizeEOL(buf), lw, parser.WithContext(pc)); err != nil { log.Error("Unable to render: %v", err) _ = lw.CloseWithError(err) return