Improve error handling/messages in Hugo Pipes

Fixes #14257
Closes #14270

Author: bep

common/herrors/file_error.go

@@ -288,6 +288,50 @@ func Unwrap(err error) error {
 	return err
 }
 
+// UnwrapFileErrors returns all FileError contained in err.
+func UnwrapFileErrors(err error) []FileError {
+	if err == nil {
+		return nil
+	}
+	errs := Errors(err)
+	var fileErrors []FileError
+	for _, e := range errs {
+		if v, ok := e.(FileError); ok {
+			fileErrors = append(fileErrors, v)
+		}
+		fileErrors = append(fileErrors, UnwrapFileErrors(errors.Unwrap(e))...)
+	}
+	return fileErrors
+}
+
+// UnwrapFileErrorsWithErrorContext tries to unwrap all FileError in err that has an ErrorContext.
+func UnwrapFileErrorsWithErrorContext(err error) []FileError {
+	errs := UnwrapFileErrors(err)
+	var n int
+	for _, e := range errs {
+		if e.ErrorContext() != nil {
+			errs[n] = e
+			n++
+		}
+	}
+	return errs[:n]
+}
+
+// Errors returns the list of errors contained in err.
+func Errors(err error) []error {
+	if err == nil {
+		return nil
+	}
+
+	type unwrapper interface {
+		Unwrap() []error
+	}
+	if u, ok := err.(unwrapper); ok {
+		return u.Unwrap()
+	}
+	return []error{err}
+}
+
 func extractFileTypePos(err error) (string, text.Position) {
 	err = Unwrap(err)
 
@@ -350,30 +394,6 @@ func UnwrapFileError(err error) FileError {
 	return nil
 }
 
-// UnwrapFileErrors tries to unwrap all FileError.
-func UnwrapFileErrors(err error) []FileError {
-	var errs []FileError
-	for err != nil {
-		if v, ok := err.(FileError); ok {
-			errs = append(errs, v)
-		}
-		err = errors.Unwrap(err)
-	}
-	return errs
-}
-
-// UnwrapFileErrorsWithErrorContext tries to unwrap all FileError in err that has an ErrorContext.
-func UnwrapFileErrorsWithErrorContext(err error) []FileError {
-	var errs []FileError
-	for err != nil {
-		if v, ok := err.(FileError); ok && v.ErrorContext() != nil {
-			errs = append(errs, v)
-		}
-		err = errors.Unwrap(err)
-	}
-	return errs
-}
-
 func extractOffsetAndType(e error) (int, string) {
 	switch v := e.(type) {
 	case *json.UnmarshalTypeError:

hugolib/hugo_sites.go

@@ -15,6 +15,7 @@ package hugolib
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"io"
 	"iter"
@@ -254,6 +255,16 @@ func (f *fatalErrorHandler) FatalError(err error) {
 	f.err = err
 }
 
+// Stop stops the fatal error handler without setting an error.
+func (f *fatalErrorHandler) Stop() {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if !f.done {
+		f.done = true
+		close(f.donec)
+	}
+}
+
 func (f *fatalErrorHandler) getErr() error {
 	f.mu.Lock()
 	defer f.mu.Unlock()
@@ -397,38 +408,35 @@ func (h *HugoSites) onPageRender() {
 	}
 }
 
-func (h *HugoSites) pickOneAndLogTheRest(errors []error) error {
-	if len(errors) == 0 {
+func (h *HugoSites) filterAndJoinErrors(errs []error) error {
+	if len(errs) == 0 {
 		return nil
 	}
 
-	var i int
-
-	for j, err := range errors {
-		// If this is in server mode, we want to return an error to the client
-		// with a file context, if possible.
-		if herrors.UnwrapFileError(err) != nil {
-			i = j
-			break
-		}
-	}
-
-	// Log the rest, but add a threshold to avoid flooding the log.
-	const errLogThreshold = 5
-
-	for j, err := range errors {
-		if j == i || err == nil {
+	seen := map[string]bool{}
+	var n int
+	for _, err := range errs {
+		if err == nil {
 			continue
 		}
-
-		if j >= errLogThreshold {
-			break
+		errMsg := herrors.Cause(err).Error() // We don't need to see many "Could not resolve "@alpinejs/persists""
+		if !seen[errMsg] {
+			seen[errMsg] = true
+			errs[n] = err
+			n++
 		}
+	}
+	errs = errs[:n]
 
-		h.Log.Errorln(err)
+	for i, err := range errs {
+		errs[i] = herrors.ImproveRenderErr(err)
 	}
 
-	return errors[i]
+	const limit = 10
+	if len(errs) > limit {
+		errs = errs[:limit]
+	}
+	return errors.Join(errs...)
 }
 
 func (h *HugoSites) isMultilingual() bool {

hugolib/hugo_sites_build.go

@@ -117,7 +117,7 @@ func (h *HugoSites) Build(config BuildCfg, events ...fsnotify.Event) error {
 			}
 			errors = append(errors, e)
 		}
-		to <- h.pickOneAndLogTheRest(errors)
+		to <- h.filterAndJoinErrors(errors)
 
 		close(to)
 	}(errCollector, errs)

hugolib/hugo_sites_build_errors_test.go

@@ -319,7 +319,9 @@ minifyOutput = true
 	b, err := TestE(t, files)
 
 	b.Assert(err, qt.IsNotNil)
-	fe := herrors.UnwrapFileError(err)
+	fes := herrors.UnwrapFileErrors(err)
+	b.Assert(len(fes), qt.Equals, 1)
+	fe := fes[0]
 	b.Assert(fe, qt.IsNotNil)
 	b.Assert(fe.Position().LineNumber, qt.Equals, 2)
 	b.Assert(fe.Position().ColumnNumber, qt.Equals, 9)
@@ -626,7 +628,9 @@ line3: 'value3'
 
 	b.Assert(err, qt.Not(qt.IsNil))
 	b.Assert(err.Error(), qt.Contains, "[2:1] non-map value is specified")
-	fe := herrors.UnwrapFileError(err)
+	fes := herrors.UnwrapFileErrors(err)
+	b.Assert(len(fes), qt.Equals, 1)
+	fe := fes[0]
 	b.Assert(fe, qt.Not(qt.IsNil))
 	pos := fe.Position()
 	b.Assert(pos.Filename, qt.Contains, filepath.FromSlash("content/_index.md"))

hugolib/integrationtest_builder.go

@@ -593,11 +593,6 @@ func (s *IntegrationTestBuilder) AssertFileExists(filename string, b bool) {
 	s.Assert(err, checker)
 }
 
-func (s *IntegrationTestBuilder) AssertIsFileError(err error) herrors.FileError {
-	s.Assert(err, qt.ErrorAs, new(herrors.FileError))
-	return herrors.UnwrapFileError(err)
-}
-
 func (s *IntegrationTestBuilder) AssertRenderCountContent(count int) {
 	s.Helper()
 	s.Assert(s.counters.contentRenderCounter.Load(), qt.Equals, uint64(count))

hugolib/site.go

@@ -1546,13 +1546,18 @@ func (s *Site) resetBuildState(sourceChanged bool) {
 
 func (s *Site) errorCollator(results <-chan error, errs chan<- error) {
 	var errors []error
+	defer func() {
+		errs <- s.h.filterAndJoinErrors(errors)
+		close(errs)
+	}()
+	const maxErrors = 10
 	for e := range results {
 		errors = append(errors, e)
+		if len(errors) >= maxErrors {
+			s.h.Stop()
+			break
+		}
 	}
-
-	errs <- s.h.pickOneAndLogTheRest(errors)
-
-	close(errs)
 }
 
 // GetPage looks up a page of a given type for the given ref.

hugolib/site_render.go

@@ -22,7 +22,6 @@ import (
 
 	"github.com/bep/logg"
 	"github.com/gohugoio/go-radix"
-	"github.com/gohugoio/hugo/common/herrors"
 	"github.com/gohugoio/hugo/hugolib/doctree"
 	"github.com/gohugoio/hugo/tpl/tplimpl"
 
@@ -115,7 +114,7 @@ func (s *Site) renderPages(ctx *siteRenderContext) error {
 
 	err := <-errs
 	if err != nil {
-		return fmt.Errorf("%v failed to render pages: %w", s.resolveDimensionNames(), herrors.ImproveRenderErr(err))
+		return fmt.Errorf("%v failed to render pages: %w", s.resolveDimensionNames(), err)
 	}
 	return nil
 }
@@ -129,6 +128,15 @@ func pageRenderer(
 ) {
 	defer wg.Done()
 
+	sendErr := func(err error) bool {
+		select {
+		case results <- err:
+			return true
+		case <-s.h.Done():
+			return false
+		}
+	}
+
 	for p := range pages {
 
 		if p.m.isStandalone() && !ctx.shouldRenderStandalonePage(p.Kind()) {
@@ -137,8 +145,11 @@ func pageRenderer(
 
 		if p.m.pageConfig.Build.PublishResources {
 			if err := p.renderResources(); err != nil {
-				s.SendError(p.errorf(err, "failed to render page resources"))
-				continue
+				if sendErr(p.errorf(err, "failed to render resources")) {
+					continue
+				} else {
+					return
+				}
 			}
 		}
 
@@ -149,8 +160,11 @@ func pageRenderer(
 
 		templ, found, err := p.resolveTemplate()
 		if err != nil {
-			s.SendError(p.errorf(err, "failed to resolve template"))
-			continue
+			if sendErr(p.errorf(err, "failed to resolve template")) {
+				continue
+			} else {
+				return
+			}
 		}
 
 		if !found {
@@ -181,12 +195,20 @@ func pageRenderer(
 		}
 
 		if err := s.renderAndWritePage(&s.PathSpec.ProcessingStats.Pages, targetPath, p, d, templ); err != nil {
-			results <- err
+			if sendErr(err) {
+				continue
+			} else {
+				return
+			}
 		}
 
 		if p.paginator != nil && p.paginator.current != nil {
 			if err := s.renderPaginator(p, templ); err != nil {
-				results <- err
+				if sendErr(err) {
+					continue
+				} else {
+					return
+				}
 			}
 		}
 	}

main.go

@@ -17,13 +17,19 @@ import (
 	"log"
 	"os"
 
+	"github.com/gohugoio/hugo/common/herrors"
+	"github.com/gohugoio/hugo/common/loggers"
+
 	"github.com/gohugoio/hugo/commands"
 )
 
 func main() {
 	log.SetFlags(0)
 	err := commands.Execute(os.Args[1:])
 	if err != nil {
-		log.Fatalf("Error: %s", err)
+		for _, e := range herrors.Errors(err) {
+			loggers.Log().Errorf("%s", e)
+		}
+		os.Exit(1)
 	}
 }

markup/tableofcontents/tableofcontents_integration_test.go

@@ -17,6 +17,7 @@ import (
 	"strings"
 	"testing"
 
+	qt "github.com/frankban/quicktest"
 	"github.com/gohugoio/hugo/hugolib"
 )
 
@@ -118,8 +119,9 @@ CONTENT
 
 	files = strings.ReplaceAll(files, `2`, `"x"`)
 
-	b, _ = hugolib.TestE(t, files)
-	b.AssertLogMatches(`error calling ToHTML: startLevel: unable to cast "x" of type string`)
+	b, err := hugolib.TestE(t, files)
+	b.Assert(err, qt.Not(qt.IsNil))
+	b.Assert(err.Error(), qt.Contains, `error calling ToHTML: startLevel: unable to cast "x" of type string`)
 }
 
 func TestHeadingsNilpointerIssue11843(t *testing.T) {