tpl/openapi: Add support for OpenAPI external file references

Fixes #8067

Author: bep

deps/deps.go

@@ -417,6 +417,9 @@ type DepsCfg struct {
 	// i18n handling.
 	TranslationProvider ResourceProvider
 
+	// Build triggered by the IntegrationTest framework.
+	IsIntegrationTest bool
+
 	// ChangesFromBuild for changes passed back to the server/watch process.
 	ChangesFromBuild chan []identity.Identity
 }

hugolib/integrationtest_builder.go

@@ -35,6 +35,7 @@ import (
 	"github.com/gohugoio/hugo/hugofs"
 	"github.com/gohugoio/hugo/hugofs/hglob"
 	"github.com/gohugoio/hugo/hugolib/sitesmatrix"
+	"github.com/gohugoio/hugo/identity"
 	"github.com/spf13/afero"
 	"github.com/spf13/cast"
 	"golang.org/x/text/unicode/norm"
@@ -838,7 +839,11 @@ func (s *IntegrationTestBuilder) initBuilder() error {
 
 		s.Assert(err, qt.IsNil)
 
-		depsCfg := deps.DepsCfg{Configs: res, Fs: fs, LogLevel: logger.Level(), StdErr: logger.StdErr()}
+		// changes received from Hugo in watch mode.
+		// In the full setup, this channel is created in the commands package.
+		changesFromBuild := make(chan []identity.Identity, 10)
+
+		depsCfg := deps.DepsCfg{Configs: res, Fs: fs, LogLevel: logger.Level(), StdErr: logger.StdErr(), ChangesFromBuild: changesFromBuild, IsIntegrationTest: true}
 		sites, err := NewHugoSites(depsCfg)
 		if err != nil {
 			initErr = err
@@ -849,6 +854,20 @@ func (s *IntegrationTestBuilder) initBuilder() error {
 			return
 		}
 
+		go func() {
+			for id := range changesFromBuild {
+				whatChanged := &WhatChanged{}
+				for _, v := range id {
+					whatChanged.Add(v)
+				}
+				bcfg := s.Cfg.BuildCfg
+				bcfg.WhatChanged = whatChanged
+				if err := s.build(bcfg); err != nil {
+					s.Fatalf("Build failed after change: %s", err)
+				}
+			}
+		}()
+
 		s.H = sites
 		s.fs = fs
 

hugolib/rebuild_test.go

@@ -845,7 +845,7 @@ func TestRebuildEditSectionRemoveDate(t *testing.T) {
 func TestRebuildVariations(t *testing.T) {
 	// t.Parallel() not supported, see https://github.com/fortytw2/leaktest/issues/4
 	// This leaktest seems to be a little bit shaky on Travis.
-	if !htesting.IsCI() {
+	if !htesting.IsRealCI() {
 		defer leaktest.CheckTimeout(t, 10*time.Second)()
 	}
 

hugolib/site.go

@@ -265,6 +265,14 @@ func NewHugoSites(cfg deps.DepsCfg) (*HugoSites, error) {
 		return nil, err
 	}
 
+	// Prevent leaking goroutines in tests.
+	if cfg.IsIntegrationTest && cfg.ChangesFromBuild != nil {
+		firstSiteDeps.BuildClosers.Add(types.CloserFunc(func() error {
+			close(cfg.ChangesFromBuild)
+			return nil
+		}))
+	}
+
 	batcherClient, err := esbuild.NewBatcherClient(firstSiteDeps)
 	if err != nil {
 		return nil, err

resources/resource.go

@@ -25,6 +25,7 @@ import (
 
 	"github.com/gohugoio/hugo/identity"
 	"github.com/gohugoio/hugo/resources/internal"
+	"github.com/spf13/cast"
 
 	"github.com/gohugoio/hugo/common/hashing"
 	"github.com/gohugoio/hugo/common/herrors"
@@ -703,6 +704,18 @@ func InternalResourceSourcePath(r resource.Resource) string {
 	return ""
 }
 
+// InternalResourceSourceContent is used internally to get the source content for a Resource.
+func InternalResourceSourceContent(ctx context.Context, r resource.Resource) (string, error) {
+	if cp, ok := r.(resource.ContentProvider); ok {
+		c, err := cp.Content(ctx)
+		if err != nil {
+			return "", err
+		}
+		return cast.ToStringE(c)
+	}
+	return "", nil
+}
+
 // InternalResourceSourcePathBestEffort is used internally to get the source path for a Resource.
 // Used for error messages etc.
 // It will fall back to the target path if the source path is not available.

tpl/openapi/openapi3/init.go

@@ -18,6 +18,7 @@ import (
 
 	"github.com/gohugoio/hugo/deps"
 	"github.com/gohugoio/hugo/tpl/internal"
+	resourcestpl "github.com/gohugoio/hugo/tpl/resources"
 )
 
 const name = "openapi3"
@@ -29,6 +30,17 @@ func init() {
 		ns := &internal.TemplateFuncsNamespace{
 			Name:    name,
 			Context: func(cctx context.Context, args ...any) (any, error) { return ctx, nil },
+			OnCreated: func(m map[string]any) {
+				for _, v := range m {
+					switch v := v.(type) {
+					case *resourcestpl.Namespace:
+						ctx.resourcesNs = v
+					}
+				}
+				if ctx.resourcesNs == nil {
+					panic("resources namespace not found")
+				}
+			},
 		}
 
 		ns.AddMethodMapping(ctx.Unmarshal,

tpl/openapi/openapi3/openapi3.go

@@ -15,16 +15,26 @@
 package openapi3
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"io"
+	"net/url"
+	"path"
+	"path/filepath"
+	"strings"
 
 	kopenapi3 "github.com/getkin/kin-openapi/openapi3"
 	"github.com/gohugoio/hugo/cache/dynacache"
+	"github.com/gohugoio/hugo/common/hashing"
+	"github.com/gohugoio/hugo/common/maps"
 	"github.com/gohugoio/hugo/deps"
 	"github.com/gohugoio/hugo/identity"
 	"github.com/gohugoio/hugo/parser/metadecoders"
+	"github.com/gohugoio/hugo/resources"
 	"github.com/gohugoio/hugo/resources/resource"
+	resourcestpl "github.com/gohugoio/hugo/tpl/resources"
+	"github.com/mitchellh/mapstructure"
 )
 
 // New returns a new instance of the openapi3-namespaced template functions.
@@ -37,8 +47,9 @@ func New(deps *deps.Deps) *Namespace {
 
 // Namespace provides template functions for the "openapi3".
 type Namespace struct {
-	cache *dynacache.Partition[string, *OpenAPIDocument]
-	deps  *deps.Deps
+	cache       *dynacache.Partition[string, *OpenAPIDocument]
+	deps        *deps.Deps
+	resourcesNs *resourcestpl.Namespace
 }
 
 // OpenAPIDocument represents an OpenAPI 3 document.
@@ -51,13 +62,35 @@ func (o *OpenAPIDocument) GetIdentityGroup() identity.Identity {
 	return o.identityGroup
 }
 
+type unmarshalOptions struct {
+	// Options passed to resources.GetRemote when resolving remote $ref.
+	GetRemote map[string]any
+}
+
 // Unmarshal unmarshals the given resource into an OpenAPI 3 document.
-func (ns *Namespace) Unmarshal(r resource.UnmarshableResource) (*OpenAPIDocument, error) {
+func (ns *Namespace) Unmarshal(ctx context.Context, args ...any) (*OpenAPIDocument, error) {
+	if len(args) < 1 || len(args) > 2 {
+		return nil, errors.New("must provide a Resource and optionally an options map")
+	}
+
+	r := args[0].(resource.UnmarshableResource)
 	key := r.Key()
 	if key == "" {
 		return nil, errors.New("no Key set in Resource")
 	}
 
+	var opts unmarshalOptions
+	if len(args) > 1 {
+		optsm, err := maps.ToStringMapE(args[1])
+		if err != nil {
+			return nil, err
+		}
+		if err := mapstructure.WeakDecode(optsm, &opts); err != nil {
+			return nil, err
+		}
+		key += "_" + hashing.HashString(optsm)
+	}
+
 	v, err := ns.cache.GetOrCreate(key, func(string) (*OpenAPIDocument, error) {
 		f := metadecoders.FormatFromStrings(r.MediaType().Suffixes()...)
 		if f == "" {
@@ -86,13 +119,84 @@ func (ns *Namespace) Unmarshal(r resource.UnmarshableResource) (*OpenAPIDocument
 			return nil, err
 		}
 
-		err = kopenapi3.NewLoader().ResolveRefsIn(s, nil)
+		var resourcePath string
+		if res, ok := r.(resource.Resource); ok {
+			resourcePath = resources.InternalResourceSourcePath(res)
+		}
+		var relDir string
+		if resourcePath != "" {
+			if rel, ok := ns.deps.Assets.MakePathRelative(filepath.FromSlash(resourcePath), true); ok {
+				relDir = filepath.Dir(rel)
+			}
+		}
+
+		var idm identity.Manager = identity.NopManager
+		if v := identity.GetDependencyManager(r); v != nil {
+			idm = v
+		}
+		idg := identity.FirstIdentity(r)
+
+		resolver := &refResolver{
+			ctx:     ctx,
+			idm:     idm,
+			opts:    opts,
+			relBase: filepath.ToSlash(relDir),
+			ns:      ns,
+		}
 
-		return &OpenAPIDocument{T: s, identityGroup: identity.FirstIdentity(r)}, err
+		loader := kopenapi3.NewLoader()
+		loader.IsExternalRefsAllowed = true
+		loader.ReadFromURIFunc = resolver.resolveExternalRef
+		err = loader.ResolveRefsIn(s, nil)
+
+		return &OpenAPIDocument{T: s, identityGroup: idg}, err
 	})
 	if err != nil {
 		return nil, err
 	}
 
 	return v, nil
 }
+
+type refResolver struct {
+	ctx     context.Context
+	idm     identity.Manager
+	opts    unmarshalOptions
+	relBase string
+	ns      *Namespace
+}
+
+// resolveExternalRef resolves external references in OpenAPI documents by either fetching
+// remote URLs or loading local files from the assets directory, depending on the reference location.
+func (r *refResolver) resolveExternalRef(loader *kopenapi3.Loader, loc *url.URL) ([]byte, error) {
+	if loc.Scheme != "" && loc.Host != "" {
+		res, err := r.ns.resourcesNs.GetRemote(loc.String(), r.opts.GetRemote)
+		if err != nil {
+			return nil, fmt.Errorf("failed to get remote ref %q: %w", loc.String(), err)
+		}
+		content, err := resources.InternalResourceSourceContent(r.ctx, res)
+		if err != nil {
+			return nil, fmt.Errorf("failed to read remote ref %q: %w", loc.String(), err)
+		}
+		r.idm.AddIdentity(identity.FirstIdentity(res))
+		return []byte(content), nil
+	}
+
+	var filename string
+	if strings.HasPrefix(loc.Path, "/") {
+		filename = loc.Path
+	} else {
+		filename = path.Join(r.relBase, loc.Path)
+	}
+
+	res := r.ns.resourcesNs.Get(filename)
+	if res == nil {
+		return nil, fmt.Errorf("local ref %q not found", loc.String())
+	}
+	content, err := resources.InternalResourceSourceContent(r.ctx, res)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read local ref %q: %w", loc.String(), err)
+	}
+	r.idm.AddIdentity(identity.FirstIdentity(res))
+	return []byte(content), nil
+}