Add resource vendoring

Fixes #13309

commands/commandeer.go

@@ -365,7 +365,11 @@ func (r *rootCommand) Name() string {
 }
 
 func (r *rootCommand) Run(ctx context.Context, cd *simplecobra.Commandeer, args []string) error {
-	b := newHugoBuilder(r, nil)
+	var vendor bool
+	if vendorCmd, ok := cd.Command.(vendoredCommand); ok {
+		vendor = vendorCmd.IsVendorCommand()
+	}
+	b := newHugoBuilder(r, nil, vendor)
 
 	if !r.buildWatch {
 		defer b.postBuild("Total", time.Now())

commands/commands.go

@@ -23,7 +23,8 @@ import (
 func newExec() (*simplecobra.Exec, error) {
 	rootCmd := &rootCommand{
 		commands: []simplecobra.Commander{
-			newHugoBuildCmd(),
+			newHugoBuildCmd(false),
+			newHugoBuildCmd(true),
 			newVersionCmd(),
 			newEnvCommand(),
 			newServerCommand(),
@@ -42,26 +43,40 @@ func newExec() (*simplecobra.Exec, error) {
 	return simplecobra.New(rootCmd)
 }
 
-func newHugoBuildCmd() simplecobra.Commander {
-	return &hugoBuildCommand{}
+func newHugoBuildCmd(vendor bool) simplecobra.Commander {
+	return &hugoBuildCommand{
+		vendor: vendor,
+	}
 }
 
 // hugoBuildCommand just delegates to the rootCommand.
 type hugoBuildCommand struct {
 	rootCmd *rootCommand
+	vendor  bool
 }
 
 func (c *hugoBuildCommand) Commands() []simplecobra.Commander {
 	return nil
 }
 
 func (c *hugoBuildCommand) Name() string {
+	if c.vendor {
+		return "vendor"
+	}
 	return "build"
 }
 
+type vendoredCommand interface {
+	IsVendorCommand() bool
+}
+
+func (c *hugoBuildCommand) IsVendorCommand() bool {
+	return c.vendor
+}
+
 func (c *hugoBuildCommand) Init(cd *simplecobra.Commandeer) error {
 	c.rootCmd = cd.Root.Command.(*rootCommand)
-	return c.rootCmd.initRootCommand("build", cd)
+	return c.rootCmd.initRootCommand(c.Name(), cd)
 }
 
 func (c *hugoBuildCommand) PreRun(cd, runner *simplecobra.Commandeer) error {

commands/hugobuilder.go

@@ -73,6 +73,8 @@ type hugoBuilder struct {
 	showErrorInBrowser bool
 
 	errState hugoBuilderErrState
+
+	vendor bool
 }
 
 var errConfigNotSet = errors.New("config not set")
@@ -1046,11 +1048,11 @@ func (c *hugoBuilder) loadConfig(cd *simplecobra.Commandeer, running bool) error
 		}
 	}
 	cfg.Set("environment", c.r.environment)
-
 	cfg.Set("internal", maps.Params{
 		"running":        running,
 		"watch":          watch,
 		"verbose":        c.r.isVerbose(),
+		"vendor":         c.vendor,
 		"fastRenderMode": c.fastRenderMode,
 	})
 

commands/server.go

@@ -84,14 +84,15 @@ const (
 	configChangeGoWork = "go work file"
 )
 
-func newHugoBuilder(r *rootCommand, s *serverCommand, onConfigLoaded ...func(reloaded bool) error) *hugoBuilder {
+func newHugoBuilder(r *rootCommand, s *serverCommand, vendor bool, onConfigLoaded ...func(reloaded bool) error) *hugoBuilder {
 	var visitedURLs *types.EvictingQueue[string]
 	if s != nil && !s.disableFastRender {
 		visitedURLs = types.NewEvictingQueue[string](20)
 	}
 	return &hugoBuilder{
 		r:              r,
 		s:              s,
+		vendor:         vendor,
 		visitedURLs:    visitedURLs,
 		fullRebuildSem: semaphore.NewWeighted(1),
 		debounce:       debounce.New(4 * time.Second),
@@ -563,6 +564,7 @@ func (c *serverCommand) PreRun(cd, runner *simplecobra.Commandeer) error {
 	c.hugoBuilder = newHugoBuilder(
 		c.r,
 		c,
+		false,
 		func(reloaded bool) error {
 			if !reloaded {
 				if err := c.createServerPorts(cd); err != nil {

common/hugio/writers.go

@@ -50,20 +50,25 @@ func NewMultiWriteCloser(writeClosers ...io.WriteCloser) io.WriteCloser {
 	return multiWriteCloser{Writer: io.MultiWriter(writers...), closers: writeClosers}
 }
 
+// NewWriteCloser creates a new io.WriteCloser with the given writer and closer.
+func NewWriteCloser(w io.Writer, closer io.Closer) io.WriteCloser {
+	return struct {
+		io.Writer
+		io.Closer
+	}{
+		w,
+		closer,
+	}
+}
+
 // ToWriteCloser creates an io.WriteCloser from the given io.Writer.
 // If it's not already, one will be created with a Close method that does nothing.
 func ToWriteCloser(w io.Writer) io.WriteCloser {
 	if rw, ok := w.(io.WriteCloser); ok {
 		return rw
 	}
 
-	return struct {
-		io.Writer
-		io.Closer
-	}{
-		w,
-		io.NopCloser(nil),
-	}
+	return NewWriteCloser(w, io.NopCloser(nil))
 }
 
 // ToReadCloser creates an io.ReadCloser from the given io.Reader.

common/hugo/hugo.go

@@ -109,6 +109,11 @@ func (i HugoInfo) IsExtended() bool {
 	return IsExtended
 }
 
+// IsVendor returns whether we're running as `hugo vendor`.
+func (i HugoInfo) IsVendor() bool {
+	return i.conf.Vendor()
+}
+
 // WorkingDir returns the project working directory.
 func (i HugoInfo) WorkingDir() string {
 	return i.conf.WorkingDir()
@@ -166,6 +171,7 @@ type ConfigProvider interface {
 	WorkingDir() string
 	IsMultihost() bool
 	IsMultilingual() bool
+	Vendor() bool
 }
 
 // NewInfo creates a new Hugo Info object.

config/allconfig/allconfig.go

@@ -69,6 +69,7 @@ type InternalConfig struct {
 	Watch          bool
 	FastRenderMode bool
 	LiveReloadPort int
+	Vendor         bool
 }
 
 // All non-params config keys for language.

config/allconfig/configlanguage.go

@@ -137,6 +137,10 @@ func (c ConfigLanguage) Watching() bool {
 	return c.m.Base.Internal.Watch
 }
 
+func (c ConfigLanguage) Vendor() bool {
+	return c.m.Base.Internal.Vendor
+}
+
 func (c ConfigLanguage) NewIdentityManager(name string, opts ...identity.ManagerOption) identity.Manager {
 	if !c.Watching() {
 		return identity.NopManager

config/commonConfig.go

@@ -131,6 +131,7 @@ func (b BuildConfig) clone() BuildConfig {
 	return b
 }
 
+// TODO1 remove, but first add a deprecation warning somewhere.
 func (b BuildConfig) UseResourceCache(err error) bool {
 	if b.UseResourceCacheWhen == "never" {
 		return false

config/configProvider.go

@@ -58,6 +58,7 @@ type AllProvider interface {
 	BuildDrafts() bool
 	Running() bool
 	Watching() bool
+	Vendor() bool
 	NewIdentityManager(name string, opts ...identity.ManagerOption) identity.Manager
 	FastRenderMode() bool
 	PrintUnusedTemplates() bool

hugofs/files/classifier.go

@@ -47,7 +47,9 @@ const (
 	ComponentFolderAssets     = "assets"
 	ComponentFolderI18n       = "i18n"
 
-	FolderResources = "resources"
+	FolderVendor = "_vendor"
+
+	FolderResources = "resources" // TODO1 remove.
 	FolderJSConfig  = "_jsconfig" // Mounted below /assets with postcss.config.js etc.
 
 	NameContentData = "_content"

hugolib/filesystems/basefs.go

@@ -235,13 +235,16 @@ type SourceFilesystems struct {
 	Archetypes *SourceFilesystem
 	Assets     *SourceFilesystem
 
+	// Note that this can not be mounted. It's currently fixed at the project root.
+	Vendor *SourceFilesystem
+
 	AssetsWithDuplicatesPreserved *SourceFilesystem
 
 	RootFss []*hugofs.RootMappingFs
 
 	// Writable filesystem on top the project's resources directory,
 	// with any sub module's resource fs layered below.
-	ResourcesCache afero.Fs
+	ResourcesCache afero.Fs // TODO1 remove this.
 
 	// The work folder (may be a composite of project and theme components).
 	Work afero.Fs
@@ -575,6 +578,7 @@ func (b *sourceFilesystemsBuilder) Build() (*SourceFilesystems, error) {
 	overlayMountsPreserveDupes := b.theBigFs.overlayMounts.WithDirsMerger(hugofs.AppendDirsMerger)
 	b.result.Data = createView(files.ComponentFolderData, overlayMountsPreserveDupes)
 	b.result.I18n = createView(files.ComponentFolderI18n, overlayMountsPreserveDupes)
+	b.result.Vendor = createView(files.FolderVendor, overlayMountsPreserveDupes)
 	b.result.AssetsWithDuplicatesPreserved = createView(files.ComponentFolderAssets, overlayMountsPreserveDupes)
 
 	contentFs := hugofs.NewComponentFs(

hugolib/hugo_sites_build.go

@@ -187,6 +187,10 @@ func (h *HugoSites) Build(config BuildCfg, events ...fsnotify.Event) error {
 		if err := h.postProcess(infol); err != nil {
 			h.SendError(fmt.Errorf("postProcess: %w", err))
 		}
+
+		if err := h.writeVendor(infol); err != nil {
+			h.SendError(fmt.Errorf("writeVendor: %w", err))
+		}
 	}
 
 	if h.Metrics != nil {
@@ -693,6 +697,25 @@ func (h *HugoSites) postProcess(l logg.LevelLogger) error {
 	return g.Wait()
 }
 
+func (h *HugoSites) writeVendor(l logg.LevelLogger) error {
+	if !h.Conf.Vendor() {
+		return nil
+	}
+	l = l.WithField("step", "writeVendor")
+	defer loggers.TimeTrackf(l, time.Now(), nil, "")
+
+	v := h.ResourceSpec.Vendorer
+	if err := v.Finalize(); err != nil {
+		return err
+	}
+
+	for _, vf := range v.VendoredResources {
+		fmt.Println("=== ==", vf.Path, vf.Hash)
+	}
+
+	return nil
+}
+
 func (h *HugoSites) writeBuildStats() error {
 	if h.ResourceSpec == nil {
 		panic("h.ResourceSpec is nil")

hugolib/integrationtest_builder.go

@@ -350,14 +350,18 @@ func (s *IntegrationTestBuilder) AssertNoRenderShortcodesArtifacts() {
 	}
 }
 
+func (s *IntegrationTestBuilder) AssertWorkingDir(root string, matches ...string) {
+	s.AssertFs(s.fs.WorkingDirReadOnly, root, matches...)
+}
+
 func (s *IntegrationTestBuilder) AssertPublishDir(matches ...string) {
-	s.AssertFs(s.fs.PublishDir, matches...)
+	s.AssertFs(s.fs.PublishDir, "", matches...)
 }
 
-func (s *IntegrationTestBuilder) AssertFs(fs afero.Fs, matches ...string) {
+func (s *IntegrationTestBuilder) AssertFs(fs afero.Fs, root string, matches ...string) {
 	s.Helper()
 	var buff bytes.Buffer
-	s.Assert(s.printAndCheckFs(fs, "", &buff), qt.IsNil)
+	s.Assert(s.printAndCheckFs(fs, root, &buff), qt.IsNil)
 	printFsLines := strings.Split(buff.String(), "\n")
 	sort.Strings(printFsLines)
 	content := strings.TrimSpace((strings.Join(printFsLines, "\n")))
@@ -665,17 +669,18 @@ func (s *IntegrationTestBuilder) initBuilder() error {
 			flags = config.New()
 		}
 
+		internal := make(maps.Params)
+
 		if s.Cfg.Running {
-			flags.Set("internal", maps.Params{
-				"running": s.Cfg.Running,
-				"watch":   s.Cfg.Running,
-			})
+			internal["running"] = true
+			internal["watch"] = true
+
 		} else if s.Cfg.Watching {
-			flags.Set("internal", maps.Params{
-				"watch": s.Cfg.Watching,
-			})
+			internal["watch"] = true
 		}
 
+		flags.Set("internal", internal)
+
 		if s.Cfg.WorkingDir != "" {
 			flags.Set("workingDir", s.Cfg.WorkingDir)
 		}

internal/js/esbuild/build.go

@@ -34,14 +34,14 @@ import (
 // NewBuildClient creates a new BuildClient.
 func NewBuildClient(fs *filesystems.SourceFilesystem, rs *resources.Spec) *BuildClient {
 	return &BuildClient{
-		rs:  rs,
+		Rs:  rs,
 		sfs: fs,
 	}
 }
 
 // BuildClient is a client for building JavaScript resources using esbuild.
 type BuildClient struct {
-	rs  *resources.Spec
+	Rs  *resources.Spec
 	sfs *filesystems.SourceFilesystem
 }
 
@@ -52,11 +52,11 @@ func (c *BuildClient) Build(opts Options) (api.BuildResult, error) {
 		dependencyManager = identity.NopManager
 	}
 
-	opts.OutDir = c.rs.AbsPublishDir
-	opts.ResolveDir = c.rs.Cfg.BaseConfig().WorkingDir // where node_modules gets resolved
+	opts.OutDir = c.Rs.AbsPublishDir
+	opts.ResolveDir = c.Rs.Cfg.BaseConfig().WorkingDir // where node_modules gets resolved
 	opts.AbsWorkingDir = opts.ResolveDir
-	opts.TsConfig = c.rs.ResolveJSConfigFile("tsconfig.json")
-	assetsResolver := newFSResolver(c.rs.Assets.Fs)
+	opts.TsConfig = c.Rs.ResolveJSConfigFile("tsconfig.json")
+	assetsResolver := newFSResolver(c.Rs.Assets.Fs)
 
 	if err := opts.validate(); err != nil {
 		return api.BuildResult{}, err
@@ -67,7 +67,7 @@ func (c *BuildClient) Build(opts Options) (api.BuildResult, error) {
 	}
 
 	var err error
-	opts.compiled.Plugins, err = createBuildPlugins(c.rs, assetsResolver, dependencyManager, opts)
+	opts.compiled.Plugins, err = createBuildPlugins(c.Rs, assetsResolver, dependencyManager, opts)
 	if err != nil {
 		return api.BuildResult{}, err
 	}
@@ -175,7 +175,7 @@ func (c *BuildClient) Build(opts Options) (api.BuildResult, error) {
 		// Return 1, log the rest.
 		for i, err := range errors {
 			if i > 0 {
-				c.rs.Logger.Errorf("js.Build failed: %s", err)
+				c.Rs.Logger.Errorf("js.Build failed: %s", err)
 			}
 		}