Handle themes in the new file cache (for images, assets)

In the newly consolidated file cache implementation, we forgot that we also look in the theme(s) for assets (SCSS transformations etc.), which is not good for Netlify and the demo sites.

Fixes #5460

Author: bep

cache/filecache/filecache.go

@@ -17,6 +17,7 @@ import (
 	"bytes"
 	"io"
 	"io/ioutil"
+	"os"
 	"path/filepath"
 	"strings"
 	"sync"
@@ -26,8 +27,6 @@ import (
 
 	"github.com/gohugoio/hugo/helpers"
 
-	"github.com/gohugoio/hugo/hugolib/paths"
-
 	"github.com/BurntSushi/locker"
 	"github.com/spf13/afero"
 )
@@ -305,22 +304,28 @@ func (f Caches) Get(name string) *Cache {
 	return f[strings.ToLower(name)]
 }
 
-// NewCachesFromPaths creates a new set of file caches from the given
+// NewCaches creates a new set of file caches from the given
 // configuration.
-func NewCachesFromPaths(p *paths.Paths) (Caches, error) {
+func NewCaches(p *helpers.PathSpec) (Caches, error) {
 	dcfg, err := decodeConfig(p)
 	if err != nil {
 		return nil, err
 	}
 
-	genDir := filepath.FromSlash("/_gen")
-
 	fs := p.Fs.Source
 
 	m := make(Caches)
 	for k, v := range dcfg {
+		var cfs afero.Fs
+
+		if v.isResourceDir {
+			cfs = p.BaseFs.Resources.Fs
+		} else {
+			cfs = fs
+		}
+
 		var baseDir string
-		if !strings.Contains(v.Dir, genDir) {
+		if !strings.HasPrefix(v.Dir, "_gen") {
 			// We do cache eviction (file removes) and since the user can set
 			// his/hers own cache directory, we really want to make sure
 			// we do not delete any files that do not belong to this cache.
@@ -331,10 +336,12 @@ func NewCachesFromPaths(p *paths.Paths) (Caches, error) {
 		} else {
 			baseDir = filepath.Join(v.Dir, k)
 		}
-		if err = fs.MkdirAll(baseDir, 0777); err != nil {
+		if err = cfs.MkdirAll(baseDir, 0777); err != nil && !os.IsExist(err) {
 			return nil, err
 		}
-		bfs := afero.NewBasePathFs(fs, baseDir)
+
+		bfs := afero.NewBasePathFs(cfs, baseDir)
+
 		m[k] = NewCache(bfs, v.MaxAge)
 	}
 

cache/filecache/filecache_config.go

@@ -20,7 +20,6 @@ import (
 	"time"
 
 	"github.com/gohugoio/hugo/helpers"
-	"github.com/gohugoio/hugo/hugolib/paths"
 
 	"github.com/bep/mapstructure"
 	"github.com/pkg/errors"
@@ -68,6 +67,10 @@ type cacheConfig struct {
 
 	// The directory where files are stored.
 	Dir string
+
+	// Will resources/_gen will get its own composite filesystem that
+	// also checks any theme.
+	isResourceDir bool
 }
 
 // GetJSONCache gets the file cache for getJSON.
@@ -90,7 +93,7 @@ func (f Caches) AssetsCache() *Cache {
 	return f[cacheKeyAssets]
 }
 
-func decodeConfig(p *paths.Paths) (cachesConfig, error) {
+func decodeConfig(p *helpers.PathSpec) (cachesConfig, error) {
 	c := make(cachesConfig)
 	valid := make(map[string]bool)
 	// Add defaults
@@ -145,10 +148,13 @@ func decodeConfig(p *paths.Paths) (cachesConfig, error) {
 
 		for i, part := range parts {
 			if strings.HasPrefix(part, ":") {
-				resolved, err := resolveDirPlaceholder(p, part)
+				resolved, isResource, err := resolveDirPlaceholder(p, part)
 				if err != nil {
 					return c, err
 				}
+				if isResource {
+					v.isResourceDir = true
+				}
 				parts[i] = resolved
 			}
 		}
@@ -159,13 +165,15 @@ func decodeConfig(p *paths.Paths) (cachesConfig, error) {
 		}
 		v.Dir = filepath.Clean(filepath.FromSlash(dir))
 
-		if isOsFs && !filepath.IsAbs(v.Dir) {
-			return c, errors.Errorf("%q must resolve to an absolute directory", v.Dir)
-		}
+		if !v.isResourceDir {
+			if isOsFs && !filepath.IsAbs(v.Dir) {
+				return c, errors.Errorf("%q must resolve to an absolute directory", v.Dir)
+			}
 
-		// Avoid cache in root, e.g. / (Unix) or c:\ (Windows)
-		if len(strings.TrimPrefix(v.Dir, filepath.VolumeName(v.Dir))) == 1 {
-			return c, errors.Errorf("%q is a root folder and not allowed as cache dir", v.Dir)
+			// Avoid cache in root, e.g. / (Unix) or c:\ (Windows)
+			if len(strings.TrimPrefix(v.Dir, filepath.VolumeName(v.Dir))) == 1 {
+				return c, errors.Errorf("%q is a root folder and not allowed as cache dir", v.Dir)
+			}
 		}
 
 		if disabled {
@@ -179,15 +187,16 @@ func decodeConfig(p *paths.Paths) (cachesConfig, error) {
 }
 
 // Resolves :resourceDir => /myproject/resources etc., :cacheDir => ...
-func resolveDirPlaceholder(p *paths.Paths, placeholder string) (string, error) {
+func resolveDirPlaceholder(p *helpers.PathSpec, placeholder string) (cacheDir string, isResource bool, err error) {
 	switch strings.ToLower(placeholder) {
 	case ":resourcedir":
-		return p.AbsResourcesDir, nil
+		return "", true, nil
 	case ":cachedir":
-		return helpers.GetCacheDir(p.Fs.Source, p.Cfg)
+		d, err := helpers.GetCacheDir(p.Fs.Source, p.Cfg)
+		return d, false, err
 	case ":project":
-		return filepath.Base(p.WorkingDir), nil
+		return filepath.Base(p.WorkingDir), false, nil
 	}
 
-	return "", errors.Errorf("%q is not a valid placeholder (valid values are :cacheDir or :resourceDir)", placeholder)
+	return "", false, errors.Errorf("%q is not a valid placeholder (valid values are :cacheDir or :resourceDir)", placeholder)
 }

cache/filecache/filecache_config_test.go

@@ -20,9 +20,10 @@ import (
 	"testing"
 	"time"
 
+	"github.com/gohugoio/hugo/helpers"
+
 	"github.com/gohugoio/hugo/config"
 	"github.com/gohugoio/hugo/hugofs"
-	"github.com/gohugoio/hugo/hugolib/paths"
 
 	"github.com/spf13/viper"
 	"github.com/stretchr/testify/require"
@@ -35,6 +36,13 @@ func TestDecodeConfig(t *testing.T) {
 
 	configStr := `
 resourceDir = "myresources"
+contentDir = "content"
+dataDir = "data"
+i18nDir = "i18n"
+layoutDir = "layouts"
+assetDir = "assets"
+archetypeDir = "archetypes"
+
 [caches]
 [caches.getJSON]
 maxAge = "10m"
@@ -50,7 +58,7 @@ dir = "/path/to/c3"
 	cfg, err := config.FromConfigString(configStr, "toml")
 	assert.NoError(err)
 	fs := hugofs.NewMem(cfg)
-	p, err := paths.New(fs, cfg)
+	p, err := helpers.NewPathSpec(fs, cfg)
 	assert.NoError(err)
 
 	decoded, err := decodeConfig(p)
@@ -75,6 +83,13 @@ func TestDecodeConfigIgnoreCache(t *testing.T) {
 
 	configStr := `
 resourceDir = "myresources"
+contentDir = "content"
+dataDir = "data"
+i18nDir = "i18n"
+layoutDir = "layouts"
+assetDir = "assets"
+archeTypedir = "archetypes"
+
 ignoreCache = true
 [caches]
 [caches.getJSON]
@@ -91,7 +106,7 @@ dir = "/path/to/c3"
 	cfg, err := config.FromConfigString(configStr, "toml")
 	assert.NoError(err)
 	fs := hugofs.NewMem(cfg)
-	p, err := paths.New(fs, cfg)
+	p, err := helpers.NewPathSpec(fs, cfg)
 	assert.NoError(err)
 
 	decoded, err := decodeConfig(p)
@@ -107,8 +122,7 @@ dir = "/path/to/c3"
 
 func TestDecodeConfigDefault(t *testing.T) {
 	assert := require.New(t)
-	cfg := viper.New()
-	cfg.Set("workingDir", filepath.FromSlash("/my/cool/hugoproject"))
+	cfg := newTestConfig()
 
 	if runtime.GOOS == "windows" {
 		cfg.Set("resourceDir", "c:\\cache\\resources")
@@ -120,7 +134,7 @@ func TestDecodeConfigDefault(t *testing.T) {
 	}
 
 	fs := hugofs.NewMem(cfg)
-	p, err := paths.New(fs, cfg)
+	p, err := helpers.NewPathSpec(fs, cfg)
 	assert.NoError(err)
 
 	decoded, err := decodeConfig(p)
@@ -129,12 +143,18 @@ func TestDecodeConfigDefault(t *testing.T) {
 
 	assert.Equal(4, len(decoded))
 
+	imgConfig := decoded[cacheKeyImages]
+	jsonConfig := decoded[cacheKeyGetJSON]
+
 	if runtime.GOOS == "windows" {
-		assert.Equal("c:\\cache\\resources\\_gen", decoded[cacheKeyImages].Dir)
+		assert.Equal("_gen", imgConfig.Dir)
 	} else {
-		assert.Equal("/cache/resources/_gen", decoded[cacheKeyImages].Dir)
-		assert.Equal("/cache/thecache/hugoproject", decoded[cacheKeyGetJSON].Dir)
+		assert.Equal("_gen", imgConfig.Dir)
+		assert.Equal("/cache/thecache/hugoproject", jsonConfig.Dir)
 	}
+
+	assert.True(imgConfig.isResourceDir)
+	assert.False(jsonConfig.isResourceDir)
 }
 
 func TestDecodeConfigInvalidDir(t *testing.T) {
@@ -144,6 +164,13 @@ func TestDecodeConfigInvalidDir(t *testing.T) {
 
 	configStr := `
 resourceDir = "myresources"
+contentDir = "content"
+dataDir = "data"
+i18nDir = "i18n"
+layoutDir = "layouts"
+assetDir = "assets"
+archeTypedir = "archetypes"
+
 [caches]
 [caches.getJSON]
 maxAge = "10m"
@@ -157,10 +184,24 @@ dir = "/"
 	cfg, err := config.FromConfigString(configStr, "toml")
 	assert.NoError(err)
 	fs := hugofs.NewMem(cfg)
-	p, err := paths.New(fs, cfg)
+	p, err := helpers.NewPathSpec(fs, cfg)
 	assert.NoError(err)
 
 	_, err = decodeConfig(p)
 	assert.Error(err)
 
 }
+
+func newTestConfig() *viper.Viper {
+	cfg := viper.New()
+	cfg.Set("workingDir", filepath.FromSlash("/my/cool/hugoproject"))
+	cfg.Set("contentDir", "content")
+	cfg.Set("dataDir", "data")
+	cfg.Set("resourceDir", "resources")
+	cfg.Set("i18nDir", "i18n")
+	cfg.Set("layoutDir", "layouts")
+	cfg.Set("archetypeDir", "archetypes")
+	cfg.Set("assetDir", "assets")
+
+	return cfg
+}