deploy: walkLocal worker pool for performance

Author: davidejones

deploy/deploy.go

@@ -36,6 +36,7 @@ import (
 	"github.com/dustin/go-humanize"
 	"github.com/gobwas/glob"
 	"github.com/gohugoio/hugo/common/loggers"
+	"github.com/gohugoio/hugo/common/para"
 	"github.com/gohugoio/hugo/config"
 	"github.com/gohugoio/hugo/deploy/deployconfig"
 	"github.com/gohugoio/hugo/media"
@@ -487,7 +488,12 @@ func knownHiddenDirectory(name string) bool {
 // walkLocal walks the source directory and returns a flat list of files,
 // using localFile.SlashPath as the map keys.
 func (d *Deployer) walkLocal(fs afero.Fs, matchers []*deployconfig.Matcher, include, exclude glob.Glob, mediaTypes media.Types, mappath func(string) string) (map[string]*localFile, error) {
-	retval := map[string]*localFile{}
+	retval := make(map[string]*localFile)
+	var mu sync.Mutex
+
+	workers := para.New(d.cfg.Workers)
+	g, _ := workers.Start(context.Background())
+
 	err := afero.Walk(fs, "", func(path string, info os.FileInfo, err error) error {
 		if err != nil {
 			return err
@@ -508,45 +514,54 @@ func (d *Deployer) walkLocal(fs afero.Fs, matchers []*deployconfig.Matcher, incl
 			return nil
 		}
 
-		// When a file system is HFS+, its filepath is in NFD form.
-		if runtime.GOOS == "darwin" {
-			path = norm.NFC.String(path)
-		}
+		// Process each file in a worker
+		g.Run(func() error {
+			// When a file system is HFS+, its filepath is in NFD form.
+			if runtime.GOOS == "darwin" {
+				path = norm.NFC.String(path)
+			}
 
-		// Check include/exclude matchers.
-		slashpath := filepath.ToSlash(path)
-		if include != nil && !include.Match(slashpath) {
-			d.logger.Infof("  dropping %q due to include\n", slashpath)
-			return nil
-		}
-		if exclude != nil && exclude.Match(slashpath) {
-			d.logger.Infof("  dropping %q due to exclude\n", slashpath)
-			return nil
-		}
+			// Check include/exclude matchers.
+			slashpath := filepath.ToSlash(path)
+			if include != nil && !include.Match(slashpath) {
+				d.logger.Infof("  dropping %q due to include\n", slashpath)
+				return nil
+			}
+			if exclude != nil && exclude.Match(slashpath) {
+				d.logger.Infof("  dropping %q due to exclude\n", slashpath)
+				return nil
+			}
 
-		// Find the first matching matcher (if any).
-		var m *deployconfig.Matcher
-		for _, cur := range matchers {
-			if cur.Matches(slashpath) {
-				m = cur
-				break
+			// Find the first matching matcher (if any).
+			var m *deployconfig.Matcher
+			for _, cur := range matchers {
+				if cur.Matches(slashpath) {
+					m = cur
+					break
+				}
 			}
-		}
-		// Apply any additional modifications to the local path, to map it to
-		// the remote path.
-		if mappath != nil {
-			slashpath = mappath(slashpath)
-		}
-		lf, err := newLocalFile(fs, path, slashpath, m, mediaTypes)
-		if err != nil {
-			return err
-		}
-		retval[lf.SlashPath] = lf
+			// Apply any additional modifications to the local path, to map it to
+			// the remote path.
+			if mappath != nil {
+				slashpath = mappath(slashpath)
+			}
+			lf, err := newLocalFile(fs, path, slashpath, m, mediaTypes)
+			if err != nil {
+				return err
+			}
+			mu.Lock()
+			retval[lf.SlashPath] = lf
+			mu.Unlock()
+			return nil
+		})
 		return nil
 	})
 	if err != nil {
 		return nil, err
 	}
+	if err := g.Wait(); err != nil {
+		return nil, err
+	}
 	return retval, nil
 }
 

deploy/deploy_test.go

@@ -623,7 +623,7 @@ func TestEndToEndSync(t *testing.T) {
 				localFs:    test.fs,
 				bucket:     test.bucket,
 				mediaTypes: media.DefaultTypes,
-				cfg:        deployconfig.DeployConfig{MaxDeletes: -1},
+				cfg:        deployconfig.DeployConfig{Workers: 2, MaxDeletes: -1},
 			}
 
 			// Initial deployment should sync remote with local.
@@ -706,7 +706,7 @@ func TestMaxDeletes(t *testing.T) {
 				localFs:    test.fs,
 				bucket:     test.bucket,
 				mediaTypes: media.DefaultTypes,
-				cfg:        deployconfig.DeployConfig{MaxDeletes: -1},
+				cfg:        deployconfig.DeployConfig{Workers: 2, MaxDeletes: -1},
 			}
 
 			// Sync remote with local.
@@ -836,7 +836,7 @@ func TestIncludeExclude(t *testing.T) {
 			}
 			deployer := &Deployer{
 				localFs: fsTest.fs,
-				cfg:     deployconfig.DeployConfig{MaxDeletes: -1}, bucket: fsTest.bucket,
+				cfg:     deployconfig.DeployConfig{Workers: 2, MaxDeletes: -1}, bucket: fsTest.bucket,
 				target:     tgt,
 				mediaTypes: media.DefaultTypes,
 			}
@@ -893,7 +893,7 @@ func TestIncludeExcludeRemoteDelete(t *testing.T) {
 			}
 			deployer := &Deployer{
 				localFs: fsTest.fs,
-				cfg:     deployconfig.DeployConfig{MaxDeletes: -1}, bucket: fsTest.bucket,
+				cfg:     deployconfig.DeployConfig{Workers: 2, MaxDeletes: -1}, bucket: fsTest.bucket,
 				mediaTypes: media.DefaultTypes,
 			}
 
@@ -945,7 +945,7 @@ func TestCompression(t *testing.T) {
 			deployer := &Deployer{
 				localFs:    test.fs,
 				bucket:     test.bucket,
-				cfg:        deployconfig.DeployConfig{MaxDeletes: -1, Matchers: []*deployconfig.Matcher{{Pattern: ".*", Gzip: true, Re: regexp.MustCompile(".*")}}},
+				cfg:        deployconfig.DeployConfig{Workers: 2, MaxDeletes: -1, Matchers: []*deployconfig.Matcher{{Pattern: ".*", Gzip: true, Re: regexp.MustCompile(".*")}}},
 				mediaTypes: media.DefaultTypes,
 			}
 
@@ -1000,7 +1000,7 @@ func TestMatching(t *testing.T) {
 			deployer := &Deployer{
 				localFs:    test.fs,
 				bucket:     test.bucket,
-				cfg:        deployconfig.DeployConfig{MaxDeletes: -1, Matchers: []*deployconfig.Matcher{{Pattern: "^subdir/aaa$", Force: true, Re: regexp.MustCompile("^subdir/aaa$")}}},
+				cfg:        deployconfig.DeployConfig{Workers: 2, MaxDeletes: -1, Matchers: []*deployconfig.Matcher{{Pattern: "^subdir/aaa$", Force: true, Re: regexp.MustCompile("^subdir/aaa$")}}},
 				mediaTypes: media.DefaultTypes,
 			}
 
@@ -1097,5 +1097,6 @@ func verifyRemote(ctx context.Context, bucket *blob.Bucket, local []*fileData) (
 func newDeployer() *Deployer {
 	return &Deployer{
 		logger: loggers.NewDefault(),
+		cfg:    deployconfig.DeployConfig{Workers: 2},
 	}
 }