Skip to content

Commit 555dfa2

Browse files
bepbep
committed
Speedup and simplify page assembly for deeper content trees
This commit moves to a forked version go-radix (fork source has not had any updates in 3 years.), whith 2 notable changes: * It's generic (using Go generics) and thus removes a lot of type conversions/assertions. * It allows nodes to be replaced during walk, which allows to partition the tree for parallel processing without worrying about locking. For this repo, this means: * The assembly step now processes nested sections in parallel, which gives a speedup for deep content trees with a slight allocation penalty (see benchmarks below). * Nodes that needs to be reinserted are inserted directly. * Also, there are some drive-by fixes of some allocation issues, e.g. avoid wrapping mutexes in returned anonomous functions, a common source of hidden allocations. ``` │ master.bench │ perf-p3.bench │ │ sec/op │ sec/op vs base │ AssembleDeepSiteWithManySections/depth=1/sectionsPerLevel=1/pagesPerSection=50-10 6.958m ± 3% 7.015m ± 3% ~ (p=0.589 n=6) AssembleDeepSiteWithManySections/depth=1/sectionsPerLevel=6/pagesPerSection=100-10 14.25m ± 1% 14.56m ± 8% ~ (p=0.394 n=6) AssembleDeepSiteWithManySections/depth=1/sectionsPerLevel=6/pagesPerSection=500-10 48.07m ± 3% 49.23m ± 3% ~ (p=0.394 n=6) AssembleDeepSiteWithManySections/depth=2/sectionsPerLevel=6/pagesPerSection=100-10 66.66m ± 4% 66.47m ± 6% ~ (p=0.485 n=6) AssembleDeepSiteWithManySections/depth=4/sectionsPerLevel=2/pagesPerSection=100-10 59.57m ± 4% 50.73m ± 5% -14.85% (p=0.002 n=6) geomean 28.54m 27.92m -2.18% │ master.bench │ perf-p3.bench │ │ B/op │ B/op vs base │ AssembleDeepSiteWithManySections/depth=1/sectionsPerLevel=1/pagesPerSection=50-10 4.513Mi ± 0% 4.527Mi ± 0% +0.33% (p=0.002 n=6) AssembleDeepSiteWithManySections/depth=1/sectionsPerLevel=6/pagesPerSection=100-10 15.35Mi ± 0% 15.49Mi ± 0% +0.94% (p=0.002 n=6) AssembleDeepSiteWithManySections/depth=1/sectionsPerLevel=6/pagesPerSection=500-10 62.50Mi ± 0% 63.19Mi ± 0% +1.10% (p=0.002 n=6) AssembleDeepSiteWithManySections/depth=2/sectionsPerLevel=6/pagesPerSection=100-10 86.78Mi ± 0% 87.73Mi ± 0% +1.09% (p=0.002 n=6) AssembleDeepSiteWithManySections/depth=4/sectionsPerLevel=2/pagesPerSection=100-10 62.96Mi ± 0% 63.66Mi ± 0% +1.12% (p=0.002 n=6) geomean 29.84Mi 30.11Mi +0.92% │ master.bench │ perf-p3.bench │ │ allocs/op │ allocs/op vs base │ AssembleDeepSiteWithManySections/depth=1/sectionsPerLevel=1/pagesPerSection=50-10 60.44k ± 0% 60.97k ± 0% +0.87% (p=0.002 n=6) AssembleDeepSiteWithManySections/depth=1/sectionsPerLevel=6/pagesPerSection=100-10 205.8k ± 0% 211.4k ± 0% +2.70% (p=0.002 n=6) AssembleDeepSiteWithManySections/depth=1/sectionsPerLevel=6/pagesPerSection=500-10 831.1k ± 0% 858.3k ± 0% +3.27% (p=0.002 n=6) AssembleDeepSiteWithManySections/depth=2/sectionsPerLevel=6/pagesPerSection=100-10 1.157M ± 0% 1.197M ± 0% +3.41% (p=0.002 n=6) AssembleDeepSiteWithManySections/depth=4/sectionsPerLevel=2/pagesPerSection=100-10 839.9k ± 0% 867.8k ± 0% +3.31% (p=0.002 n=6) geomean 398.5k 409.3k +2.71% ```
1 parent 3d21b06 commit 555dfa2

20 files changed

+455
-469
lines changed

‎common/hstrings/strings.go‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,16 @@ func GetOrCompileRegexp(pattern string) (re *regexp.Regexp, err error) {
9797
return reCache.getOrCompileRegexp(pattern)
9898
}
9999

100+
// HasAnyPrefix checks if the string s has any of the prefixes given.
101+
func HasAnyPrefix(s string, prefixes ...string) bool {
102+
for _, p := range prefixes {
103+
if strings.HasPrefix(s, p) {
104+
return true
105+
}
106+
}
107+
return false
108+
}
109+
100110
// InSlice checks if a string is an element of a slice of strings
101111
// and returns a boolean value.
102112
func InSlice(arr []string, el string) bool {

‎go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@ require (
44
github.com/BurntSushi/locker v0.0.0-20171006230638-a6e239ea1c69
55
github.com/JohannesKaufmann/html-to-markdown/v2 v2.4.0
66
github.com/alecthomas/chroma/v2 v2.20.0
7-
github.com/armon/go-radix v1.0.1-0.20221118154546-54df44f2176c
87
github.com/aws/aws-sdk-go-v2 v1.40.0
98
github.com/aws/aws-sdk-go-v2/service/cloudfront v1.57.0
109
github.com/bep/clocks v0.5.0
@@ -38,6 +37,7 @@ require (
3837
github.com/gobwas/glob v0.2.3
3938
github.com/goccy/go-yaml v1.18.0
4039
github.com/gohugoio/go-i18n/v2 v2.1.3-0.20251018145728-cfcc22d823c6
40+
github.com/gohugoio/go-radix v1.2.0
4141
github.com/gohugoio/hashstructure v0.6.0
4242
github.com/gohugoio/httpcache v0.8.0
4343
github.com/gohugoio/hugo-goldmark-extensions/extras v0.5.0

‎go.sum‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -102,8 +102,6 @@ github.com/alecthomas/chroma/v2 v2.20.0 h1:sfIHpxPyR07/Oylvmcai3X/exDlE8+FA820NT
102102
github.com/alecthomas/chroma/v2 v2.20.0/go.mod h1:e7tViK0xh/Nf4BYHl00ycY6rV7b8iXBksI9E359yNmA=
103103
github.com/alecthomas/repr v0.5.1 h1:E3G4t2QbHTSNpPKBgMTln5KLkZHLOcU7r37J4pXBuIg=
104104
github.com/alecthomas/repr v0.5.1/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
105-
github.com/armon/go-radix v1.0.1-0.20221118154546-54df44f2176c h1:651/eoCRnQ7YtSjAnSzRucrJz+3iGEFt+ysraELS81M=
106-
github.com/armon/go-radix v1.0.1-0.20221118154546-54df44f2176c/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
107105
github.com/aws/aws-sdk-go v1.55.7 h1:UJrkFq7es5CShfBwlWAC8DA077vp8PyVbQd3lqLiztE=
108106
github.com/aws/aws-sdk-go v1.55.7/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
109107
github.com/aws/aws-sdk-go-v2 v1.40.0 h1:/WMUA0kjhZExjOQN2z3oLALDREea1A7TobfuiBrKlwc=
@@ -276,6 +274,8 @@ github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw=
276274
github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA=
277275
github.com/gohugoio/go-i18n/v2 v2.1.3-0.20251018145728-cfcc22d823c6 h1:pxlAea9eRwuAnt/zKbGqlFO2ZszpIe24YpOVLf+N+4I=
278276
github.com/gohugoio/go-i18n/v2 v2.1.3-0.20251018145728-cfcc22d823c6/go.mod h1:m5hu1im5Qc7LDycVLvee6MPobJiRLBYHklypFJR0/aE=
277+
github.com/gohugoio/go-radix v1.2.0 h1:D5GTk8jIoeXirBSc2P4E4NdHKDrenk9k9N0ctU5Yrhg=
278+
github.com/gohugoio/go-radix v1.2.0/go.mod h1:k6vDa0ebpbpgtzSj9lPGJcA4AZwJ9xUNObUy2vczPFM=
279279
github.com/gohugoio/hashstructure v0.6.0 h1:7wMB/2CfXoThFYhdWRGv3u3rUM761Cq29CxUW+NltUg=
280280
github.com/gohugoio/hashstructure v0.6.0/go.mod h1:lapVLk9XidheHG1IQ4ZSbyYrXcaILU1ZEP/+vno5rBQ=
281281
github.com/gohugoio/httpcache v0.8.0 h1:hNdsmGSELztetYCsPVgjA960zSa4dfEqqF/SficorCU=

‎hugofs/rootmapping_fs.go‎

Lines changed: 37 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ import (
3030
"github.com/gohugoio/hugo/hugofs/files"
3131
"github.com/gohugoio/hugo/hugofs/hglob"
3232

33-
radix "github.com/armon/go-radix"
33+
radix "github.com/gohugoio/go-radix"
3434
"github.com/spf13/afero"
3535
)
3636

@@ -42,17 +42,12 @@ var _ ReverseLookupProvder = (*RootMappingFs)(nil)
4242
// root mappings with some optional metadata about the root.
4343
// Note that From represents a virtual root that maps to the actual filename in To.
4444
func NewRootMappingFs(fs afero.Fs, rms ...*RootMapping) (*RootMappingFs, error) {
45-
rootMapToReal := radix.New()
46-
realMapToRoot := radix.New()
45+
rootMapToReal := radix.New[[]*RootMapping]()
46+
realMapToRoot := radix.New[[]*RootMapping]()
4747
id := fmt.Sprintf("rfs-%d", rootMappingFsCounter.Add(1))
4848

49-
addMapping := func(key string, rm *RootMapping, to *radix.Tree) {
50-
var mappings []*RootMapping
51-
v, found := to.Get(key)
52-
if found {
53-
// There may be more than one language pointing to the same root.
54-
mappings = v.([]*RootMapping)
55-
}
49+
addMapping := func(key string, rm *RootMapping, to *radix.Tree[[]*RootMapping]) {
50+
mappings, _ := to.Get(key)
5651
mappings = append(mappings, rm)
5752
to.Insert(key, mappings)
5853
}
@@ -232,8 +227,8 @@ var _ FilesystemUnwrapper = (*RootMappingFs)(nil)
232227
type RootMappingFs struct {
233228
id string
234229
afero.Fs
235-
rootMapToReal *radix.Tree
236-
realMapToRoot *radix.Tree
230+
rootMapToReal *radix.Tree[[]*RootMapping]
231+
realMapToRoot *radix.Tree[[]*RootMapping]
237232
}
238233

239234
var rootMappingFsCounter atomic.Int32
@@ -279,9 +274,8 @@ func (fs *RootMappingFs) UnwrapFilesystem() afero.Fs {
279274

280275
// Filter creates a copy of this filesystem with only mappings matching a filter.
281276
func (fs RootMappingFs) Filter(f func(m *RootMapping) bool) *RootMappingFs {
282-
rootMapToReal := radix.New()
283-
fs.rootMapToReal.Walk(func(b string, v any) bool {
284-
rms := v.([]*RootMapping)
277+
rootMapToReal := radix.New[[]*RootMapping]()
278+
var walkFn radix.WalkFn[[]*RootMapping] = func(b string, rms []*RootMapping) (radix.WalkFlag, []*RootMapping, error) {
285279
var nrms []*RootMapping
286280
for _, rm := range rms {
287281
if f(rm) {
@@ -291,8 +285,9 @@ func (fs RootMappingFs) Filter(f func(m *RootMapping) bool) *RootMappingFs {
291285
if len(nrms) != 0 {
292286
rootMapToReal.Insert(b, nrms)
293287
}
294-
return false
295-
})
288+
return radix.WalkContinue, nil, nil
289+
}
290+
fs.rootMapToReal.Walk(walkFn)
296291

297292
fs.rootMapToReal = rootMapToReal
298293

@@ -385,21 +380,18 @@ func (fs *RootMappingFs) ReverseLookupComponent(component, filename string) ([]C
385380

386381
func (fs *RootMappingFs) hasPrefix(prefix string) bool {
387382
hasPrefix := false
388-
fs.rootMapToReal.WalkPrefix(prefix, func(b string, v any) bool {
383+
var walkFn radix.WalkFn[[]*RootMapping] = func(b string, rms []*RootMapping) (radix.WalkFlag, []*RootMapping, error) {
389384
hasPrefix = true
390-
return true
391-
})
385+
return radix.WalkStop, nil, nil
386+
}
387+
fs.rootMapToReal.WalkPrefix(prefix, walkFn)
392388

393389
return hasPrefix
394390
}
395391

396392
func (fs *RootMappingFs) getRoot(key string) []*RootMapping {
397-
v, found := fs.rootMapToReal.Get(key)
398-
if !found {
399-
return nil
400-
}
401-
402-
return v.([]*RootMapping)
393+
v, _ := fs.rootMapToReal.Get(key)
394+
return v
403395
}
404396

405397
func (fs *RootMappingFs) getRoots(key string) (string, []*RootMapping) {
@@ -418,7 +410,7 @@ func (fs *RootMappingFs) getRoots(key string) (string, []*RootMapping) {
418410
break
419411
}
420412

421-
for _, rm := range vv.([]*RootMapping) {
413+
for _, rm := range vv {
422414
if !seen[rm] {
423415
seen[rm] = true
424416
roots = append(roots, rm)
@@ -439,34 +431,33 @@ func (fs *RootMappingFs) getRoots(key string) (string, []*RootMapping) {
439431

440432
func (fs *RootMappingFs) getRootsReverse(key string) (string, []*RootMapping) {
441433
tree := fs.realMapToRoot
442-
s, v, found := tree.LongestPrefix(key)
443-
if !found {
444-
return "", nil
445-
}
446-
return s, v.([]*RootMapping)
434+
s, v, _ := tree.LongestPrefix(key)
435+
return s, v
447436
}
448437

449438
func (fs *RootMappingFs) getRootsWithPrefix(prefix string) []*RootMapping {
450439
var roots []*RootMapping
451-
fs.rootMapToReal.WalkPrefix(prefix, func(b string, v any) bool {
452-
roots = append(roots, v.([]*RootMapping)...)
453-
return false
454-
})
440+
var walkFn radix.WalkFn[[]*RootMapping] = func(b string, v []*RootMapping) (radix.WalkFlag, []*RootMapping, error) {
441+
roots = append(roots, v...)
442+
return radix.WalkContinue, nil, nil
443+
}
444+
fs.rootMapToReal.WalkPrefix(prefix, walkFn)
455445

456446
return roots
457447
}
458448

459449
func (fs *RootMappingFs) getAncestors(prefix string) []keyRootMappings {
460450
var roots []keyRootMappings
461-
fs.rootMapToReal.WalkPath(prefix, func(s string, v any) bool {
451+
var walkFn radix.WalkFn[[]*RootMapping] = func(s string, v []*RootMapping) (radix.WalkFlag, []*RootMapping, error) {
462452
if strings.HasPrefix(prefix, s+filepathSeparator) {
463453
roots = append(roots, keyRootMappings{
464454
key: s,
465-
roots: v.([]*RootMapping),
455+
roots: v,
466456
})
467457
}
468-
return false
469-
})
458+
return radix.WalkContinue, nil, nil
459+
}
460+
fs.rootMapToReal.WalkPath(prefix, walkFn)
470461

471462
return roots
472463
}
@@ -593,7 +584,7 @@ func (rfs *RootMappingFs) collectDirEntries(prefix string) ([]iofs.DirEntry, err
593584

594585
// Next add any file mounts inside the given directory.
595586
prefixInside := prefix + filepathSeparator
596-
rfs.rootMapToReal.WalkPrefix(prefixInside, func(s string, v any) bool {
587+
var walkFn radix.WalkFn[[]*RootMapping] = func(s string, rms []*RootMapping) (radix.WalkFlag, []*RootMapping, error) {
597588
if (strings.Count(s, filepathSeparator) - level) != 1 {
598589
// This directory is not part of the current, but we
599590
// need to include the first name part to make it
@@ -603,7 +594,7 @@ func (rfs *RootMappingFs) collectDirEntries(prefix string) ([]iofs.DirEntry, err
603594
name := parts[0]
604595

605596
if seen[name] {
606-
return false
597+
return radix.WalkContinue, nil, nil
607598
}
608599
seen[name] = true
609600
opener := func() (afero.File, error) {
@@ -613,10 +604,9 @@ func (rfs *RootMappingFs) collectDirEntries(prefix string) ([]iofs.DirEntry, err
613604
fi := newDirNameOnlyFileInfo(name, nil, opener)
614605
fis = append(fis, fi)
615606

616-
return false
607+
return radix.WalkContinue, nil, nil
617608
}
618609

619-
rms := v.([]*RootMapping)
620610
for _, rm := range rms {
621611
name := filepath.Base(rm.From)
622612
if seen[name] {
@@ -630,8 +620,9 @@ func (rfs *RootMappingFs) collectDirEntries(prefix string) ([]iofs.DirEntry, err
630620
fis = append(fis, fi)
631621
}
632622

633-
return false
634-
})
623+
return radix.WalkContinue, nil, nil
624+
}
625+
rfs.rootMapToReal.WalkPrefix(prefixInside, walkFn)
635626

636627
// Finally add any ancestor dirs with files in this directory.
637628
ancestors := rfs.getAncestors(prefix)

‎hugolib/content_map.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -241,8 +241,8 @@ func (m *pageMap) AddFi(fi hugofs.FileMetaInfo, buildConfig *BuildCfg) (pageSour
241241
key := pi.Base()
242242
tree := m.treeResources
243243

244-
commit := tree.Lock(true)
245-
defer commit()
244+
tree.Lock(true)
245+
defer tree.Unlock(true)
246246

247247
if pi.IsContent() {
248248
pm, err := h.newPageMetaSourceFromFile(fi)

0 commit comments

Comments
 (0)