Skip to content

Commit 26f31ff

Browse files
committed
hugolib: Improve performance of content trees with many sections
Hugo's build process is roughly divided into three steps: 1. Process content (walk file system and insert source nodes into content tree) 2. Assemble content (assemble pages and resources according to sites matrix) 3. Render content In #13679 we consolidated the page creation logic into one place (the assemble step). This made it much simpler to reason about, but it lost us some performance esp. in big content trees. This commit re-introduces parallelization in the first step in the assemble step by handling each top level section in its own goroutine. This gives significant performance improvements for content trees with many sections. Compared to master: ``` AssembleDeepSiteWithManySections/depth=1/sectionsPerLevel=6/pagesPerSection=100-10 19.26m ± ∞ ¹ 14.54m ± ∞ ¹ -24.52% (p=0.029 n=4) AssembleDeepSiteWithManySections/depth=2/sectionsPerLevel=2/pagesPerSection=100-10 19.74m ± ∞ ¹ 16.45m ± ∞ ¹ -16.71% (p=0.029 n=4) AssembleDeepSiteWithManySections/depth=2/sectionsPerLevel=6/pagesPerSection=100-10 106.18m ± ∞ ¹ 71.23m ± ∞ ¹ -32.91% (p=0.029 n=4) AssembleDeepSiteWithManySections/depth=3/sectionsPerLevel=2/pagesPerSection=100-10 38.85m ± ∞ ¹ 30.47m ± ∞ ¹ -21.59% (p=0.029 n=4) ```
1 parent bca171b commit 26f31ff

File tree

10 files changed

+257
-103
lines changed

10 files changed

+257
-103
lines changed

‎common/collections/stack.go‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,13 @@
1313

1414
package collections
1515

16-
import "slices"
16+
import (
17+
"iter"
18+
"slices"
19+
"sync"
1720

18-
import "sync"
21+
"github.com/gohugoio/hugo/common/hiter"
22+
)
1923

2024
// Stack is a simple LIFO stack that is safe for concurrent use.
2125
type Stack[T any] struct {
@@ -60,6 +64,11 @@ func (s *Stack[T]) Len() int {
6064
return len(s.items)
6165
}
6266

67+
// All returns all items in the stack, from bottom to top.
68+
func (s *Stack[T]) All() iter.Seq2[int, T] {
69+
return hiter.Lock2(slices.All(s.items), s.mu.RLock, s.mu.RUnlock)
70+
}
71+
6372
func (s *Stack[T]) Drain() []T {
6473
s.mu.Lock()
6574
defer s.mu.Unlock()

‎common/hiter/iter.go‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,3 +38,29 @@ func Concat2[K, V any](seqs ...iter.Seq2[K, V]) iter.Seq2[K, V] {
3838
}
3939
}
4040
}
41+
42+
// Lock returns an iterator that locks before iterating and unlocks after.
43+
func Lock[V any](seq iter.Seq[V], lock, unlock func()) iter.Seq[V] {
44+
return func(yield func(V) bool) {
45+
lock()
46+
defer unlock()
47+
for e := range seq {
48+
if !yield(e) {
49+
return
50+
}
51+
}
52+
}
53+
}
54+
55+
// Lock2 returns an iterator that locks before iterating and unlocks after.
56+
func Lock2[K, V any](seq iter.Seq2[K, V], lock, unlock func()) iter.Seq2[K, V] {
57+
return func(yield func(K, V) bool) {
58+
lock()
59+
defer unlock()
60+
for k, v := range seq {
61+
if !yield(k, v) {
62+
return
63+
}
64+
}
65+
}
66+
}

0 commit comments

Comments
 (0)