testing: Revise usage of b.N and b.Loop() in benchmarks

Author: bep

common/hreflect/helpers_test.go

@@ -152,17 +152,18 @@ func TestIndirectInterface(t *testing.T) {
 }
 
 func BenchmarkIsContextType(b *testing.B) {
+	const size = 1000
 	type k string
 	b.Run("value", func(b *testing.B) {
 		ctx := context.Background()
-		ctxs := make([]reflect.Type, b.N)
-		for i := 0; b.Loop(); i++ {
+		ctxs := make([]reflect.Type, size)
+		for i := range size {
 			ctxs[i] = reflect.TypeOf(context.WithValue(ctx, k("key"), i))
 		}
 
-		b.ResetTimer()
-		for i := 0; i < b.N; i++ {
-			if !IsContextType(ctxs[i]) {
+		for i := 0; b.Loop(); i++ {
+			idx := i % size
+			if !IsContextType(ctxs[idx]) {
 				b.Fatal("not context")
 			}
 		}

common/hstrings/strings_test.go

@@ -71,6 +71,7 @@ func TestUniqueStringsSorted(t *testing.T) {
 	c.Assert(UniqueStringsSorted(nil), qt.IsNil)
 }
 
+// Note that these cannot use b.Loop() because of golang/go#27217.
 func BenchmarkUniqueStrings(b *testing.B) {
 	input := []string{"a", "b", "d", "e", "d", "h", "a", "i"}
 
@@ -84,15 +85,14 @@ func BenchmarkUniqueStrings(b *testing.B) {
 	})
 
 	b.Run("Reuse slice", func(b *testing.B) {
-		b.StopTimer()
 		inputs := make([][]string, b.N)
-		for i := 0; b.Loop(); i++ {
+		for i := 0; i < b.N; i++ {
 			inputc := make([]string, len(input))
 			copy(inputc, input)
 			inputs[i] = inputc
 		}
-		b.StartTimer()
-		for i := 0; b.Loop(); i++ {
+		b.ResetTimer()
+		for i := 0; i < b.N; i++ {
 			inputc := inputs[i]
 
 			result := UniqueStringsReuse(inputc)
@@ -103,14 +103,13 @@ func BenchmarkUniqueStrings(b *testing.B) {
 	})
 
 	b.Run("Reuse slice sorted", func(b *testing.B) {
-		b.StopTimer()
 		inputs := make([][]string, b.N)
 		for i := 0; i < b.N; i++ {
 			inputc := make([]string, len(input))
 			copy(inputc, input)
 			inputs[i] = inputc
 		}
-		b.StartTimer()
+		b.ResetTimer()
 		for i := 0; i < b.N; i++ {
 			inputc := inputs[i]
 

compare/compare_strings_test.go

@@ -63,6 +63,7 @@ func TestLexicographicSort(t *testing.T) {
 	c.Assert(s, qt.DeepEquals, []string{"A", "b", "Ba", "ba", "ba", "Bz"})
 }
 
+// // Note that this cannot use b.Loop() because of golang/go#27217.
 func BenchmarkStringSort(b *testing.B) {
 	prototype := []string{"b", "Bz", "zz", "ba", "αβδ αβδ αβδ", "A", "Ba", "ba", "nnnnasdfnnn", "AAgæåz", "αβδC"}
 	b.Run("LessStrings", func(b *testing.B) {

hugolib/cascade_test.go

@@ -34,15 +34,11 @@ func BenchmarkCascade(b *testing.B) {
 		langs := allLangs[0:i]
 		b.Run(fmt.Sprintf("langs-%d", len(langs)), func(b *testing.B) {
 			c := qt.New(b)
-			b.StopTimer()
-			builders := make([]*sitesBuilder, b.N)
-			for i := 0; b.Loop(); i++ {
-				builders[i] = newCascadeTestBuilder(b, langs)
-			}
-			b.StartTimer()
 
-			for i := 0; b.Loop(); i++ {
-				builder := builders[i]
+			for b.Loop() {
+				b.StopTimer()
+				builder := newCascadeTestBuilder(b, langs)
+				b.StartTimer()
 				err := builder.BuildE(BuildCfg{})
 				c.Assert(err, qt.IsNil)
 				first := builder.H.Sites[0]
@@ -75,16 +71,13 @@ kind = '{section,term}'
 			T:           b,
 			TxtarString: files,
 		}
-		builders := make([]*IntegrationTestBuilder, b.N)
-
-		for i := range builders {
-			builders[i] = NewIntegrationTestBuilder(cfg)
-		}
-
 		b.ResetTimer()
 
-		for i := 0; i < b.N; i++ {
-			builders[i].Build()
+		for b.Loop() {
+			b.StopTimer()
+			builder := NewIntegrationTestBuilder(cfg)
+			b.StartTimer()
+			builder.Build()
 		}
 	})
 }

hugolib/hugo_smoke_test.go

@@ -743,14 +743,12 @@ func BenchmarkBaseline(b *testing.B) {
 		T:           b,
 		TxtarString: benchmarkBaselineFiles(false),
 	}
-	builders := make([]*IntegrationTestBuilder, b.N)
 
-	for i := range builders {
-		builders[i] = NewIntegrationTestBuilder(cfg)
-	}
-
-	for i := 0; i < b.N; i++ {
-		builders[i].Build()
+	for b.Loop() {
+		b.StopTimer()
+		builder := NewIntegrationTestBuilder(cfg)
+		b.StartTimer()
+		builder.Build()
 	}
 }
 

hugolib/page_permalink_test.go

@@ -64,7 +64,6 @@ func TestPermalink(t *testing.T) {
 	}
 
 	for i, test := range tests {
-		test := test
 		t.Run(fmt.Sprintf("%s-%d", test.file, i), func(t *testing.T) {
 			t.Parallel()
 			c := qt.New(t)

hugolib/pagecollections_test.go

@@ -37,37 +37,35 @@ categories:
 `
 
 func BenchmarkGetPage(b *testing.B) {
-	var (
-		cfg, fs = newTestCfg()
-		r       = rand.New(rand.NewSource(time.Now().UnixNano()))
-	)
+	cfg, fs := newTestCfg()
 
 	configs, err := loadTestConfigFromProvider(cfg)
 	if err != nil {
 		b.Fatal(err)
 	}
 
-	for i := range 10 {
+	const size = 10
+
+	for i := range size {
 		for j := range 100 {
 			writeSource(b, fs, filepath.Join("content", fmt.Sprintf("sect%d", i), fmt.Sprintf("page%d.md", j)), "CONTENT")
 		}
 	}
 
 	s := buildSingleSite(b, deps.DepsCfg{Fs: fs, Configs: configs}, BuildCfg{SkipRender: true})
 
-	pagePaths := make([]string, b.N)
-
-	for i := 0; b.Loop(); i++ {
-		pagePaths[i] = fmt.Sprintf("sect%d", r.Intn(10))
+	pagePaths := make([]string, size)
+	for i := range size {
+		pagePaths[i] = fmt.Sprintf("sect%d", i)
 	}
 
-	for i := 0; i < b.N; i++ {
+	for i := 0; b.Loop(); i++ {
 		home, _ := s.getPage(nil, "/")
 		if home == nil {
 			b.Fatal("Home is nil")
 		}
 
-		p, _ := s.getPage(nil, pagePaths[i])
+		p, _ := s.getPage(nil, pagePaths[i%size])
 		if p == nil {
 			b.Fatal("Section is nil")
 		}
@@ -113,20 +111,21 @@ func TestBenchmarkGetPageRegular(t *testing.T) {
 
 func BenchmarkGetPageRegular(b *testing.B) {
 	r := rand.New(rand.NewSource(time.Now().UnixNano()))
+	const size = 100
 
 	b.Run("From root", func(b *testing.B) {
 		s := createGetPageRegularBenchmarkSite(b)
 		c := qt.New(b)
 
-		pagePaths := make([]string, b.N)
+		pagePaths := make([]string, size)
 
-		for i := 0; i < b.N; i++ {
-			pagePaths[i] = path.Join(fmt.Sprintf("/sect%d", r.Intn(10)), fmt.Sprintf("page%d.md", r.Intn(100)))
+		for i := range size {
+			pagePaths[i] = path.Join(fmt.Sprintf("/sect%d", r.Intn(10)), fmt.Sprintf("page%d.md", i))
 		}
 
 		b.ResetTimer()
-		for i := 0; i < b.N; i++ {
-			page, _ := s.getPage(nil, pagePaths[i])
+		for i := 0; b.Loop(); i++ {
+			page, _ := s.getPage(nil, pagePaths[i%size])
 			c.Assert(page, qt.Not(qt.IsNil))
 		}
 	})
@@ -136,17 +135,15 @@ func BenchmarkGetPageRegular(b *testing.B) {
 		c := qt.New(b)
 		allPages := s.RegularPages()
 
-		pagePaths := make([]string, b.N)
-		pages := make([]page.Page, b.N)
+		pagePaths := make([]string, size)
+		pages := allPages[:size]
 
-		for i := 0; i < b.N; i++ {
-			pagePaths[i] = fmt.Sprintf("page%d.md", r.Intn(100))
-			pages[i] = allPages[r.Intn(len(allPages)/3)]
+		for i := range size {
+			pagePaths[i] = fmt.Sprintf("page%d.md", i)
 		}
 
-		b.ResetTimer()
-		for i := 0; i < b.N; i++ {
-			page, _ := s.getPage(pages[i], pagePaths[i])
+		for i := 0; b.Loop(); i++ {
+			page, _ := s.getPage(pages[i%size], pagePaths[i%size])
 			c.Assert(page, qt.Not(qt.IsNil))
 		}
 	})

hugolib/rebuild_test.go

@@ -1787,22 +1787,15 @@ func BenchmarkRebuildContentFileChange(b *testing.B) {
 		T:           b,
 		TxtarString: files,
 		Running:     true,
-		// Verbose:     true,
-		// LogLevel: logg.LevelInfo,
 	}
-	builders := make([]*IntegrationTestBuilder, b.N)
 
-	for i := range builders {
-		builders[i] = NewIntegrationTestBuilder(cfg)
-		builders[i].Build()
-	}
-
-	for i := 0; i < b.N; i++ {
-		bb := builders[i]
+	for b.Loop() {
+		b.StopTimer()
+		bb := NewIntegrationTestBuilder(cfg).Build()
+		b.StartTimer()
 		bb.EditFileReplaceFunc("content/mysect/p123/index.md", func(s string) string {
 			return s + "... Edited"
 		}).Build()
-		// fmt.Println(bb.LogString())
 	}
 }
 

hugolib/shortcode_test.go

@@ -242,6 +242,7 @@ CSV: {{< myShort >}}
 	)
 }
 
+// Note that this cannot use b.Loop() because of golang/go#27217.
 func BenchmarkReplaceShortcodeTokens(b *testing.B) {
 	type input struct {
 		in           []byte
@@ -263,7 +264,7 @@ func BenchmarkReplaceShortcodeTokens(b *testing.B) {
 
 	cnt := 0
 	in := make([]input, b.N*len(data))
-	for b.Loop() {
+	for i := 0; i < b.N; i++ {
 		for _, this := range data {
 			replacements := make(map[string]shortcodeRenderer)
 			for k, v := range this.replacements {
@@ -279,7 +280,8 @@ func BenchmarkReplaceShortcodeTokens(b *testing.B) {
 
 	cnt = 0
 	ctx := context.Background()
-	for i := 0; b.Loop(); i++ {
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
 		for j := range data {
 			currIn := in[cnt]
 			cnt++
@@ -335,14 +337,12 @@ title: "Markdown Shortcode"
 		T:           b,
 		TxtarString: files,
 	}
-	builders := make([]*IntegrationTestBuilder, b.N)
 
-	for i := range builders {
-		builders[i] = NewIntegrationTestBuilder(cfg)
-	}
-
-	for i := 0; i < b.N; i++ {
-		builders[i].Build()
+	for b.Loop() {
+		b.StopTimer()
+		builder := NewIntegrationTestBuilder(cfg)
+		b.StartTimer()
+		builder.Build()
 	}
 }