Skip to content

Commit a91cc6c

Browse files
authored
Speed up unit tests by removing duplicate work (#3216)
* Add utility to find tags in go modules * Modify main Unix target on GH actions to use new findtaggedmodules * Use temp file for passing list of tagged modules instead of setting bash specific flags * Use tempfile instead of hardcoded path to allow multiple runs on same host * Standardize CI make target for non-unix platforms * Use bash for Windows workflow * Remove -race flag from 386 arch since it's not supported * Use Unix style separators even on Windows since we're using bash shell * Use eachmodule as runner for build-tagged-modules target * Move back to default Windows runner
1 parent 7cdc7c8 commit a91cc6c

File tree

3 files changed

+155
-3
lines changed

3 files changed

+155
-3
lines changed

‎.github/workflows/go.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@ jobs:
7171
run: go install golang.org/x/lint/golint@latest
7272

7373
- name: Test
74-
run: make unit
74+
run: make ci-test-no-generate-no-race
7575

7676
windows-tests:
7777
name: Windows SDK Tests
@@ -97,4 +97,4 @@ jobs:
9797
run: ./ci-find-smithy-go.sh
9898

9999
- name: Test
100-
run: make vet build unit-test
100+
run: make ci-test-no-generate

‎Makefile‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -291,6 +291,19 @@ build-modules-%:
291291
&& go run . -p $(subst _,/,$(subst build-modules-,,$@)) ${EACHMODULE_FLAGS} \
292292
"go test ${BUILD_TAGS} ${RUN_NONE} ./..."
293293

294+
TEMP_FILE := $(shell mktemp)
295+
build-tagged-modules:
296+
@# Compiles modules tagged with BUILD_TAGS.
297+
@# This just ensures the modules compile correctly and doesn't actually produce any executable.
298+
@# This also runs "vet" analyzers on the module
299+
cd ./internal/repotools/cmd/findtaggedmodules \
300+
&& go run . $(BUILD_TAGS) > "$(TEMP_FILE)";
301+
cd ./internal/repotools/cmd/eachmodule \
302+
&& while read module; do \
303+
echo "Testing module: $$module"; \
304+
(go run . -p "$$module" ${EACHMODULE_FLAGS} "go test ${BUILD_TAGS} ${RUN_NONE} -vet=all ./...") || { echo "Tests failed for module: $$module"; exit 1; }; \
305+
done < "$(TEMP_FILE)";
306+
294307
go-build-modules-%:
295308
@# build command that uses the pattern to define the root path that the
296309
@# module testing will start from. Strips off the "build-modules-" and
@@ -315,6 +328,16 @@ test-race-modules-%:
315328
&& go run . -p $(subst _,/,$(subst test-race-modules-,,$@)) ${EACHMODULE_FLAGS} \
316329
"go test -timeout=2m ${UNIT_TEST_TAGS} -race -cpu=4 ./..."
317330

331+
test-race-vet-modules-%:
332+
@# Test command that uses the pattern to define the root path that the
333+
@# module testing will start from. Strips off the "test-race-modules-" and
334+
@# replaces all "_" with "/".
335+
@#
336+
@# e.g. test-race-modules-internal_protocoltest
337+
cd ./internal/repotools/cmd/eachmodule \
338+
&& go run . -p $(subst _,/,$(subst test-race-vet-modules-,,$@)) ${EACHMODULE_FLAGS} \
339+
"go test -timeout=2m ${UNIT_TEST_TAGS} -race -vet=all -cpu=4 ./..."
340+
318341
test-modules-%:
319342
@# Test command that uses the pattern to define the root path that the
320343
@# module testing will start from. Strips off the "test-modules-" and
@@ -373,7 +396,8 @@ api-diff-modules-%:
373396
.PHONY: ci-test ci-test-no-generate ci-test-generate-validate
374397

375398
ci-test: generate unit-race ci-test-generate-validate
376-
ci-test-no-generate: unit-race
399+
ci-test-no-generate: lint build-tagged-modules test-race-vet-modules-.
400+
ci-test-no-generate-no-race: lint build-tagged-modules test-modules-.
377401

378402
ci-test-generate-validate:
379403
@echo "CI test validate no generated code changes"
Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
/*
2+
findtaggedmodules finds modules that contain given go build tags.
3+
4+
Given a directory and a list of tags, the command looks at all .go files
5+
inside the directory to find any that contain the tags. Once found, it
6+
finds which module they belong to and returns all directories where any
7+
of the tags are found.
8+
9+
It always returns paths Unix-style with a forward slash ("/") as separator
10+
11+
Usage:
12+
13+
findtagmodules -tags [tag1,tag2] [-p path]
14+
15+
The flags are:
16+
17+
-tags
18+
List of tags to look for, passed as "tag1,tag2"
19+
-p
20+
Root path to search from. Default to the repo root
21+
*/
22+
package main
23+
24+
import (
25+
"flag"
26+
"fmt"
27+
"io/fs"
28+
"log"
29+
"os"
30+
"path/filepath"
31+
"strings"
32+
33+
repotools "github.com/awslabs/aws-go-multi-module-repository-tools"
34+
)
35+
36+
var (
37+
tags string
38+
rootPath string
39+
)
40+
41+
func init() {
42+
flag.StringVar(&tags, "tags", "", "Comma-separated list of build tags to search for")
43+
flag.StringVar(&rootPath, "p", "", "Root path to search from (defaults to repo root)")
44+
}
45+
46+
func main() {
47+
flag.Parse()
48+
49+
if tags == "" {
50+
log.Fatal("must specify -tags")
51+
}
52+
53+
targetTags := strings.Split(tags, ",")
54+
for i := range targetTags {
55+
targetTags[i] = strings.TrimSpace(targetTags[i])
56+
}
57+
58+
repoRoot, err := repotools.FindRepoRoot(rootPath)
59+
if err != nil {
60+
log.Fatalf("failed to find repo root: %v", err)
61+
}
62+
63+
if rootPath == "" {
64+
rootPath = repoRoot
65+
} else {
66+
rootPath = filepath.Join(repoRoot, rootPath)
67+
}
68+
69+
var boots repotools.Boots
70+
71+
if err := filepath.Walk(rootPath, boots.Walk); err != nil {
72+
log.Fatalf("failed to walk directory: %v", err)
73+
}
74+
75+
for _, modPath := range boots.Modules() {
76+
if modPath == rootPath {
77+
continue
78+
}
79+
found, err := hasAnyTag(modPath, targetTags)
80+
if err != nil {
81+
log.Fatalf("found an error searching for tags: %v", err)
82+
}
83+
if found {
84+
relPath, err := filepath.Rel(repoRoot, modPath)
85+
// Use Unix style path
86+
relPath = filepath.ToSlash(relPath)
87+
if err != nil {
88+
fmt.Println(modPath)
89+
} else {
90+
fmt.Println(relPath)
91+
}
92+
}
93+
}
94+
}
95+
96+
func hasAnyTag(modPath string, targetTags []string) (bool, error) {
97+
found := false
98+
err := filepath.WalkDir(modPath, func(path string, info fs.DirEntry, err error) error {
99+
if err != nil || !strings.HasSuffix(path, ".go") {
100+
return nil
101+
}
102+
103+
file, err := os.Open(path)
104+
if err != nil {
105+
return err
106+
}
107+
defer file.Close()
108+
109+
// Read first two lines only
110+
buf := make([]byte, 200)
111+
n, _ := file.Read(buf)
112+
lines := strings.Split(string(buf[:n]), "\n")
113+
114+
for i := 0; i < 2 && i < len(lines); i++ {
115+
line := strings.TrimSpace(lines[i])
116+
if strings.HasPrefix(line, "//go:build") || strings.HasPrefix(line, "// +build") {
117+
for _, tag := range targetTags {
118+
if strings.Contains(line, tag) {
119+
found = true
120+
return nil
121+
}
122+
}
123+
}
124+
}
125+
return nil
126+
})
127+
return found, err
128+
}

0 commit comments

Comments
 (0)