Skip to content

Commit 70175d3

Browse files
dgageotdgageot
authored
Merge pull request #1812 from gtardif/binary_e2e_tests
Binary e2e tests
2 parents e2ae58b + d7a5e4c commit 70175d3

6 files changed

Lines changed: 154 additions & 23 deletions

File tree

‎.github/workflows/ci.yml‎

Lines changed: 14 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -20,10 +20,10 @@ jobs:
2020
runs-on: ubuntu-latest
2121
steps:
2222
- name: Checkout
23-
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
23+
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
2424

2525
- name: Set up Go
26-
uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 # v6.2.0
26+
uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 # v6.2.0
2727
with:
2828
go-version: "1.26.0"
2929
cache: true
@@ -43,10 +43,10 @@ jobs:
4343
runs-on: ubuntu-latest
4444
steps:
4545
- name: Checkout
46-
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
46+
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
4747

4848
- name: Set up Go
49-
uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 # v6.2.0
49+
uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 # v6.2.0
5050
with:
5151
go-version: "1.26.0"
5252
cache: true
@@ -57,16 +57,18 @@ jobs:
5757
version: 3.x
5858

5959
- name: Run tests
60-
run: task test
60+
run: |
61+
task test
62+
task test-binary
6163
6264
license-check:
6365
runs-on: ubuntu-latest
6466
steps:
6567
- name: Checkout
66-
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
68+
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
6769

6870
- name: Set up Go
69-
uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 # v6.2.0
71+
uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 # v6.2.0
7072
with:
7173
go-version: "1.26.0"
7274
cache: true
@@ -82,10 +84,10 @@ jobs:
8284
runs-on: ubuntu-latest
8385
steps:
8486
- name: Checkout
85-
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
87+
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
8688

8789
- name: Set up Go
88-
uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 # v6.2.0
90+
uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 # v6.2.0
8991
with:
9092
go-version: "1.26.0"
9193
cache: true
@@ -106,7 +108,7 @@ jobs:
106108
runs-on: ubuntu-latest
107109
steps:
108110
- name: Checkout
109-
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
111+
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
110112

111113
- name: Hub login
112114
if: github.event_name != 'pull_request'
@@ -116,11 +118,11 @@ jobs:
116118
password: ${{ secrets.DOCKERPUBLICBOT_WRITE_PAT }}
117119

118120
- name: Set up Docker Buildx
119-
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
121+
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
120122

121123
- name: Docker metadata
122124
id: meta
123-
uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5.10.0
125+
uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5.10.0
124126
with:
125127
images: docker/cagent
126128
tags: |

‎Taskfile.yml‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ tasks:
2525
desc: Build the application binary
2626
cmds:
2727
- go build -ldflags "{{.LDFLAGS}}" -o {{.BUILD_DIR}}/{{.BINARY_NAME}} {{.MAIN_PKG}}
28-
- ln -sf {{.USER_WORKING_DIR}}/{{.BUILD_DIR}}/{{.BINARY_NAME}} {{.HOME}}/bin/{{.BINARY_NAME}}
28+
- '{{if ne .CI "true"}}ln -sf {{.USER_WORKING_DIR}}/{{.BUILD_DIR}}/{{.BINARY_NAME}} {{.HOME}}/bin/{{.BINARY_NAME}}{{end}}'
2929
sources:
3030
- "{{.GO_SOURCES}}"
3131
- "**/*.template"
@@ -40,7 +40,9 @@ tasks:
4040
desc: Deploy the docker agent cli-plugin
4141
deps: ["build"]
4242
cmds:
43+
- mkdir -p ~/.docker/cli-plugins
4344
- cp "{{.BUILD_DIR}}/{{.BINARY_NAME}}" ~/.docker/cli-plugins/{{.CLI_PLUGIN_BINARY_NAME}}
45+
- cp "{{.BUILD_DIR}}/{{.BINARY_NAME}}" {{.BUILD_DIR}}/{{.CLI_PLUGIN_BINARY_NAME}}
4446

4547
lint:
4648
desc: Run golangci-lint
@@ -63,7 +65,12 @@ tasks:
6365
test:
6466
aliases: [t]
6567
desc: Run tests
66-
cmd: CAGENT_MODELS_GATEWAY= OPENAI_API_KEY= ANTHROPIC_API_KEY= GOOGLE_API_KEY= GOOGLE_GENAI_USE_VERTEXAI= MISTRAL_API_KEY= GITHUB_TOKEN= go test {{.CLI_ARGS}} ./...
68+
cmd: CAGENT_MODELS_GATEWAY= OPENAI_API_KEY= ANTHROPIC_API_KEY= GOOGLE_API_KEY= GOOGLE_GENAI_USE_VERTEXAI= MISTRAL_API_KEY= GITHUB_TOKEN= go test {{.CLI_ARGS}} ./...
69+
70+
test-binary:
71+
deps: ["deploy-local"]
72+
desc: Run tests on build binary
73+
cmd: CAGENT_MODELS_GATEWAY= OPENAI_API_KEY= ANTHROPIC_API_KEY= GOOGLE_API_KEY= GOOGLE_GENAI_USE_VERTEXAI= MISTRAL_API_KEY= GITHUB_TOKEN= go test -tags=binary_required {{.CLI_ARGS}} ./e2e/binary/...
6774

6875
build-local:
6976
desc: Build binaries for local host platform

‎cmd/root/root.go‎

Lines changed: 17 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -186,15 +186,7 @@ We collect anonymous usage data to help improve cagent. To disable:
186186
}
187187
return nil
188188
}
189-
if rootCmd.RunE != nil {
190-
originalRunE := rootCmd.RunE
191-
rootCmd.RunE = func(cmd *cobra.Command, args []string) error {
192-
if err := originalRunE(cmd, args); err != nil {
193-
return processErr(cmd.Context(), err, stderr, rootCmd)
194-
}
195-
return nil
196-
}
197-
}
189+
setErrorHandlingRecursive(rootCmd, processErr)
198190
return rootCmd
199191
}, metadata.Metadata{
200192
SchemaVersion: "0.1.0",
@@ -212,6 +204,22 @@ func setContextRecursive(ctx context.Context, cmd *cobra.Command) {
212204
}
213205
}
214206

207+
func setErrorHandlingRecursive(cmd *cobra.Command, processErr func(context.Context, error, io.Writer, *cobra.Command) error) {
208+
if cmd.RunE != nil {
209+
originalRunE := cmd.RunE
210+
cmd.RunE = func(cmd *cobra.Command, args []string) error {
211+
if err := originalRunE(cmd, args); err != nil {
212+
return processErr(cmd.Context(), err, cmd.ErrOrStderr(), cmd)
213+
}
214+
return nil
215+
}
216+
}
217+
218+
for _, child := range cmd.Commands() {
219+
setErrorHandlingRecursive(child, processErr)
220+
}
221+
}
222+
215223
// defaultToRun prepends "run" to the argument list when no subcommand is
216224
// specified so that bare "cagent" (or "cagent --debug", etc.) launches the
217225
// default agent. Help flags (--help / -h) are left alone.

‎e2e/binary/binary_test.go‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
//go:build binary_required
2+
// +build binary_required
3+
4+
package binary
5+
6+
import (
7+
"testing"
8+
9+
"github.com/stretchr/testify/require"
10+
)
11+
12+
const binDir = "../../bin"
13+
14+
func TestHelpInAllExecMode(t *testing.T) {
15+
t.Run("cli plugin help", func(t *testing.T) {
16+
res, err := Exec("docker", "agent", "help")
17+
require.NoError(t, err)
18+
require.Contains(t, res.Stdout, "docker agent run ./agent.yaml")
19+
})
20+
21+
t.Run("cagent help", func(t *testing.T) {
22+
res, err := Exec(binDir+"/cagent", "help")
23+
require.NoError(t, err)
24+
require.Contains(t, res.Stdout, "cagent run ./agent.yaml")
25+
})
26+
27+
t.Run("docker-agent help", func(t *testing.T) {
28+
res, err := Exec(binDir+"/docker-agent", "help")
29+
require.NoError(t, err)
30+
require.Contains(t, res.Stdout, "docker-agent run ./agent.yaml")
31+
})
32+
}
33+
34+
func TestExecMissingKeys(t *testing.T) {
35+
t.Run("cli plugin exec", func(t *testing.T) {
36+
res, err := Exec("docker", "agent", "run", "--exec", "./test-agent.yaml")
37+
require.Error(t, err)
38+
require.Contains(t, res.Stderr, "environment variables must be set")
39+
require.Contains(t, res.Stderr, "OPENAI_API_KEY")
40+
})
41+
42+
t.Run("cagent exec", func(t *testing.T) {
43+
res, err := Exec(binDir+"/cagent", "run", "--exec", "./test-agent.yaml")
44+
require.Error(t, err)
45+
require.Contains(t, res.Stderr, "environment variables must be set")
46+
require.Contains(t, res.Stderr, "OPENAI_API_KEY")
47+
})
48+
49+
t.Run("docker-agent exec", func(t *testing.T) {
50+
res, err := Exec(binDir+"/docker-agent", "run", "--exec", "./test-agent.yaml")
51+
require.Error(t, err)
52+
require.Contains(t, res.Stderr, "environment variables must be set")
53+
require.Contains(t, res.Stderr, "OPENAI_API_KEY")
54+
})
55+
}

‎e2e/binary/shellout.go‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
package binary
2+
3+
import (
4+
"bytes"
5+
"context"
6+
"errors"
7+
"fmt"
8+
"os"
9+
"os/exec"
10+
"strings"
11+
)
12+
13+
// CmdResult output of a command, including stdout and stderr
14+
type CmdResult struct {
15+
Stdout string
16+
Stderr string
17+
}
18+
19+
func ExecWithContext(ctx context.Context, cmd string, args ...string) (CmdResult, error) {
20+
return ExecWithContextInDir(ctx, "", cmd, args, nil)
21+
}
22+
23+
func Exec(cmd string, args ...string) (CmdResult, error) {
24+
return ExecWithContextInDir(context.Background(), "", cmd, args, nil)
25+
}
26+
27+
func ExecWithContextAndEnv(ctx context.Context, env []string, cmd string, args ...string) (CmdResult, error) {
28+
return ExecWithContextInDir(ctx, "", cmd, args, env)
29+
}
30+
31+
func ExecWithContextInDir(ctx context.Context, dir, cmd string, args, env []string) (CmdResult, error) {
32+
command := exec.CommandContext(ctx, cmd, args...)
33+
command.Dir = dir
34+
command.Env = append(os.Environ(), env...)
35+
res, err := runCmd(command)
36+
if err != nil {
37+
return res, fmt.Errorf("executing '%s %s' : %s: %s", cmd, strings.Join(args, " "), err.Error(), res.Stdout+"\n"+res.Stderr)
38+
}
39+
return res, nil
40+
}
41+
42+
func runCmd(c *exec.Cmd) (CmdResult, error) {
43+
if c.Stdout != nil {
44+
return CmdResult{}, errors.New("exec: Stdout already set")
45+
}
46+
if c.Stderr != nil {
47+
return CmdResult{}, errors.New("exec: Stderr already set")
48+
}
49+
var outBuffer bytes.Buffer
50+
var errBuffer bytes.Buffer
51+
c.Stdout = &outBuffer
52+
c.Stderr = &errBuffer
53+
err := c.Run()
54+
return CmdResult{outBuffer.String(), errBuffer.String()}, err
55+
}

‎e2e/binary/test-agent.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
version: "2"
2+
agents:
3+
root:
4+
model: openai/gpt-4o

0 commit comments

Comments
 (0)