diff --git a/dev/breeze/src/airflow_breeze/commands/kubernetes_commands.py b/dev/breeze/src/airflow_breeze/commands/kubernetes_commands.py index ac5bbbfad045b..c064e7400b3f4 100644 --- a/dev/breeze/src/airflow_breeze/commands/kubernetes_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/kubernetes_commands.py @@ -2593,7 +2593,9 @@ def _lang_sdk_build_go_bundle( go_example's go.mod ``replace``s go-sdk by relative path, so the build runs in a scratch workspace mirroring the repo layout with ``upstream_go_sdk`` at ``/go-sdk``, - letting the unmodified directive resolve against the upstream copy. + letting the unmodified directive resolve against the upstream copy. The scratch go_example is + re-tidied before packing so its go.sum reconciles to that upstream go-sdk (which may differ from + the in-repo go-sdk its committed go.sum was tidied against). """ go_dir = staging / "go-artifacts" go_dir.mkdir(parents=True, exist_ok=True) @@ -2611,12 +2613,20 @@ def _lang_sdk_build_go_bundle( # CGO_ENABLED=0 yields a fully static binary that runs on the stock worker. The package built is # the current dir (".") because go_example is its own module. + # + # go_example's go.sum is tidied against the in-repo go-sdk, but the bundle is built against the + # upstream-main go-sdk copied in above. When a branch changes go-sdk's dependency graph those two + # go-sdks differ, and Go refuses to build on the resulting go.sum drift. Re-tidy the scratch copy + # first so the build reconciles to whichever go-sdk it is actually compiled against; the committed + # go.sum is untouched and stays guarded by the check-go-example-mod-tidy prek hook. if native: get_console(output=output).print("[info]Building Go bundle with the host Go toolchain") + go_env = {**os.environ, "CGO_ENABLED": "0"} + run_command(["go", "mod", "tidy"], cwd=example_path, env=go_env, output=output, check=True) run_command( ["go", "tool", "airflow-go-pack", "--output", str(output_bin), "."], cwd=example_path, - env={**os.environ, "CGO_ENABLED": "0"}, + env=go_env, output=output, check=True, ) @@ -2628,26 +2638,30 @@ def _lang_sdk_build_go_bundle( # the real go_example's gitignored cache dir so the caches persist across scratch workspaces. (LANG_SDK_GO_EXAMPLE_PATH / ".home").mkdir(parents=True, exist_ok=True) get_console(output=output).print(f"[info]Building Go bundle in {LANG_SDK_GO_BUILDER_IMAGE}") + docker_base = [ + "docker", + "run", + "--rm", + "--user", + uid_gid, + "-e", + f"HOME={go_example_ctr}/.home", + "-e", + "USER=airflow", + "-e", + "CGO_ENABLED=0", + "-v", + f"{workspace}:/repo", + "-v", + f"{LANG_SDK_GO_EXAMPLE_PATH / '.home'}:{go_example_ctr}/.home", + "-w", + go_example_ctr, + LANG_SDK_GO_BUILDER_IMAGE, + ] + run_command([*docker_base, "go", "mod", "tidy"], output=output, check=True) run_command( [ - "docker", - "run", - "--rm", - "--user", - uid_gid, - "-e", - f"HOME={go_example_ctr}/.home", - "-e", - "USER=airflow", - "-e", - "CGO_ENABLED=0", - "-v", - f"{workspace}:/repo", - "-v", - f"{LANG_SDK_GO_EXAMPLE_PATH / '.home'}:{go_example_ctr}/.home", - "-w", - go_example_ctr, - LANG_SDK_GO_BUILDER_IMAGE, + *docker_base, "go", "tool", "airflow-go-pack", diff --git a/dev/breeze/tests/test_kubernetes_lang_sdk_commands.py b/dev/breeze/tests/test_kubernetes_lang_sdk_commands.py index ab5eb9946ea53..31332c24d1675 100644 --- a/dev/breeze/tests/test_kubernetes_lang_sdk_commands.py +++ b/dev/breeze/tests/test_kubernetes_lang_sdk_commands.py @@ -93,6 +93,10 @@ def test_native_uses_host_go_toolchain(self, mock_run, tmp_path, go_example, ups # The workspace mirrors the repo layout, with go-sdk swapped for the upstream copy. assert (workspace_example.parent / "go-sdk" / "marker.go").read_text() == "upstream" assert (tmp_path / "go-artifacts" / kubernetes_commands.LANG_SDK_GO_BUNDLE_NAME).exists() + # The scratch copy is re-tidied against the upstream go-sdk before packing, in the same dir. + tidy_call = mock_run.call_args_list[0] + assert tidy_call.args[0] == ["go", "mod", "tidy"] + assert tidy_call.kwargs["cwd"] == workspace_example @mock.patch.object(kubernetes_commands, "run_command") def test_container_mode_runs_in_docker(self, mock_run, tmp_path, go_example, upstream_go_sdk): @@ -108,6 +112,11 @@ def test_container_mode_runs_in_docker(self, mock_run, tmp_path, go_example, ups assert repo_mount.split(":")[0] != str(go_example.parent) home_mount = next(m for m in mounts if m.endswith("/.home")) assert home_mount.startswith(str(go_example / ".home")) + # The scratch copy is re-tidied in the same container image before packing. + tidy_cmd = mock_run.call_args_list[0].args[0] + assert tidy_cmd[0] == "docker" + assert kubernetes_commands.LANG_SDK_GO_BUILDER_IMAGE in tidy_cmd + assert tidy_cmd[-3:] == ["go", "mod", "tidy"] class TestLangSdkBuildJavaJar: diff --git a/go-sdk/go.mod b/go-sdk/go.mod index f321181d5fbef..bd3942939a1c1 100644 --- a/go-sdk/go.mod +++ b/go-sdk/go.mod @@ -49,10 +49,9 @@ require ( ) require ( - github.com/MatusOllah/slogcolor v1.6.0 github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect github.com/evanphx/go-hclog-slog v0.0.0-20240717231540-be48fc4c4df5 - github.com/fatih/color v1.18.0 + github.com/fatih/color v1.18.0 // indirect github.com/google/uuid v1.6.0 github.com/jarcoal/httpmock v1.4.0 github.com/mattn/go-colorable v0.1.14 // indirect diff --git a/go-sdk/go.sum b/go-sdk/go.sum index 811925904f3c9..38512f16109ad 100644 --- a/go-sdk/go.sum +++ b/go-sdk/go.sum @@ -1,5 +1,3 @@ -github.com/MatusOllah/slogcolor v1.6.0 h1:JAKer0xj5l1jYTXyQvs5ggqmJqYDuLnxgR9jfMAd+sI= -github.com/MatusOllah/slogcolor v1.6.0/go.mod h1:5y1H50XuQIBvuYTJlmokWi+4FuPiJN5L7Z0jM4K4bYA= github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk= github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ= github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk= diff --git a/go-sdk/pkg/config/config.go b/go-sdk/pkg/config/config.go index 54622743735a3..743ef50a867cd 100644 --- a/go-sdk/pkg/config/config.go +++ b/go-sdk/pkg/config/config.go @@ -25,8 +25,6 @@ import ( "strings" "time" - "github.com/MatusOllah/slogcolor" - "github.com/fatih/color" cc "github.com/ivanpirog/coloredcobra" "github.com/spf13/cobra" "github.com/spf13/pflag" @@ -90,7 +88,6 @@ func Configure(cmd *cobra.Command) error { } func makeLogger(v *viper.Viper) *slog.Logger { - opts := *slogcolor.DefaultOptions leveler := &slog.LevelVar{} // TODO: Should we have consistency with Airflow's config option? That would mean "logging.logging_level" here @@ -106,17 +103,20 @@ func makeLogger(v *viper.Viper) *slog.Logger { cobra.CheckErr(err) } - opts.Level = leveler - opts.LevelTags = map[slog.Level]string{ - logging.LevelTrace: color.New(color.FgHiGreen).Sprint("TRACE"), - slog.LevelDebug: color.New(color.BgCyan, color.FgHiWhite).Sprint("DEBUG"), - slog.LevelInfo: color.New(color.BgGreen, color.FgHiWhite).Sprint("INFO "), - slog.LevelWarn: color.New(color.BgYellow, color.FgHiWhite).Sprint("WARN "), - slog.LevelError: color.New(color.BgRed, color.FgHiWhite).Sprint("ERROR"), + opts := &slog.HandlerOptions{ + Level: leveler, + ReplaceAttr: func(groups []string, a slog.Attr) slog.Attr { + // Render our custom TRACE level by name rather than "DEBUG-4" + if a.Key == slog.LevelKey { + if level, ok := a.Value.Any().(slog.Level); ok && level == logging.LevelTrace { + a.Value = slog.StringValue("TRACE") + } + } + return a + }, } - log := slog.New(slogcolor.NewHandler(os.Stderr, &opts)) - return log + return slog.New(slog.NewTextHandler(os.Stderr, opts)) } func SetupViper(cfgFile string) (*viper.Viper, error) { diff --git a/kubernetes-tests/lang_sdk/README.md b/kubernetes-tests/lang_sdk/README.md index f4840623c33f0..52e081c197f5e 100644 --- a/kubernetes-tests/lang_sdk/README.md +++ b/kubernetes-tests/lang_sdk/README.md @@ -74,6 +74,12 @@ Everything else — `airflow-core/`, `task-sdk/`, the deployed Airflow image, an `go_example`/`java_example` harness fixtures — still comes from the checked-out branch as before, so a backport of a core/task-sdk fix to a release-test branch keeps testing against current SDK code. +Because the branch's `go_example` and upstream main's `go-sdk` can diverge (a branch may change +go-sdk's dependency graph while `go_example`'s committed `go.sum` is tidied against the in-repo +go-sdk), the Go bundle build re-runs `go mod tidy` in its scratch workspace before packing, so the +build reconciles to whichever `go-sdk` it is compiled against. The committed `go_example` `go.sum` +is untouched and stays guarded by the `check-go-example-mod-tidy` prek hook. + ## Running it The artifacts, localstack, config, and Helm release are provisioned by a single breeze diff --git a/kubernetes-tests/lang_sdk/go_example/go.mod b/kubernetes-tests/lang_sdk/go_example/go.mod index 7b210c53808fa..5b06693630e01 100644 --- a/kubernetes-tests/lang_sdk/go_example/go.mod +++ b/kubernetes-tests/lang_sdk/go_example/go.mod @@ -5,7 +5,6 @@ go 1.25.0 require github.com/apache/airflow/go-sdk v0.0.0 require ( - github.com/MatusOllah/slogcolor v1.6.0 // indirect github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect github.com/evanphx/go-hclog-slog v0.0.0-20240717231540-be48fc4c4df5 // indirect github.com/fatih/color v1.18.0 // indirect diff --git a/kubernetes-tests/lang_sdk/go_example/go.sum b/kubernetes-tests/lang_sdk/go_example/go.sum index 89c6e2466c6c4..f76761578e649 100644 --- a/kubernetes-tests/lang_sdk/go_example/go.sum +++ b/kubernetes-tests/lang_sdk/go_example/go.sum @@ -1,5 +1,3 @@ -github.com/MatusOllah/slogcolor v1.6.0 h1:JAKer0xj5l1jYTXyQvs5ggqmJqYDuLnxgR9jfMAd+sI= -github.com/MatusOllah/slogcolor v1.6.0/go.mod h1:5y1H50XuQIBvuYTJlmokWi+4FuPiJN5L7Z0jM4K4bYA= github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk= github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ= github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk=