Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 34 additions & 20 deletions dev/breeze/src/airflow_breeze/commands/kubernetes_commands.py
Original file line number Diff line number Diff line change
Expand Up @@ -2593,7 +2593,9 @@ def _lang_sdk_build_go_bundle(

go_example's go.mod ``replace``s go-sdk by relative path, so the build runs in a scratch
workspace mirroring the repo layout with ``upstream_go_sdk`` at ``<workspace>/go-sdk``,
letting the unmodified directive resolve against the upstream copy.
letting the unmodified directive resolve against the upstream copy. The scratch go_example is
re-tidied before packing so its go.sum reconciles to that upstream go-sdk (which may differ from
the in-repo go-sdk its committed go.sum was tidied against).
"""
go_dir = staging / "go-artifacts"
go_dir.mkdir(parents=True, exist_ok=True)
Expand All @@ -2611,12 +2613,20 @@ def _lang_sdk_build_go_bundle(

# CGO_ENABLED=0 yields a fully static binary that runs on the stock worker. The package built is
# the current dir (".") because go_example is its own module.
#
# go_example's go.sum is tidied against the in-repo go-sdk, but the bundle is built against the
# upstream-main go-sdk copied in above. When a branch changes go-sdk's dependency graph those two
# go-sdks differ, and Go refuses to build on the resulting go.sum drift. Re-tidy the scratch copy
# first so the build reconciles to whichever go-sdk it is actually compiled against; the committed
# go.sum is untouched and stays guarded by the check-go-example-mod-tidy prek hook.
if native:
get_console(output=output).print("[info]Building Go bundle with the host Go toolchain")
go_env = {**os.environ, "CGO_ENABLED": "0"}
run_command(["go", "mod", "tidy"], cwd=example_path, env=go_env, output=output, check=True)
run_command(
["go", "tool", "airflow-go-pack", "--output", str(output_bin), "."],
cwd=example_path,
env={**os.environ, "CGO_ENABLED": "0"},
env=go_env,
output=output,
check=True,
)
Expand All @@ -2628,26 +2638,30 @@ def _lang_sdk_build_go_bundle(
# the real go_example's gitignored cache dir so the caches persist across scratch workspaces.
(LANG_SDK_GO_EXAMPLE_PATH / ".home").mkdir(parents=True, exist_ok=True)
get_console(output=output).print(f"[info]Building Go bundle in {LANG_SDK_GO_BUILDER_IMAGE}")
docker_base = [
"docker",
"run",
"--rm",
"--user",
uid_gid,
"-e",
f"HOME={go_example_ctr}/.home",
"-e",
"USER=airflow",
"-e",
"CGO_ENABLED=0",
"-v",
f"{workspace}:/repo",
"-v",
f"{LANG_SDK_GO_EXAMPLE_PATH / '.home'}:{go_example_ctr}/.home",
"-w",
go_example_ctr,
LANG_SDK_GO_BUILDER_IMAGE,
]
run_command([*docker_base, "go", "mod", "tidy"], output=output, check=True)
run_command(
[
"docker",
"run",
"--rm",
"--user",
uid_gid,
"-e",
f"HOME={go_example_ctr}/.home",
"-e",
"USER=airflow",
"-e",
"CGO_ENABLED=0",
"-v",
f"{workspace}:/repo",
"-v",
f"{LANG_SDK_GO_EXAMPLE_PATH / '.home'}:{go_example_ctr}/.home",
"-w",
go_example_ctr,
LANG_SDK_GO_BUILDER_IMAGE,
*docker_base,
"go",
"tool",
"airflow-go-pack",
Expand Down
9 changes: 9 additions & 0 deletions dev/breeze/tests/test_kubernetes_lang_sdk_commands.py
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,10 @@ def test_native_uses_host_go_toolchain(self, mock_run, tmp_path, go_example, ups
# The workspace mirrors the repo layout, with go-sdk swapped for the upstream copy.
assert (workspace_example.parent / "go-sdk" / "marker.go").read_text() == "upstream"
assert (tmp_path / "go-artifacts" / kubernetes_commands.LANG_SDK_GO_BUNDLE_NAME).exists()
# The scratch copy is re-tidied against the upstream go-sdk before packing, in the same dir.
tidy_call = mock_run.call_args_list[0]
assert tidy_call.args[0] == ["go", "mod", "tidy"]
assert tidy_call.kwargs["cwd"] == workspace_example

@mock.patch.object(kubernetes_commands, "run_command")
def test_container_mode_runs_in_docker(self, mock_run, tmp_path, go_example, upstream_go_sdk):
Expand All @@ -108,6 +112,11 @@ def test_container_mode_runs_in_docker(self, mock_run, tmp_path, go_example, ups
assert repo_mount.split(":")[0] != str(go_example.parent)
home_mount = next(m for m in mounts if m.endswith("/.home"))
assert home_mount.startswith(str(go_example / ".home"))
# The scratch copy is re-tidied in the same container image before packing.
tidy_cmd = mock_run.call_args_list[0].args[0]
assert tidy_cmd[0] == "docker"
assert kubernetes_commands.LANG_SDK_GO_BUILDER_IMAGE in tidy_cmd
assert tidy_cmd[-3:] == ["go", "mod", "tidy"]


class TestLangSdkBuildJavaJar:
Expand Down
3 changes: 1 addition & 2 deletions go-sdk/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -49,10 +49,9 @@ require (
)

require (
github.com/MatusOllah/slogcolor v1.6.0
github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect
github.com/evanphx/go-hclog-slog v0.0.0-20240717231540-be48fc4c4df5
github.com/fatih/color v1.18.0
github.com/fatih/color v1.18.0 // indirect
github.com/google/uuid v1.6.0
github.com/jarcoal/httpmock v1.4.0
github.com/mattn/go-colorable v0.1.14 // indirect
Expand Down
2 changes: 0 additions & 2 deletions go-sdk/go.sum
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
github.com/MatusOllah/slogcolor v1.6.0 h1:JAKer0xj5l1jYTXyQvs5ggqmJqYDuLnxgR9jfMAd+sI=
github.com/MatusOllah/slogcolor v1.6.0/go.mod h1:5y1H50XuQIBvuYTJlmokWi+4FuPiJN5L7Z0jM4K4bYA=
github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk=
github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ=
github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk=
Expand Down
24 changes: 12 additions & 12 deletions go-sdk/pkg/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,6 @@ import (
"strings"
"time"

"github.com/MatusOllah/slogcolor"
"github.com/fatih/color"
cc "github.com/ivanpirog/coloredcobra"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
Expand Down Expand Up @@ -90,7 +88,6 @@ func Configure(cmd *cobra.Command) error {
}

func makeLogger(v *viper.Viper) *slog.Logger {
opts := *slogcolor.DefaultOptions
leveler := &slog.LevelVar{}

// TODO: Should we have consistency with Airflow's config option? That would mean "logging.logging_level" here
Expand All @@ -106,17 +103,20 @@ func makeLogger(v *viper.Viper) *slog.Logger {
cobra.CheckErr(err)
}

opts.Level = leveler
opts.LevelTags = map[slog.Level]string{
logging.LevelTrace: color.New(color.FgHiGreen).Sprint("TRACE"),
slog.LevelDebug: color.New(color.BgCyan, color.FgHiWhite).Sprint("DEBUG"),
slog.LevelInfo: color.New(color.BgGreen, color.FgHiWhite).Sprint("INFO "),
slog.LevelWarn: color.New(color.BgYellow, color.FgHiWhite).Sprint("WARN "),
slog.LevelError: color.New(color.BgRed, color.FgHiWhite).Sprint("ERROR"),
opts := &slog.HandlerOptions{
Level: leveler,
ReplaceAttr: func(groups []string, a slog.Attr) slog.Attr {
// Render our custom TRACE level by name rather than "DEBUG-4"
if a.Key == slog.LevelKey {
if level, ok := a.Value.Any().(slog.Level); ok && level == logging.LevelTrace {
a.Value = slog.StringValue("TRACE")
}
}
return a
},
}

log := slog.New(slogcolor.NewHandler(os.Stderr, &opts))
return log
return slog.New(slog.NewTextHandler(os.Stderr, opts))
}

func SetupViper(cfgFile string) (*viper.Viper, error) {
Expand Down
6 changes: 6 additions & 0 deletions kubernetes-tests/lang_sdk/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,12 @@ Everything else — `airflow-core/`, `task-sdk/`, the deployed Airflow image, an
`go_example`/`java_example` harness fixtures — still comes from the checked-out branch as before, so a
backport of a core/task-sdk fix to a release-test branch keeps testing against current SDK code.

Because the branch's `go_example` and upstream main's `go-sdk` can diverge (a branch may change
go-sdk's dependency graph while `go_example`'s committed `go.sum` is tidied against the in-repo
go-sdk), the Go bundle build re-runs `go mod tidy` in its scratch workspace before packing, so the
build reconciles to whichever `go-sdk` it is compiled against. The committed `go_example` `go.sum`
is untouched and stays guarded by the `check-go-example-mod-tidy` prek hook.

## Running it

The artifacts, localstack, config, and Helm release are provisioned by a single breeze
Expand Down
1 change: 0 additions & 1 deletion kubernetes-tests/lang_sdk/go_example/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ go 1.25.0
require github.com/apache/airflow/go-sdk v0.0.0

require (
github.com/MatusOllah/slogcolor v1.6.0 // indirect
github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect
github.com/evanphx/go-hclog-slog v0.0.0-20240717231540-be48fc4c4df5 // indirect
github.com/fatih/color v1.18.0 // indirect
Expand Down
2 changes: 0 additions & 2 deletions kubernetes-tests/lang_sdk/go_example/go.sum
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
github.com/MatusOllah/slogcolor v1.6.0 h1:JAKer0xj5l1jYTXyQvs5ggqmJqYDuLnxgR9jfMAd+sI=
github.com/MatusOllah/slogcolor v1.6.0/go.mod h1:5y1H50XuQIBvuYTJlmokWi+4FuPiJN5L7Z0jM4K4bYA=
github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk=
github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ=
github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk=
Expand Down
Loading