diff --git a/buck2/golang/.buckconfig b/buck2/golang/.buckconfig new file mode 100644 index 00000000..260c1f66 --- /dev/null +++ b/buck2/golang/.buckconfig @@ -0,0 +1,32 @@ +[cells] + root = . + prelude = prelude + toolchains = toolchains + none = none + +[cell_aliases] + config = prelude + fbcode = none + fbsource = none + buck = none + +[external_cells] + prelude = bundled + +[parser] + target_platform_detector_spec = target:root//...->prelude//platforms:default + +[buck2] +digest_algorithms = SHA256 + +[buck2_re_client] +engine_address = .cluster.engflow.com +action_cache_address = .cluster.engflow.com +cas_address = .cluster.engflow.com +http_headers = x-engflow-auth-method:jwt-v0,x-engflow-auth-token:LONG_JWT_STRING + +[build] + execution_platforms = root//platforms:remote_platform + +[project] + ignore = .git diff --git a/buck2/golang/.buckroot b/buck2/golang/.buckroot new file mode 100644 index 00000000..e69de29b diff --git a/buck2/golang/.gitignore b/buck2/golang/.gitignore new file mode 100644 index 00000000..0a0ddb2e --- /dev/null +++ b/buck2/golang/.gitignore @@ -0,0 +1 @@ +/buck-out diff --git a/buck2/golang/README.md b/buck2/golang/README.md new file mode 100644 index 00000000..603c5404 --- /dev/null +++ b/buck2/golang/README.md @@ -0,0 +1,52 @@ +# EngFlow RE + Buck2 Go example + +This example demonstrates use of EngFlow RE for a simple Go lang project built with [Buck2](https://github.com/facebook/buck2) using the prelude. + +It is based on three existing samples in the Buck2 upstream repo and the EngFlow samples repo: + +* Simple go project with prelude - https://github.com/facebook/buck2/tree/main/examples/with_prelude/go +* Buck2 remote execution integration with EngFlow - https://github.com/facebook/buck2/tree/main/examples/remote_execution/engflow +* EngFlow go example - https://github.com/EngFlow/example/tree/main/go + +### Example structure + +In the `platforms` cell we specify: +* The platform used for remote execution in this project `root//platforms:remote_platform`, which includes the definition of the Docker image used for remote execution, and that defines constraints for targets to run in the remote execution environment. This platform provides an `ExecutionPlatformRegistrationInfo`. +* The action keys `root//platforms:remote_execution_action_keys`, which provides a default `BuildModeInfo` that is needed for RE of tests to function properly. +* The platform `image` configured in `platforms/defs.bzl`, notably, uses a different image than other Buck2 samples in this repo. Specifically, it uses a public AWS image that has `go` preinstalled. This is because, unlike Bazel go rules, Buck2 go rules do not include a hermetic go binary. + +In the `toolchains` cell we specify: + +* The remote go toolchain `root//toolchains:remote_go_toolchain` and remote go bootstrap toolchain `root//toolchains:remote_go_bootstrap_toolchain`, which are compatible with the remote execution environment. These toolchains are configured for `go_arch` set to `amd64` and `go_os` set to `linux`. +* The remote test execution toolchain, `root//toolchains:remote_test_execution_toolchain`. This toolchain defines platform options in the form of `capabilities`. Critically these include the `container-image`. This toolchain is identical to the one in the `buck2/cpp` sample in this repo. + +The `main` cell and `library` cell: + +* Contains an copied version of https://github.com/facebook/buck2/tree/main/examples/with_prelude/go that works with Buck2 and RE as configured in this sample project. + +To test this cell with RE run (after setting up `.buckconfig` as indicated below): + +``` +buck2 test //go/greeting:greeting_test +``` + +You can also build the `main` for this sample by running: + +``` +buck2 build //go:hello +``` + +### Relevant configs in `.buckconfig` + +The EngFlow endpoint and certificate should be configured as the +following: + +```ini +[buck2_re_client] +engine_address = .cluster.engflow.com +action_cache_address = .cluster.engflow.com +cas_address = .cluster.engflow.com +http_headers = x-engflow-auth-method:jwt-v0,x-engflow-auth-token:LONG_JWT_STRING + ``` + +To obtain the value of `LONG_JWT_STRING`, log into https://.cluster.engflow.com/gettingstarted and use the value of `x-engflow-auth-token` in section `Method 2: JWT`. diff --git a/buck2/golang/go/BUCK b/buck2/golang/go/BUCK new file mode 100644 index 00000000..c4e07961 --- /dev/null +++ b/buck2/golang/go/BUCK @@ -0,0 +1,21 @@ +# Copyright 2022 EngFlow Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +go_binary( + name = "hello", + srcs = ["main.go"], + deps = [ + "//go/greeting:greeting", + ], +) diff --git a/buck2/golang/go/greeting/BUCK b/buck2/golang/go/greeting/BUCK new file mode 100644 index 00000000..1af167a4 --- /dev/null +++ b/buck2/golang/go/greeting/BUCK @@ -0,0 +1,25 @@ +# Copyright 2022 EngFlow Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +go_library( + name = "greeting", + srcs = glob(["*.go"]), + visibility = ["PUBLIC"], +) + +go_test( + name = "greeting_test", + srcs = glob(["*.go"]), + remote_execution_action_key_providers = "//platforms:remote_execution_action_keys", +) diff --git a/buck2/golang/go/greeting/greeting.go b/buck2/golang/go/greeting/greeting.go new file mode 100644 index 00000000..f089ff9d --- /dev/null +++ b/buck2/golang/go/greeting/greeting.go @@ -0,0 +1,7 @@ + +package greeting + +// Greeting returns a greeting message +func Greeting() string { + return "Hello, world!" +} diff --git a/buck2/golang/go/greeting/greeting_test.go b/buck2/golang/go/greeting/greeting_test.go new file mode 100644 index 00000000..beefdfce --- /dev/null +++ b/buck2/golang/go/greeting/greeting_test.go @@ -0,0 +1,9 @@ +package greeting + +import "testing" + +func TestHello(t *testing.T) { + if Greeting() != "Hello, world!" { + t.Errorf("Greeting() = %v, want \"Hello, world!\"", Greeting()) + } +} diff --git a/buck2/golang/go/main.go b/buck2/golang/go/main.go new file mode 100644 index 00000000..9750cfb8 --- /dev/null +++ b/buck2/golang/go/main.go @@ -0,0 +1,11 @@ +package main + +import ( + "fmt" + + "go/greeting" +) + +func main() { + fmt.Println(greeting.Greeting()) +} diff --git a/buck2/golang/platforms/BUCK b/buck2/golang/platforms/BUCK new file mode 100644 index 00000000..cc1595f9 --- /dev/null +++ b/buck2/golang/platforms/BUCK @@ -0,0 +1,30 @@ +# Copyright 2022 EngFlow Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load(":defs.bzl", "platforms") +load(":defs.bzl", "action_keys") + +# This platform configures details of remote execution. +platforms( + name = "remote_platform", +) + +# This action_key provides a default BuildModeInfo that is needed for RE of tests to function properly. +# The values in `cell` and `mode` can be used, in practice, to create cache silos. Any values can be given to these attributes. +action_keys( + name = "remote_execution_action_keys", + cell = "standard", + mode = "standard", + visibility = ["PUBLIC"], +) diff --git a/buck2/golang/platforms/defs.bzl b/buck2/golang/platforms/defs.bzl new file mode 100644 index 00000000..700efd88 --- /dev/null +++ b/buck2/golang/platforms/defs.bzl @@ -0,0 +1,68 @@ +# Copyright 2022 EngFlow Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This platform is essentially the same as the one provided in https://github.com/facebook/buck2/blob/804d62242214455d51787f7c8c96a1e12c75ec32/examples/remote_execution/engflow/platforms/defs.bzl +# The main difference is we enable passing CPU and OS constraints and we use the sample EngFlow RE image. +load("@prelude//:build_mode.bzl", "BuildModeInfo") + +def _platforms(ctx): + constraints = dict() + configuration = ConfigurationInfo( + constraints = constraints, + values = {}, + ) + + # A bookworm image with go pre-installed. + # Unlike Bazel go_toolchain, Buck2 go_toolchain does not include a hermetic go binary. + image = "docker://public.ecr.aws/docker/library/golang:1.23.3-bookworm@sha256:3f3b9daa3de608f3e869cd2ff8baf21555cf0fca9fd34251b8f340f9b7c30ec5" + name = ctx.label.raw_target() + platform = ExecutionPlatformInfo( + label = ctx.label.raw_target(), + configuration = configuration, + executor_config = CommandExecutorConfig( + local_enabled = False, + remote_enabled = True, + use_limited_hybrid = False, + remote_execution_properties = { + "container-image": image, + }, + remote_execution_use_case = "buck2-default", + # TODO: Use output_paths + remote_output_paths = "strict", + ), + ) + + return [ + DefaultInfo(), + ExecutionPlatformRegistrationInfo(platforms = [platform]), + ] + +def _action_keys(ctx): + return [ + DefaultInfo(), + BuildModeInfo(cell = ctx.attrs.cell, mode = ctx.attrs.mode), + ] + +platforms = rule( + attrs = {}, + impl = _platforms +) + +action_keys = rule( + attrs = { + "cell": attrs.string(), + "mode": attrs.string(), + }, + impl = _action_keys +) diff --git a/buck2/golang/toolchains/BUCK b/buck2/golang/toolchains/BUCK new file mode 100644 index 00000000..82eb6a3e --- /dev/null +++ b/buck2/golang/toolchains/BUCK @@ -0,0 +1,65 @@ + +# Copyright 2022 EngFlow Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("defs.bzl", "remote_go_toolchain", "remote_go_bootstrap_toolchain") +load("@prelude//toolchains:cxx.bzl", "cxx_tools_info_toolchain") +load("@prelude//toolchains:python.bzl", "system_python_bootstrap_toolchain") +load("@prelude//toolchains:remote_test_execution.bzl", "remote_test_execution_toolchain") +load("@prelude//tests:test_toolchain.bzl", "noop_test_toolchain") + +# Python toolchain used in scripts that bootstrap other aspects of the Buck2 prelude. +system_python_bootstrap_toolchain( + name = "python_bootstrap", + visibility = ["PUBLIC"], +) + +remote_go_bootstrap_toolchain( + name = "go_bootstrap", + visibility = ["PUBLIC"], +) + +# c++ toolchain required implicitly for py targets. +cxx_tools_info_toolchain( + name = "cxx", + visibility = ["PUBLIC"], +) + +remote_go_toolchain( + name = "go", + visibility = ["PUBLIC"], +) + +# Default toolchain for remote execution of tests. +# Note it defines a profile with a capability that defines the `container-image` that matches the one defined in //platforms:remote_platform. +# Capabilities are passed to the RE service to find workers that match them as Platform options. +remote_test_execution_toolchain( + name = "remote_test_execution", + visibility = ["PUBLIC"], + default_profile = "cxx_re_toolchain", + profiles = { + "cxx_re_toolchain": { + "use_case": "cxx-testing", + "capabilities": { + "container-image" : "docker://gcr.io/bazel-public/ubuntu2004-java11@sha256:69a78f121230c6d5cbfe2f4af8ce65481aa3f2acaaaf8e899df335f1ac1b35b5", + }, + } + }, +) + +# In some cases the execution of test can fail looking for this `noop_test_toolchain`. +noop_test_toolchain( + name = "test", + visibility = ["PUBLIC"], +) diff --git a/buck2/golang/toolchains/defs.bzl b/buck2/golang/toolchains/defs.bzl new file mode 100644 index 00000000..f1570548 --- /dev/null +++ b/buck2/golang/toolchains/defs.bzl @@ -0,0 +1,96 @@ +# Copyright 2022 EngFlow Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@prelude//go:toolchain.bzl", "GoToolchainInfo") +load("@prelude//go_bootstrap:go_bootstrap.bzl", "GoBootstrapToolchainInfo") +load("@prelude//utils:cmd_script.bzl", "ScriptOs", "cmd_script") + +def _remote_go_bootstrap_toolchain_impl(ctx): + go_arch = "amd64" + go_os = "linux" + + script_os = ScriptOs("unix") + go = "go" + + return [ + DefaultInfo(), + GoBootstrapToolchainInfo( + env_go_arch = go_arch, + env_go_os = go_os, + go = RunInfo(cmd_script(ctx, "go", cmd_args(go), script_os)), + go_wrapper = ctx.attrs.go_wrapper[RunInfo], + ), + ] + +remote_go_bootstrap_toolchain = rule( + impl = _remote_go_bootstrap_toolchain_impl, + doc = """Example remote go toolchain rules. Usage: + remote_go_bootstrap_toolchain( + name = "go_bootstrap", + visibility = ["PUBLIC"], + )""", + attrs = { + "go_wrapper": attrs.default_only(attrs.dep(providers = [RunInfo], default = "prelude//go/tools:go_wrapper_py")), + }, + is_toolchain_rule = True, +) + +def _remote_go_toolchain_impl(ctx): + go_arch = "amd64" + go_os = "linux" + + script_os = ScriptOs("unix") + go = "go" + + return [ + DefaultInfo(), + GoToolchainInfo( + assembler = RunInfo(cmd_script(ctx, "asm", cmd_args(go, "tool", "asm"), script_os)), + cgo = RunInfo(cmd_script(ctx, "cgo", cmd_args(go, "tool", "cgo"), script_os)), + cgo_wrapper = ctx.attrs.cgo_wrapper[RunInfo], + concat_files = ctx.attrs.concat_files[RunInfo], + compiler = RunInfo(cmd_script(ctx, "compile", cmd_args(go, "tool", "compile"), script_os)), + cover = RunInfo(cmd_script(ctx, "cover", cmd_args(go, "tool", "cover"), script_os)), + default_cgo_enabled = False, + env_go_arch = go_arch, + env_go_os = go_os, + external_linker_flags = [], + gen_stdlib_importcfg = ctx.attrs.gen_stdlib_importcfg[RunInfo], + go = RunInfo(cmd_script(ctx, "go", cmd_args(go), script_os)), + go_wrapper = ctx.attrs.go_wrapper[RunInfo], + linker = RunInfo(cmd_script(ctx, "link", cmd_args(go, "tool", "link"), script_os)), + packer = RunInfo(cmd_script(ctx, "pack", cmd_args(go, "tool", "pack"), script_os)), + tags = [], + linker_flags = [], + assembler_flags = [], + compiler_flags = [], + ), + ] + + +remote_go_toolchain = rule( + impl = _remote_go_toolchain_impl, + doc = """Example remote go toolchain rules. Usage: + remote_go_toolchain( + name = "go", + visibility = ["PUBLIC"], + )""", + attrs = { + "cgo_wrapper": attrs.default_only(attrs.dep(providers = [RunInfo], default = "prelude//go/tools:cgo_wrapper")), + "concat_files": attrs.default_only(attrs.dep(providers = [RunInfo], default = "prelude//go_bootstrap/tools:go_concat_files")), + "gen_stdlib_importcfg": attrs.default_only(attrs.dep(providers = [RunInfo], default = "prelude//go/tools:gen_stdlib_importcfg")), + "go_wrapper": attrs.default_only(attrs.dep(providers = [RunInfo], default = "prelude//go_bootstrap/tools:go_go_wrapper")), + }, + is_toolchain_rule = True, +) diff --git a/buck2/python/README.md b/buck2/python/README.md index d651161a..09dbc504 100644 --- a/buck2/python/README.md +++ b/buck2/python/README.md @@ -5,13 +5,14 @@ This example demonstrates use of EngFlow RE for a simple python project built wi It is based on three existing samples in the Buck2 upstream repo and the EngFlow samples repo: * Simple python project with prelude - https://github.com/facebook/buck2/tree/main/examples/with_prelude/python -* Buck2 remote execution integration with EngFlow - https://github.com/facebook/buck2/tree/804d62242214455d51787f7c8c96a1e12c75ec32/examples/remote_execution/engflow +* Buck2 remote execution integration with EngFlow - https://github.com/facebook/buck2/tree/main/examples/remote_execution/engflow * EngFlow python example - https://github.com/EngFlow/example/tree/main/python ### Example structure In the `platforms` cell we specify: * The platform used for remote execution in this project `root//platforms:remote_platform`, which includes the definition of the Docker image used for remote execution, and that defines constraints for targets to run in the remote execution environment. This platform provides an `ExecutionPlatformRegistrationInfo`. +* The action keys `root//platforms:remote_execution_action_keys`, which provides a default `BuildModeInfo` that is needed for RE of tests to function properly. In the `toolchains` cell we specify: