# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under both the MIT license found in the
# LICENSE-MIT file in the root directory of this source tree and the Apache
# License, Version 2.0 found in the LICENSE-APACHE file in the root directory
# of this source tree.
# Implementation of the `genrule` build rule.
load("@prelude//:cache_mode.bzl", "CacheModeInfo")
load("@prelude//:genrule_local_labels.bzl", "genrule_labels_require_local")
load("@prelude//:genrule_prefer_local_labels.bzl", "genrule_labels_prefer_local")
load("@prelude//:genrule_toolchain.bzl", "GenruleToolchainInfo")
load("@prelude//:is_full_meta_repo.bzl", "is_full_meta_repo")
load("@prelude//android:build_only_native_code.bzl", "is_build_only_native_code")
load("@prelude//os_lookup:defs.bzl", "Os", "OsLookup")
load("@prelude//utils:expect.bzl", "expect")
load("@prelude//utils:utils.bzl", "flatten", "value_or")
GENRULE_OUT_DIR = "out"
# Currently, some rules require running from the project root, so provide an
# opt-in list for those here. Longer-term, these should be ported to actual
# rule implementations in v2, rather then using `genrule`s.
_BUILD_ROOT_LABELS = set([
# The buck2 test suite
"buck2_test_build_root",
"antlir_macros",
"rust_bindgen",
"haskell_hsc",
"cql_cxx_genrule",
"clang-module",
"cuda_build_root",
"bundle_pch_genrule", # Compiles C++, and so need to run from build root
"lpm_package",
"haskell_dll",
"fnlc_build",
"udf_sql",
"redex_genrule", # T148016945
"pxl", # T151533831
"app_modules_genrule", # produces JSON containing file paths that are read from the root dir.
"android_langpack_strings", # produces JSON containing file paths that are read from the root dir.
"windows_long_path_issue", # Windows: relative path length exceeds PATH_MAX, program cannot access file
"flowtype_ota_safety_target", # produces JSON containing file paths that are project-relative
"ctrlr_setting_paths",
"llvm_buck_genrule",
])
# In Buck1 the SRCS environment variable is only set if the substring SRCS is on the command line.
# That's a horrible heuristic, and doesn't account for users accessing $SRCS from a shell script.
# But in some cases, $SRCS is so large it breaks the process limit, so have a label to opt in to
# that behavior.
_NO_SRCS_ENVIRONMENT_LABEL = "no_srcs_environment"
_WINDOWS_ENV_SUBSTITUTIONS = [
# Replace $OUT and ${OUT}
(regex("\\$(OUT\\b|\\{OUT\\})"), "%OUT%"),
(regex("\\$(SRCDIR\\b|\\{SRCDIR\\})"), "%SRCDIR%"),
(regex("\\$(SRCS\\b|\\{SRCS\\})"), "%SRCS%"),
(regex("\\$(TMP\\b|\\{TMP\\})"), "%TMP%"),
]
def _requires_build_root(ctx: AnalysisContext) -> bool:
for label in ctx.attrs.labels:
if label in _BUILD_ROOT_LABELS:
return True
return False
def _requires_local(ctx: AnalysisContext) -> bool:
return genrule_labels_require_local(ctx.attrs.labels)
def _prefers_local(ctx: AnalysisContext) -> bool:
return genrule_labels_prefer_local(ctx.attrs.labels)
def _ignore_artifacts(ctx: AnalysisContext) -> bool:
return "buck2_ignore_artifacts" in ctx.attrs.labels
def _requires_no_srcs_environment(ctx: AnalysisContext) -> bool:
return _NO_SRCS_ENVIRONMENT_LABEL in ctx.attrs.labels
# We don't want to use cache mode in open source because the config keys that drive it aren't wired up
_USE_CACHE_MODE = is_full_meta_repo()
# Extra attributes required by every genrule based on genrule_impl
def genrule_attributes() -> dict[str, Attr]:
attributes = {
"always_print_stderr": attrs.bool(default = False),
"metadata_env_var": attrs.option(attrs.string(), default = None),
"metadata_path": attrs.option(attrs.string(), default = None),
"no_outputs_cleanup": attrs.bool(default = False),
"remote_execution_dependencies": attrs.list(attrs.dict(key = attrs.string(), value = attrs.string()), default = []),
"_build_only_native_code": attrs.default_only(attrs.bool(default = is_build_only_native_code())),
"_genrule_toolchain": attrs.default_only(attrs.toolchain_dep(default = "toolchains//:genrule", providers = [GenruleToolchainInfo])),
}
if _USE_CACHE_MODE and not read_root_config("fb", "cache_mode") == None:
attributes["_cache_mode"] = attrs.dep(default = read_root_config("fb", "cache_mode"))
return attributes
def _get_cache_mode(ctx: AnalysisContext) -> CacheModeInfo:
if _USE_CACHE_MODE:
return ctx.attrs._cache_mode[CacheModeInfo]
else:
return CacheModeInfo(allow_cache_uploads = False, cache_bust_genrules = False)
def genrule_impl(ctx: AnalysisContext) -> list[Provider]:
# Directories:
# sh - sh file
# src - sources files
# out - where outputs go
# `src` is the current directory
# Buck1 uses `.` as output, but that won't work since
# Buck2 clears the output directory before execution, and thus src/sh too.
return process_genrule(ctx, ctx.attrs.out, ctx.attrs.outs)
def _declare_output(ctx: AnalysisContext, path: str) -> Artifact:
if path == ".":
return ctx.actions.declare_output(GENRULE_OUT_DIR, dir = True)
elif path.endswith("/"):
return ctx.actions.declare_output(GENRULE_OUT_DIR, path[:-1], dir = True)
else:
return ctx.actions.declare_output(GENRULE_OUT_DIR, path)
def _project_output(out: Artifact, path: str) -> Artifact:
if path == ".":
return out
elif path.endswith("/"):
return out.project(path[:-1], hide_prefix = True)
else:
return out.project(path, hide_prefix = True)
def process_genrule(
ctx: AnalysisContext,
out_attr: [str, None],
outs_attr: [dict, None],
extra_env_vars: dict = {},
identifier: [str, None] = None,
other_outputs: list[Artifact] = [],
genrule_error_handler: [typing.Callable[[ActionErrorCtx], list[ActionSubError]], None] = None) -> list[Provider]:
if (out_attr != None) and (outs_attr != None):
fail("Only one of `out` and `outs` should be set. Got out=`%s`, outs=`%s`" % (repr(out_attr), repr(outs_attr)))
local_only = _requires_local(ctx)
prefer_local = _prefers_local(ctx)
# NOTE: Eventually we shouldn't require local_only here, since we should be
# fine with caching local fallbacks if necessary (or maybe that should be
# disallowed as a matter of policy), but for now let's be safe.
cacheable = value_or(ctx.attrs.cacheable, True) and (local_only or prefer_local)
executable_outs = getattr(ctx.attrs, "executable_outs", None)
# TODO(cjhopman): verify output paths are ".", "./", or forward-relative.
if out_attr != None:
out_artifact = _declare_output(ctx, out_attr)
named_outputs = {}
default_outputs = [out_artifact]
expect(executable_outs == None, "`executable_outs` should not be set when `out` is set")
elif outs_attr != None:
out_artifact = ctx.actions.declare_output(GENRULE_OUT_DIR, dir = True)
named_outputs = {
name: [_project_output(out_artifact, path) for path in outputs]
for (name, outputs) in outs_attr.items()
}
outs_names = outs_attr.keys()
if executable_outs != None:
for executable_out in executable_outs:
expect(executable_out in outs_names, "Value in `executable_outs` {} is not in `outs`".format(executable_out))
default_outputs = [
_project_output(out_artifact, path)
for path in (ctx.attrs.default_outs or [])
]
if len(default_outputs) == 0:
# We want building to force something to be built, so make sure it contains at least one artifact
default_outputs = [out_artifact]
else:
fail("One of `out` or `outs` should be set. Got `%s`" % repr(ctx.attrs))
# Some custom rules use `process_genrule` but doesn't set this attribute.
is_windows = hasattr(ctx.attrs, "_exec_os_type") and ctx.attrs._exec_os_type[OsLookup].os == Os("windows")
if is_windows:
path_sep = "\\"
cmd = ctx.attrs.cmd_exe if ctx.attrs.cmd_exe != None else ctx.attrs.cmd
if cmd == None:
fail("One of `cmd` or `cmd_exe` should be set.")
else:
path_sep = "/"
cmd = ctx.attrs.bash if ctx.attrs.bash != None else ctx.attrs.cmd
if cmd == None:
fail("One of `cmd` or `bash` should be set.")
replace_regex = []
# For backwards compatibility with Buck1.
if is_windows:
for re, sub in _WINDOWS_ENV_SUBSTITUTIONS:
replace_regex.append((re, sub))
for extra_env_var in extra_env_vars:
replace_regex.append(
(regex("\\$(%s\\b|\\{%s\\})" % (extra_env_var, extra_env_var)), "%%%s%%" % extra_env_var),
)
cmd = cmd_args(cmd, ignore_artifacts = _ignore_artifacts(ctx), replace_regex = replace_regex)
if type(ctx.attrs.srcs) == type([]):
# FIXME: We should always use the short_path, but currently that is sometimes blank.
# See fbcode//buck2/tests/targets/rules/genrule:genrule-dot-input for a test that exposes it.
symlinks = {src.short_path: src for src in ctx.attrs.srcs}
if len(symlinks) != len(ctx.attrs.srcs):
for src in ctx.attrs.srcs:
name = src.short_path
if symlinks[name] != src:
msg = "genrule srcs include duplicative name: `{}`. ".format(name)
msg += "`{}` conflicts with `{}`".format(symlinks[name].owner, src.owner)
fail(msg)
else:
symlinks = ctx.attrs.srcs
srcs_artifact = ctx.actions.symlinked_dir("srcs" if not identifier else "{}-srcs".format(identifier), symlinks)
if ctx.attrs.environment_expansion_separator:
delimiter = ctx.attrs.environment_expansion_separator
else:
delimiter = " "
# Setup environment variables.
srcs = cmd_args(delimiter = delimiter)
for symlink in symlinks:
srcs.add(cmd_args(srcs_artifact, format = path_sep.join([".", "{}", symlink.replace("/", path_sep)])))
env_vars = {
"GEN_DIR": "GEN_DIR_DEPRECATED",
"OUT": out_artifact.as_output(),
"SRCDIR": cmd_args(srcs_artifact, format = path_sep.join([".", "{}"])),
"SRCS": srcs,
} | {k: cmd_args(v) for k, v in getattr(ctx.attrs, "env", {}).items()}
# RE will cache successful actions that don't produce the desired outptuts,
# so if that happens and _then_ we add a local-only label, we'll get a
# cache hit on the action that didn't produce the outputs and get the error
# again (thus making the label useless). So, when a local-only label is
# set, we make the action *different*.
if local_only:
env_vars["__BUCK2_LOCAL_ONLY_CACHE_BUSTER"] = ""
# see comment above
if prefer_local:
env_vars["__BUCK2_PREFER_LOCAL_CACHE_BUSTER"] = ""
# For now, when uploads are enabled, be safe and avoid sharing cache hits.
cache_bust = _get_cache_mode(ctx).cache_bust_genrules
if cacheable and cache_bust:
env_vars["__BUCK2_ALLOW_CACHE_UPLOADS_CACHE_BUSTER"] = ""
if _requires_no_srcs_environment(ctx):
env_vars.pop("SRCS")
for key, value in extra_env_vars.items():
env_vars[key] = value
# Create required directories.
if is_windows:
script = [
cmd_args(srcs_artifact, format = "if not exist .\\{}\\..\\out mkdir .\\{}\\..\\out"),
cmd_args("if NOT \"%TEMP%\" == \"\" set \"TMP=%TEMP%\""),
]
script_extension = "bat"
else:
script = [
# Use a somewhat unique exit code so this can get retried on RE (T99656531).
cmd_args(srcs_artifact, format = "mkdir -p ./{}/../out || exit 99"),
cmd_args("export TMP=${TMPDIR:-/tmp}"),
]
script_extension = "sh"
# Actually define the operation, relative to where we changed to
script.append(cmd)
hidden = []
genrule_toolchain = ctx.attrs._genrule_toolchain[GenruleToolchainInfo]
zip_scrubber = genrule_toolchain.zip_scrubber
if not is_windows and zip_scrubber != None:
zip_outputs = [output for output in default_outputs + flatten(named_outputs.values()) if output.extension == ".zip"]
if zip_outputs:
hidden.append(zip_scrubber)
# Any outputs that are .zip files need to be "scrubbed" to ensure that they are deterministic.
script = [
cmd_args("ORIGINAL_DIR_FOR_ZIP_SCRUBBING=$(pwd)"),
] + script + [
cmd_args('cd "$ORIGINAL_DIR_FOR_ZIP_SCRUBBING"'),
] + [
cmd_args(zip_scrubber, output, delimiter = " ", quote = "shell")
for output in zip_outputs
]
# Some rules need to run from the build root, but for everything else, `cd`
# into the sandboxed source dir and relative all paths to that.
if not _requires_build_root(ctx):
srcs_dir = srcs_artifact
if is_windows:
rewrite_scratch_path = cmd_args(
cmd_args(ctx.label.project_root, relative_to = srcs_artifact),
format = 'set "BUCK_SCRATCH_PATH={}\\%BUCK_SCRATCH_PATH%"',
)
else:
srcs_dir = cmd_args(srcs_dir, quote = "shell")
rewrite_scratch_path = cmd_args(
cmd_args(ctx.label.project_root, quote = "shell", relative_to = srcs_artifact),
format = "export BUCK_SCRATCH_PATH={}/$BUCK_SCRATCH_PATH",
)
# Relativize all paths in the command to the sandbox dir.
for script_cmd in script:
script_cmd.relative_to(srcs_artifact)
script = (
[
# Rewrite BUCK_SCRATCH_PATH
rewrite_scratch_path,
# Change to the directory that genrules expect.
cmd_args(srcs_dir, format = "cd {}"),
] +
script
)
# Relative all paths in the env to the sandbox dir.
env_vars = {
key: cmd_args(value, relative_to = srcs_artifact)
for key, value in env_vars.items()
}
if is_windows:
# Should be in the beginning.
# Odd, why is this a single string. How does it not end up getting quoted and being weird?
script = [cmd_args("@echo off")] + script
sh_script, _ = ctx.actions.write(
"sh/genrule.{}".format(script_extension) if not identifier else "sh/{}-genrule.{}".format(identifier, script_extension),
script,
is_executable = True,
allow_args = True,
)
if is_windows:
script_args = ["cmd.exe", "/v:off", "/c", sh_script]
else:
script_args = ["/usr/bin/env", "bash", "-e", sh_script]
# Only set metadata arguments when they are non-null
metadata_args = {}
if ctx.attrs.metadata_env_var:
metadata_args["metadata_env_var"] = ctx.attrs.metadata_env_var
if ctx.attrs.metadata_path:
metadata_args["metadata_path"] = ctx.attrs.metadata_path
if ctx.attrs.remote_execution_dependencies:
metadata_args["remote_execution_dependencies"] = ctx.attrs.remote_execution_dependencies
category = "genrule"
if ctx.attrs.type != None:
# As of 09/2021, all genrule types were legal snake case if their dashes and periods were replaced with underscores.
category += "_" + ctx.attrs.type.replace("-", "_").replace(".", "_")
ctx.actions.run(
cmd_args(script_args, hidden = [cmd, srcs_artifact, out_artifact.as_output()] + hidden),
env = env_vars,
local_only = local_only,
prefer_local = prefer_local,
weight = value_or(ctx.attrs.weight, 1),
allow_cache_upload = cacheable,
category = category,
identifier = identifier,
no_outputs_cleanup = ctx.attrs.no_outputs_cleanup,
always_print_stderr = ctx.attrs.always_print_stderr,
error_handler = genrule_error_handler,
**metadata_args
)
sub_targets = {}
for (k, v) in named_outputs.items():
sub_target_providers = [DefaultInfo(default_outputs = v)]
if executable_outs != None and k in executable_outs:
sub_target_providers.append(RunInfo(args = cmd_args(v)))
sub_targets[k] = sub_target_providers
providers = [DefaultInfo(
default_outputs = default_outputs,
sub_targets = sub_targets,
other_outputs = other_outputs,
)]
# The cxx_genrule also forwards here, and that doesn't have .executable, so use getattr
if getattr(ctx.attrs, "executable", False):
providers.append(RunInfo(args = cmd_args(default_outputs)))
return providers