dependent_targets.bxl•14.5 kB
# TLDR: This is the heart of what's powering our CI: Fletcher Nichol
# It finds and then filters targets in Buck2 and can be ran locally via:
# bxl dependent_targets <flags>
# e.g. bxl dependent_targets --publish_rootfs true --modified_file app/web/Dockerfile
# it will list any applicable target of variant publish-rootfs when the file app/web/Dockerfile
def _dependent_targets_impl(ctx):
targets = _compute_targets(ctx)
filtered = _filter_targets(ctx, targets)
# Print each target string, one per line
for target in filtered:
ctx.output.print(target.label)
dependent_targets = bxl_main(
impl = _dependent_targets_impl,
cli_args = {
"check_doc": cli_args.bool(
default = False,
doc = """Filter targets to testable doc checks.""",
),
"check_format": cli_args.bool(
default = False,
doc = """Filter targets to testable format checks.""",
),
"check_lint": cli_args.bool(
default = False,
doc = """Filter targets to testable lint checks.""",
),
"check_type": cli_args.bool(
default = False,
doc = """Filter targets to testable type checks.""",
),
"test_doc": cli_args.bool(
default = False,
doc = """Filter targets to testable doc tests.""",
),
"test_integration": cli_args.bool(
default = False,
doc = """Filter targets to testable integration tests.""",
),
"test_unit": cli_args.bool(
default = False,
doc = """Filter targets to testable unit tests.""",
),
"release_docker": cli_args.bool(
default = False,
doc = """Filter targets to releasable docker images.""",
),
"promote_docker": cli_args.bool(
default = False,
doc = """Filter targets to runnable docker promotions.""",
),
"publish_binary": cli_args.bool(
default = False,
doc = """Filter targets to binary publish targets.""",
),
"publish_omnibus": cli_args.bool(
default = False,
doc = """Filter targets to omnibus publish targets.""",
),
"publish_rootfs": cli_args.bool(
default = False,
doc = """Filter targets to rootfs publish targets.""",
),
"global_file": cli_args.list(
cli_args.string(),
default = [],
doc = """Source file which impacts the global Buck2 configuration (ex: .buckconfig).""",
),
"buck_file": cli_args.list(
cli_args.string(),
default = [],
doc = """A new or modified `BUCK` file.""",
),
"prelude_file": cli_args.list(
cli_args.string(),
default = [],
doc = """A new or modified `*.bzl` file in a prelude (ex: prelude/defs.bzl).""",
),
"deleted_file": cli_args.list(
cli_args.string(),
default = [],
doc = """A deleted source file underin a BUCK package (ex: lib/foo/src/mod.rs).""",
),
"modified_file": cli_args.list(
cli_args.string(),
default = [],
doc = """A modified source file under a BUCK package (ex: lib/foo/src/mod.rs).""",
),
"rdeps_universe": cli_args.list(
cli_args.string(),
default = [
"root//...",
"prelude-si//...",
"toolchains//...",
],
doc = """A modified source file under a BUCK package (ex: lib/foo/src/mod.rs).""",
),
},
)
# Computes affected targets given various CLI file options.
def _compute_targets(ctx: bxl.Context) -> bxl.UnconfiguredTargetSet:
targets = utarget_set()
# Add affected targets if global project files are provided.
#
# Note: when a global project file is provided, *all* targets are considered affected.
if len(ctx.cli_args.global_file) > 0:
global_targets = _dependent_global_file_targets(ctx)
targets = targets + global_targets
# Add affected targets for all provided `BUCK` files.
#
# Note: when a `BUCK` file has been added or modified, all of its targets and associated reverse
# dependencies are considered affected.
if len(ctx.cli_args.buck_file) > 0:
buck_targets = _dependent_buck_file_targets(ctx, ctx.cli_args.buck_file)
targets = targets + buck_targets
# Add affected targets for all provided prelude files (i.e. `*.bzl` files).
#
# Note: when a `.bzl` file has been added or modified, all of the `BUCK` files which load this
# source and associated reverse dependencies are considered affected.
if len(ctx.cli_args.prelude_file) > 0:
prelude_targets = _dependent_prelude_file_targets(ctx)
targets = targets + prelude_targets
# Add affected targets for all provided deleted files.
#
# Note: for each deleted file, walk up the directory tree to find the nearest `BUCK` file, and
# use each parent `BUCK` file to determine all of its targets and associated reverse
# dependencies.
if len(ctx.cli_args.deleted_file) > 0:
deleted_targets = _dependent_deleted_file_targets(ctx)
targets = targets + deleted_targets
# Add affected targets for all provided added or modified files.
#
# Note: when a source file has been added or modified, all owned targets and associated reverse
# dependencies are considered affected.
if len(ctx.cli_args.modified_file) > 0:
modified_targets = _dependent_modified_file_targets(ctx)
targets = targets + modified_targets
return targets
# Filters a `TargetSet` given CLI boolean filter options.
def _filter_targets(ctx: bxl.Context, targets: bxl.UnconfiguredTargetSet) -> bxl.UnconfiguredTargetSet:
filtered = utarget_set()
has_filtered = False
if ctx.cli_args.check_doc:
has_filtered = True
filtered = filtered + ctx.uquery().attrregexfilter("name", "^check-doc(-.+)?$", targets)
if ctx.cli_args.check_format:
has_filtered = True
filtered = filtered + ctx.uquery().attrregexfilter("name", "^check-format(-.+)?$", targets)
if ctx.cli_args.check_lint:
has_filtered = True
filtered = filtered + ctx.uquery().attrregexfilter("name", "^check-lint(-.+)?$", targets)
if ctx.cli_args.check_type:
has_filtered = True
filtered = filtered + ctx.uquery().attrregexfilter("name", "^check-type(-.+)?$", targets)
if ctx.cli_args.test_doc:
has_filtered = True
filtered = filtered + ctx.uquery().attrregexfilter("name", "^test-doc(-.+)?$", targets)
if ctx.cli_args.test_integration:
has_filtered = True
filtered = filtered + ctx.uquery().attrregexfilter("name", "^test-integration(-.+)?$", targets)
if ctx.cli_args.test_unit:
has_filtered = True
filtered = filtered + ctx.uquery().attrregexfilter("name", "^test-unit(-.+)?$", targets)
if ctx.cli_args.release_docker:
has_filtered = True
filtered = filtered + ctx.uquery().kind("docker_image_release", targets)
if ctx.cli_args.publish_binary:
has_filtered = True
filtered = filtered + ctx.uquery().attrregexfilter("name", "^publish-binary(-.+)?$", targets)
if ctx.cli_args.publish_omnibus:
has_filtered = True
filtered = filtered + ctx.uquery().attrregexfilter("name", "^publish-omnibus(-.+)?$", targets)
if ctx.cli_args.publish_rootfs:
has_filtered = True
filtered = filtered + ctx.uquery().attrregexfilter("name", "^publish-rootfs(-.+)?$", targets)
if ctx.cli_args.promote_docker:
has_filtered = True
filtered = filtered + _filtered_promote_docker_targets(ctx, targets)
# If no filtering was requested, return the full, unfiltered set of targets
if not has_filtered:
filtered = targets
return filtered
# Returns a filtered list of `promote-docker` targets
#
# Every relevant `:release` target should become a `:promote` target as the promotions are triggered
# from merge to the main branch whereas the releases are triggered in the merge queue pipeline. In
# other words, every necessary release has a 1:1 promotion that needs to happen later.
def _filtered_promote_docker_targets(ctx: bxl.Context, targets: bxl.UnconfiguredTargetSet) -> bxl.UnconfiguredTargetSet:
release_targets = ctx.uquery().kind("docker_image_release", targets)
promote_target_strs = map(
lambda e: e.replace(":release", ":promote"),
map(lambda e: str(e.label), release_targets),
)
promote_targets = ctx.unconfigured_targets(promote_target_strs)
return promote_targets
# Returns a filtered list of `omnibus` targets
#
# Every relevant `:release` target should become a `:promote` target as the promotions are triggered
# from merge to the main branch whereas the releases are triggered in the merge queue pipeline. In
# other words, every necessary release has a 1:1 promotion that needs to happen later.
def _filtered_promote_docker_targets(ctx: bxl.Context, targets: bxl.UnconfiguredTargetSet) -> bxl.UnconfiguredTargetSet:
release_targets = ctx.uquery().kind("docker_image_release", targets)
promote_target_strs = map(
lambda e: e.replace(":release", ":promote"),
map(lambda e: str(e.label), release_targets),
)
promote_targets = ctx.unconfigured_targets(promote_target_strs)
return promote_targets
# Computes a list of targets for all targets in the project.
def _dependent_global_file_targets(ctx: bxl.Context) -> bxl.UnconfiguredTargetSet:
results = utarget_set()
for universe in ctx.cli_args.rdeps_universe:
results = results + ctx.uquery().eval(universe)
return results
# Computes a list of targets for all targets affected by given `BUCK` files.
def _dependent_buck_file_targets(ctx: bxl.Context, buck_files: list[str]) -> bxl.UnconfiguredTargetSet:
buck_target_strs = _buck_files_to_targets(buck_files)
return _rdeps_for_targets(ctx, buck_target_strs)
def _dependent_prelude_file_targets(ctx: bxl.Context) -> bxl.UnconfiguredTargetSet:
# TODO(nick,fletcher): we need to figure this out. Use "rbuildfile" maybe?
print("xxx TODO prelude files: {}".format(ctx.cli_args.prelude_file))
return utarget_set()
# Computes a list of target strings for all targets affected by given deleted files.
def _dependent_deleted_file_targets(ctx: bxl.Context) -> bxl.UnconfiguredTargetSet:
buck_files = []
for deleted_file in ctx.cli_args.deleted_file:
if ctx.fs.is_file(deleted_file):
fail("expected deleted file, but it exists:", deleted_file)
buck_file = _find_parent_buck_file(ctx, deleted_file)
if buck_file:
buck_files.append(buck_file)
return _dependent_buck_file_targets(ctx, buck_files)
# Computes a list of targets for all targets affected by given modified files.
def _dependent_modified_file_targets(ctx: bxl.Context) -> bxl.UnconfiguredTargetSet:
modified_files = map(
_normalize_target_str,
filter(lambda e: ctx.fs.is_file(e), ctx.cli_args.modified_file),
)
query = "owner('%s')"
results_set = ctx.uquery().eval(query, query_args = modified_files)
targets = utarget_set()
for results in results_set.values():
targets = targets + results
return _rdeps_for_targets(ctx, map(lambda e: "{}".format(e.label), targets))
# Computes a list of targets which are reverse dependencies of given targets within a universe.
def _rdeps_for_targets(ctx: bxl.Context, targets: list[str]) -> bxl.UnconfiguredTargetSet:
raw_universe = map(_normalize_target_str, ctx.cli_args.rdeps_universe)
universe = ctx.unconfigured_targets(raw_universe)
raw_targets = map(_normalize_target_str, targets)
targets = ctx.unconfigured_targets(raw_targets)
results = ctx.uquery().rdeps(
universe,
targets,
)
return results
# Returns a file path to the nearest `BUCK` file in parent directories of a given file path.
def _find_parent_buck_file(ctx: bxl.Context, deleted_file: str) -> str:
for i in range(1, len(deleted_file.split("/")), 1):
candidate = "{}/{}".format(deleted_file.rsplit("/", i)[0], "BUCK")
if ctx.fs.is_file(candidate):
return candidate
candidate_v2 = "{}/{}".format(deleted_file.rsplit("/", i)[0], "BUCK.v2")
if ctx.fs.is_file(candidate_v2):
return candidate_v2
candidate = "BUCK"
if ctx.fs.is_file(candidate):
return candidate
candidate_v2 = "BUCK.v2"
if ctx.fs.is_file(candidate_v2):
return candidate_v2
return ""
# Returns a normalized/qualified Buck2 target string from a file path, namespaced in the correct
# cell.
def _normalize_target_str(file_str: str) -> str:
if file_str.startswith("prelude-si//"):
return file_str
elif file_str.startswith("prelude-si/"):
return "prelude-si//{}".format(file_str.split("/", 1)[1])
elif file_str.startswith("prelude//"):
return file_str
elif file_str.startswith("prelude/"):
return "prelude//{}".format(file_str.split("/", 1)[1])
elif file_str.startswith("bxl//"):
return file_str
elif file_str.startswith("bxl/"):
return "bxl//{}".format(file_str.split("/", 1)[1])
elif file_str.startswith("toolchains//"):
return file_str
elif file_str.startswith("toolchains/"):
return "toolchains//{}".format(file_str.split("/", 1)[1])
elif file_str.startswith("root//"):
return file_str
elif file_str.startswith("//"):
return "root" + file_str
else:
return "root//{}".format(file_str)
# Returns a normlalized file target string for a path string containing a `BUCK` file.
def _normalize_buck_file_target_str(buck_file_str: str) -> str:
normalized = buck_file_str.rstrip("BUCK.v2").rstrip("BUCK")
if normalized.endswith("//"):
result = normalized + ":"
return result
else:
result = normalized.rstrip("/") + ":"
return result
# Returns a list of target selectors for a given list of `BUCK` files.
def _buck_files_to_targets(buck_files: list[str]) -> list[str]:
return map(_normalize_buck_file_target_str, map(_normalize_target_str, buck_files))