knobs.rs•69 kB
//! Tunable limits and parameters for conductors.
//! These knobs can all be overridden in production by setting a key in Consul
//! at `conductor/<partition-id>/knobs/<knob-name>` and the desired value.
//!
//! See go/knobs-playbook for instructions.
//!
//! Every knob here should have a comment explaining what it's for and the
//! upper/lower bounds if applicable so an oncall engineer can adjust these
//! safely for a backend if needed.
//!
//! When running locally, these knobs can all be overridden with an environment
//! variable.
#![deny(missing_docs)]
use std::{
num::{
NonZeroU32,
NonZeroUsize,
},
sync::LazyLock,
time::Duration,
};
use cmd_util::env::env_config;
use crate::fastrace_helpers::SamplingConfig;
/// This exists solely to allow knobs to have separate defaults for local
/// execution and prod (running in Nomad). Don't export this outside of
/// this module. We assume that if we're running in Nomad, we're in production
/// which for now is always true.
static IS_PROD: LazyLock<bool> = LazyLock::new(|| std::env::var("NOMAD_ALLOC_ID").is_ok());
/// Returns the `local` value if we are running locally, and the `prod` value if
/// we are running in production. Just syntactic sugar to shorten the knob
/// declarations below.
/// Note that it's generally a bad idea to have separate configurations for
/// local development and production as our local test environment won't match
/// what's really running in production, but there are a few knobs where the
/// production default would make local development slower/harder.
fn prod_override<T>(local_value: T, prod_value: T) -> T {
if *IS_PROD {
return prod_value;
}
local_value
}
/// Set a consistent thread stack size regardless of environment. This is
/// 2x Rust's default: https://doc.rust-lang.org/nightly/std/thread/index.html#stack-size
pub static RUNTIME_STACK_SIZE: LazyLock<usize> =
LazyLock::new(|| env_config("RUNTIME_STACK_SIZE", 4 * 1024 * 1024));
/// 0 -> default (number of cores)
pub static RUNTIME_WORKER_THREADS: LazyLock<usize> =
LazyLock::new(|| env_config("RUNTIME_WORKER_THREADS", 0));
/// Disable the Tokio scheduler's LIFO slot optimization, which may
/// help with tail latencies until they improve its implementation.
/// See https://docs.rs/tokio/latest/tokio/runtime/struct.Builder.html#method.disable_lifo_slot.
pub static RUNTIME_DISABLE_LIFO_SLOT: LazyLock<bool> =
LazyLock::new(|| env_config("RUNTIME_DISABLE_LIFO_SLOT", true));
/// Maximum size of the UDF cache. Default 100MiB.
pub static UDF_CACHE_MAX_SIZE: LazyLock<usize> =
LazyLock::new(|| env_config("UDF_CACHE_MAX_SIZE", 104857600));
/// Maximum size of the shared UDF cache in Conductor. Default 1GiB.
pub static SHARED_UDF_CACHE_MAX_SIZE: LazyLock<usize> =
LazyLock::new(|| env_config("SHARED_UDF_CACHE_MAX_SIZE", 1024 * 1048576));
/// How many UDF execution logs to keep in memory.
pub static MAX_UDF_EXECUTION: LazyLock<usize> =
LazyLock::new(|| env_config("MAX_UDF_EXECUTION", 1000));
/// What is the metrics aggregation window for UDF metrics?
pub static UDF_METRICS_BUCKET_WIDTH: LazyLock<Duration> =
LazyLock::new(|| Duration::from_secs(env_config("UDF_METRICS_BUCKET_WIDTH_SECS", 60)));
/// How many UDF metrics buckets do we keep in-memory? This defaults to 60s * 60
/// = 1 hour.
pub static UDF_METRICS_MAX_BUCKETS: LazyLock<usize> =
LazyLock::new(|| env_config("UDF_METRICS_MAX_BUCKETS", 60));
/// Minimum duration to record in a histogram bucket.
pub static UDF_METRICS_MIN_DURATION: LazyLock<Duration> =
LazyLock::new(|| Duration::from_millis(env_config("UDF_METRICS_MIN_DURATION_MS", 1)));
/// Maximum duration to record in a histogram bucket.
pub static UDF_METRICS_MAX_DURATION: LazyLock<Duration> = LazyLock::new(|| {
Duration::from_millis(env_config("UDF_METRICS_MAX_DURATION_MS", 15 * 60 * 1000))
});
/// How many significant figures to store in a histogram bucket.
pub static UDF_METRICS_SIGNIFICANT_FIGURES: LazyLock<u8> =
LazyLock::new(|| env_config("UDF_METRICS_SIGNIFICANT_FIGURES", 2));
/// How often to flush function activity reports to analytics (in seconds).
pub static UDF_ANALYTICS_POLL_TIME: LazyLock<u64> =
LazyLock::new(|| env_config("UDF_ANALYTICS_POLL_TIME", 60));
/// Enables the heap worker memory report.
pub static HEAP_WORKER_PRINT_REPORT: LazyLock<bool> =
LazyLock::new(|| env_config("HEAP_WORKER_PRINT_REPORT", false));
/// How often the heap worker prints a report, if enabled.
pub static HEAP_WORKER_REPORT_INTERVAL_SECONDS: LazyLock<Duration> =
LazyLock::new(|| Duration::from_secs(env_config("HEAP_WORKER_REPORT_INTERVAL_SECONDS", 30)));
/// This is our official action timeout. This is how much the user code
/// should be allowed to run. Note that we buffer some overhead and the actual
/// Node.js process timeout is higher. We also have separate timeout for V8
/// syscalls.
///
/// NOTE: If you update this, make sure to update the actions resource limits in
/// the docs.
pub static ACTION_USER_TIMEOUT: LazyLock<Duration> =
LazyLock::new(|| Duration::from_secs(env_config("ACTIONS_USER_TIMEOUT_SECS", 600)));
/// Max number of rows we will read when calculating document deltas.
pub static DOCUMENT_DELTAS_LIMIT: LazyLock<usize> =
LazyLock::new(|| env_config("DOCUMENT_DELTAS_LIMIT", 128));
/// Max number of rows we will read when calculating snapshot pages.
/// Each document can be up to `::value::MAX_USER_SIZE`
/// Note that this is a pro feature, so we can afford more memory.
pub static SNAPSHOT_LIST_LIMIT: LazyLock<usize> =
LazyLock::new(|| env_config("SNAPSHOT_LIST_LIMIT", 1024));
/// The size of the log manager's event receive buffer.
pub static LOG_MANAGER_EVENT_RECV_BUFFER_SIZE: LazyLock<usize> =
LazyLock::new(|| env_config("LOG_MANAGER_EVENT_RECV_BUFFER_SIZE", 4096));
/// The aggregation interval at which the log manager empties its event receiver
/// buffer.
pub static LOG_MANAGER_AGGREGATION_INTERVAL_MILLIS: LazyLock<u64> =
LazyLock::new(|| env_config("LOG_MANAGER_AGGREGATION_INTERVAL", 5000));
/// Max number of times a mutation can retry due to OCC conflicts.
pub static UDF_EXECUTOR_OCC_MAX_RETRIES: LazyLock<usize> =
LazyLock::new(|| env_config("UDF_EXECUTOR_OCC_MAX_RETRIES", 4));
/// Initial backoff when we encounter an OCC conflict.
pub static UDF_EXECUTOR_OCC_INITIAL_BACKOFF: LazyLock<Duration> =
LazyLock::new(|| Duration::from_millis(env_config("UDF_EXECUTOR_OCC_INITIAL_BACKOFF_MS", 10)));
/// Maximum expontial backoff when facing repeated OCC conflicts.
pub static UDF_EXECUTOR_OCC_MAX_BACKOFF: LazyLock<Duration> =
LazyLock::new(|| Duration::from_millis(env_config("UDF_EXECUTOR_OCC_MAX_BACKOFF_MS", 2000)));
/// The time for which a backend will stay around, after getting preempted,
/// answering health checks but not serving traffic.
///
/// This MUST be set to >= the `healthy_deadline` in backend.nomad. Otherwise
/// a pre-emption loop can be triggered. The old backend process dies due to
/// this timeout, and then is restarted by nomad because the new deployment
/// hasn't become healthy yet. The restarted old process then pre-empts the new
/// process, causing it to die and be restarted etc.
pub static LEASE_LOST_COOL_DOWN: LazyLock<Duration> = LazyLock::new(|| {
Duration::from_secs(env_config(
"LEASE_LOST_COOL_DOWN_SECS",
prod_override(5, 130),
))
});
/// How long the queue must be nonempty before we consider traffic to be
/// "congested" and start shedding traffic. When we are idle (not congested) it
/// is how long each request can live in the queue.
pub static CODEL_QUEUE_IDLE_EXPIRATION_MILLIS: LazyLock<Duration> =
LazyLock::new(|| Duration::from_millis(env_config("CODEL_QUEUE_IDLE_EXPIRATION_MILLIS", 5000)));
/// How long each request can live in the queue when we are congested.
pub static CODEL_QUEUE_CONGESTED_EXPIRATION_MILLIS: LazyLock<Duration> = LazyLock::new(|| {
Duration::from_millis(env_config("CODEL_QUEUE_CONGESTED_EXPIRATION_MILLIS", 50))
});
/// Default page size (in number of docuemnts) used when loading documents from
/// the database.
pub static DEFAULT_DOCUMENTS_PAGE_SIZE: LazyLock<u32> =
LazyLock::new(|| env_config("DEFAULT_DOCUMENTS_PAGE_SIZE", 100));
/// Maximum number of documents it's okay to load into memory at once.
/// Note each document can be up to `::value::MAX_SIZE`.
pub static DOCUMENTS_IN_MEMORY: LazyLock<usize> =
LazyLock::new(|| env_config("DOCUMENTS_IN_MEMORY", 512));
/// Length of the HTTP server TCP backlog.
pub static HTTP_SERVER_TCP_BACKLOG: LazyLock<u32> =
LazyLock::new(|| env_config("HTTP_SERVER_TCP_BACKLOG", 256));
/// The max concurrent of concurrent HTTP requests. This also limits Node.js
/// action callbacks concurrency since those go over http.
pub static HTTP_SERVER_MAX_CONCURRENT_REQUESTS: LazyLock<usize> =
LazyLock::new(|| env_config("HTTP_SERVER_MAX_CONCURRENT_REQUESTS", 1024));
/// Max number of user writes in a transaction. Make sure to also increase
/// `MAX_INSERT_SIZE` in mysql/src/lib.rs and postgres/src/lib.rs.
pub static TRANSACTION_MAX_NUM_USER_WRITES: LazyLock<usize> =
LazyLock::new(|| env_config("TRANSACTION_MAX_NUM_USER_WRITES", 16000));
/// Max size of user writes in a transaction, in bytes
pub static TRANSACTION_MAX_USER_WRITE_SIZE_BYTES: LazyLock<usize> = LazyLock::new(|| {
env_config("TRANSACTION_MAX_USER_WRITE_SIZE_BYTES", 1 << 24) // 16 MiB
});
/// SnapshotManager maintains a bounded time range of versions,
/// determined by `MAX_TRANSACTION_WINDOW`, allowing the `Database` layer to
/// begin a transaction in any timestamp within that range.
pub static MAX_TRANSACTION_WINDOW: LazyLock<Duration> =
LazyLock::new(|| Duration::from_secs(env_config("MAX_TRANSACTION_WINDOW_SECONDS", 10)));
/// Maximum size in bytes of arguments to a function.
pub static FUNCTION_MAX_ARGS_SIZE: LazyLock<usize> = LazyLock::new(|| {
env_config("FUNCTION_MAX_ARGS_SIZE", 1 << 24) // 16 MiB
});
/// Maximum size in bytes of the result of a function.
pub static FUNCTION_MAX_RESULT_SIZE: LazyLock<usize> = LazyLock::new(|| {
env_config("FUNCTION_MAX_RESULT_SIZE", 1 << 24) // 16 MiB
});
/// When a function exceeds FUNCTION_LIMIT_WARNING_RATIO * a corresponding
/// limit value, we add a warning log line.
pub static FUNCTION_LIMIT_WARNING_RATIO: LazyLock<f64> = LazyLock::new(|| {
env_config("FUNCTION_LIMIT_WARNING_RATIO", 0.8) // 80%
});
/// We might generate a number of system documents for each UDF write. For
/// example, creating 4000 user documents in new tables, might result in adding
/// an additional 8000 system documents. If we hit this error, this is a system
/// error, not a developer one. If you increase this value, make sure to also
/// increase MAX_INSERT_SIZE in mysql/src/lib.rs and postgres/src/lib.rs.
pub static TRANSACTION_MAX_SYSTEM_NUM_WRITES: LazyLock<usize> =
LazyLock::new(|| env_config("TRANSACTION_MAX_SYSTEM_NUM_WRITES", 40000));
/// We write user modules in system tables and those can get quite large.
/// Similar to the above if we hit this limit, we should count this as system
/// error and do a use case specific validation to avoid hitting this.
pub static TRANSACTION_MAX_SYSTEM_WRITE_SIZE_BYTES: LazyLock<usize> = LazyLock::new(|| {
env_config("TRANSACTION_MAX_SYSTEM_WRITE_SIZE_BYTES", 1 << 27) // 128 MiB
});
/// Maximum number of scheduled transactions.
pub static TRANSACTION_MAX_NUM_SCHEDULED: LazyLock<usize> =
LazyLock::new(|| env_config("TRANSACTION_MAX_NUM_SCHEDULED", 1000));
/// Maximum number of scheduled jobs to cancel in a single transaction.
pub static MAX_JOBS_CANCEL_BATCH: LazyLock<usize> =
LazyLock::new(|| env_config("MAX_JOBS_CANCEL_BATCH", 1000));
/// Maximum size of the arguments to a scheduled function.
pub static TRANSACTION_MAX_SCHEDULED_TOTAL_ARGUMENT_SIZE_BYTES: LazyLock<usize> =
LazyLock::new(|| {
env_config(
"TRANSACTION_MAX_SCHEDULED_TOTAL_ARGUMENT_SIZE_BYTES",
1 << 24,
) // 16 MiB
});
/// Number of scheduled jobs that can execute in parallel.
// Note that the current algorithm for executing ready jobs has up to
// SCHEDULED_JOB_EXECUTION_PARALLELISM overhead for every executed job, so we
// don't want to set this number too high.
pub static SCHEDULED_JOB_EXECUTION_PARALLELISM: LazyLock<usize> =
LazyLock::new(|| env_config("SCHEDULED_JOB_EXECUTION_PARALLELISM", 10));
/// Initial backoff in milliseconds on a system error from a scheduled job.
pub static SCHEDULED_JOB_INITIAL_BACKOFF: LazyLock<Duration> =
LazyLock::new(|| Duration::from_millis(env_config("SCHEDULED_JOB_INITIAL_BACKOFF_MS", 500)));
/// Max backoff in seconds on a system error from a scheduled job.
/// Scheduled jobs can hit many OCCs, so we may need to slow them down if they
/// hit errors repeatedly.
pub static SCHEDULED_JOB_MAX_BACKOFF: LazyLock<Duration> = LazyLock::new(|| {
Duration::from_secs(env_config("SCHEDULED_JOB_MAX_BACKOFF_SECS", 2 * 60 * 60))
});
/// Initial backoff in milliseconds on a system error from the scheduled job
/// garbage collector.
pub static SCHEDULED_JOB_GARBAGE_COLLECTION_INITIAL_BACKOFF: LazyLock<Duration> =
LazyLock::new(|| {
Duration::from_millis(env_config(
"SCHEDULED_JOB_GARBAGE_COLLECTION_INITIAL_BACKOFF_MS",
10,
))
});
/// Max backoff in seconds on a system error from the scheduled job garbage
/// collector.
pub static SCHEDULED_JOB_GARBAGE_COLLECTION_MAX_BACKOFF: LazyLock<Duration> = LazyLock::new(|| {
Duration::from_secs(env_config(
"SCHEDULED_JOB_GARBAGE_COLLECTION_MAX_BACKOFF_SECS",
30,
))
});
/// How long completed scheduled jobs are kept before getting garbage collected.
pub static SCHEDULED_JOB_RETENTION: LazyLock<Duration> = LazyLock::new(|| {
Duration::from_secs(env_config(
"SCHEDULED_JOB_RETENTION",
60 * 60 * 24 * 7, // 1 week
))
});
/// Maximum number of scheduled jobs to garbage collect in a single transaction
pub static SCHEDULED_JOB_GARBAGE_COLLECTION_BATCH_SIZE: LazyLock<usize> =
LazyLock::new(|| env_config("SCHEDULED_JOB_GARBAGE_COLLECTION_BATCH_SIZE", 1000));
/// Delay between runs of the scheduled job garbage collector.
/// If too low, the garbage collector will run frequently with small batches,
/// which is less efficient. If too high, the garbage collector might fall
/// behind.
pub static SCHEDULED_JOB_GARBAGE_COLLECTION_DELAY: LazyLock<Duration> =
LazyLock::new(|| Duration::from_secs(env_config("SCHEDULED_JOB_GARBAGE_COLLECTION_DELAY", 10)));
/// Maximum number of syscalls that can run in a batch together when
/// awaited in parallel. Higher values improve latency, while lower ones
/// protect one isolate from hogging database connections.
pub static MAX_SYSCALL_BATCH_SIZE: LazyLock<usize> =
LazyLock::new(|| env_config("MAX_SYSCALL_BATCH_SIZE", 16));
/// Maximum depth of query/mutation -> query/mutation calls within the reactor.
/// We put a low limit on this for now so users with infinite loops won't starve
/// all of the threads on a single node.
pub static MAX_REACTOR_CALL_DEPTH: LazyLock<usize> =
LazyLock::new(|| env_config("MAX_REACTOR_CALL_DEPTH", 8));
/// Default number of records to fetch from an index if a prefetch hint is not
/// provided.
pub static DEFAULT_QUERY_PREFETCH: LazyLock<usize> =
LazyLock::new(|| env_config("DEFAULT_QUERY_PREFETCH", 100));
/// Number of rows that can be read in a transaction.
pub static TRANSACTION_MAX_READ_SIZE_ROWS: LazyLock<usize> =
LazyLock::new(|| env_config("TRANSACTION_MAX_READ_SIZE_ROWS", 32000));
/// Number of bytes that can be read in a transaction.
pub static TRANSACTION_MAX_READ_SIZE_BYTES: LazyLock<usize> = LazyLock::new(|| {
env_config("TRANSACTION_MAX_READ_SIZE_BYTES", 1 << 24) // 16 MiB
});
/// Maximum number of intervals that can be read in a transaction.
pub static TRANSACTION_MAX_READ_SET_INTERVALS: LazyLock<usize> =
LazyLock::new(|| env_config("TRANSACTION_MAX_READ_SET_INTERVALS", 4096));
/// Intervals that can be read in a transaction before warning.
pub static TRANSACTION_WARN_READ_SET_INTERVALS: LazyLock<usize> =
LazyLock::new(|| env_config("TRANSACTION_WARN_READ_SET_INTERVALS", 3072));
/// Write max_repeatable_ts if there have been no commits for this duration.
/// The actual period is jittered between 1x and 2x this value.
pub static MAX_REPEATABLE_TIMESTAMP_IDLE_FREQUENCY: LazyLock<Duration> = LazyLock::new(|| {
Duration::from_secs(env_config(
"MAX_REPEATABLE_TIMESTAMP_IDLE_FREQUENCY",
60 * 60,
))
});
/// This is the max duration between a Commit and bumping max_repeatable_ts.
/// When reading from a follower persistence, we can only read commits at
/// timestamps <= max_repeatable_ts (because commits > max_repeatable_ts are
/// actively being written), so this is the delay between a commit and the
/// commit being visible from db-verifier and other follower reads.
pub static MAX_REPEATABLE_TIMESTAMP_COMMIT_DELAY: LazyLock<Duration> =
LazyLock::new(|| Duration::from_secs(env_config("MAX_REPEATABLE_TIMESTAMP_COMMIT_DELAY", 5)));
/// The maximum delay between runs of retention, this is now only used for error
/// backoff and the initial delay when backend is started.
pub static MAX_RETENTION_DELAY_SECONDS: LazyLock<Duration> =
LazyLock::new(|| Duration::from_secs(env_config("RETENTION_DELETE_FREQUENCY", 60)));
/// How many parallel threads to use for deleting index entries that have
/// expired.
pub static INDEX_RETENTION_DELETE_PARALLEL: LazyLock<usize> =
LazyLock::new(|| env_config("INDEX_RETENTION_DELETE_PARALLEL", 4));
/// How many parallel threads to use for deleting document log entries that have
/// expired.
pub static DOCUMENT_RETENTION_DELETE_PARALLEL: LazyLock<usize> =
LazyLock::new(|| env_config("DOCUMENT_RETENTION_DELETE_PARALLEL", 1));
/// INDEX_RETENTION_DELAY determines the size of the index retention window.
///
/// Larger window means we keep around old snapshots for longer, which can cause
/// performance problems in UDFs if there are many tombstones.
///
/// Smaller window means we break snapshot reads faster.
pub static INDEX_RETENTION_DELAY: LazyLock<Duration> =
LazyLock::new(|| Duration::from_secs(env_config("INDEX_RETENTION_DELAY", 4 * 60)));
/// DOCUMENT_RETENTION_DELAY determines the size of the document retention
/// window.
///
/// Larger window means we keep around longer windows for our write ahead log
/// to be valid.
///
/// Smaller window means we keep less historical data around.
pub static DOCUMENT_RETENTION_DELAY: LazyLock<Duration> = LazyLock::new(|| {
Duration::from_secs(env_config("DOCUMENT_RETENTION_DELAY", 60 * 60 * 24 * 80))
});
/// When to start rejecting new additions to the search memory index.
pub static TEXT_INDEX_SIZE_HARD_LIMIT: LazyLock<usize> =
LazyLock::new(|| env_config("SEARCH_INDEX_SIZE_HARD_LIMIT", 100 * (1 << 20))); // 100 MiB
/// When to start rejecting new additions to the vector memory index.
/// Because they're closely related, this is also used by the vector compaction
/// worker to determine the largest size for a "small" segment. Small segments
/// are merged more aggressively by the compaction worker than large segments.
pub static VECTOR_INDEX_SIZE_HARD_LIMIT: LazyLock<usize> =
LazyLock::new(|| env_config("VECTOR_INDEX_SIZE_HARD_LIMIT", 100 * (1 << 20))); // 100 MiB
/// Whether indexes will be backfilled. Likely only disabled if index backfill
/// is breaking an instance.
pub static ENABLE_INDEX_BACKFILL: LazyLock<bool> =
LazyLock::new(|| env_config("INDEX_BACKFILL_ENABLE", true));
/// Number of index chunks processed per second during a backfill.
pub static INDEX_BACKFILL_CHUNK_RATE: LazyLock<usize> =
LazyLock::new(|| env_config("INDEX_BACKFILL_CHUNK_RATE", 8));
/// How many index entries to write within a single database transaction.
/// Value is a tradeoff between grouping work, vs tying up resources on the
/// database, vs holding all entries in memory.
pub static INDEX_BACKFILL_CHUNK_SIZE: LazyLock<usize> =
LazyLock::new(|| env_config("INDEX_BACKFILL_CHUNK_SIZE", 256));
/// Number of workers to use for index backfill.
pub static INDEX_BACKFILL_WORKERS: LazyLock<usize> =
LazyLock::new(|| env_config("INDEX_BACKFILL_WORKERS", 1));
/// Chunk size of index entries for deleting from Persistence.
pub static INDEX_RETENTION_DELETE_CHUNK: LazyLock<usize> =
LazyLock::new(|| env_config("INDEX_RETENTION_DELETE_CHUNK", 512));
/// Chunk size of documents for deleting from Persistence.
pub static DOCUMENT_RETENTION_DELETE_CHUNK: LazyLock<usize> =
LazyLock::new(|| env_config("DOCUMENT_RETENTION_DELETE_CHUNK", 256));
/// Batch size of index entries to delete between checkpoints.
pub static RETENTION_DELETE_BATCH: LazyLock<usize> =
LazyLock::new(|| env_config("RETENTION_DELETE_BATCH", 10000));
/// Whether retention deletes are enabled.
pub static RETENTION_DELETES_ENABLED: LazyLock<bool> =
LazyLock::new(|| env_config("RETENTION_DELETES_ENABLED", true));
/// Whether retention document deletes are enabled.
pub static RETENTION_DOCUMENT_DELETES_ENABLED: LazyLock<bool> =
LazyLock::new(|| env_config("RETENTION_DOCUMENT_DELETES_ENABLED", true));
/// Enable or disable failing insert/update/deletes when retention is behind.
pub static RETENTION_FAIL_ENABLED: LazyLock<bool> =
LazyLock::new(|| env_config("RETENTION_FAIL_ENABLED", false));
/// Insert/update/delete will start to fail if retention is retention window *
/// this value behind (e.g. 4 * 20 = 1 hour 20 minutes)
pub static RETENTION_FAIL_START_MULTIPLIER: LazyLock<usize> =
LazyLock::new(|| env_config("RETENTION_FAIL_START_MULTIPLIER", 20));
/// All insert/update/deletes will if retention is retention window * this value
/// behind (e.g. 4 * 40 = 2 hours and 4 minutes).
pub static RETENTION_FAIL_ALL_MULTIPLIER: LazyLock<usize> =
LazyLock::new(|| env_config("RETENTION_FAIL_ALL_MULTIPLIER", 40));
/// Time in between batches of deletes for document retention. This value is
/// also used to jitter document retention on startup to avoid a thundering
/// herd.
pub static DOCUMENT_RETENTION_BATCH_INTERVAL_SECONDS: LazyLock<Duration> = LazyLock::new(|| {
Duration::from_secs_f64(env_config(
"DOCUMENT_RETENTION_BATCH_INTERVAL_SECONDS",
60.0,
))
});
/// Documents-per-second rate limit for document retention
/// Note that while this serves as an upper bound, retention speed is mostly
/// limited by `DOCUMENT_RETENTION_BATCH_INTERVAL_SECONDS`,
/// `DOCUMENT_RETENTION_DELETE_CHUNK`, and `DOCUMENT_RETENTION_DELETE_PARALLEL`
pub static DOCUMENT_RETENTION_RATE_LIMIT: LazyLock<NonZeroU32> = LazyLock::new(|| {
env_config(
"DOCUMENT_RETENTION_RATE_LIMIT",
NonZeroU32::new(1024).unwrap(),
)
});
/// Maximum scanned documents within a single run for document retention unless
/// there are a bunch of writes at single timestamp. Then, we go until there are
/// no more writes at that timestamp.
pub static DOCUMENT_RETENTION_MAX_SCANNED_DOCUMENTS: LazyLock<usize> =
LazyLock::new(|| env_config("DOCUMENT_RETENTION_MAX_SCANNED_DOCUMENTS", 10000));
/// Size at which a search index will be queued for snapshotting.
pub static SEARCH_INDEX_SIZE_SOFT_LIMIT: LazyLock<usize> =
LazyLock::new(|| env_config("SEARCH_INDEX_SIZE_SOFT_LIMIT", 10 * (1 << 20))); // 10 MiB
/// Configures the search index worker's rate limit on pages processed per
/// second.
pub static SEARCH_INDEX_WORKER_PAGES_PER_SECOND: LazyLock<NonZeroU32> = LazyLock::new(|| {
env_config(
"SEARCH_INDEX_WORKER_PAGES_PER_SECOND",
NonZeroU32::new(2).unwrap(),
)
});
/// Don't allow database workers to have more than an hour of uncheckpointed
/// data.
///
/// For search/vector index workers - Note that fast-forwarding will keep the
/// index's timestamp up-to-date if its table hasn't had any writes. This isn't
/// perfect since ideally we'd bound the number and total size of log entries
/// read for bootstrapping, but it's good enough until we have better commit
/// statistics that aren't reset at restart. It's still expensive to walk the
/// DocumentRevisionStream to build new segments, so this value needs to be low
/// enough to not block the search index flushers for too long, or else writes
/// will start failing. This is why we set this value lower for pro users (10m).
pub static DATABASE_WORKERS_MAX_CHECKPOINT_AGE: LazyLock<Duration> =
LazyLock::new(|| Duration::from_secs(env_config("DATABASE_WORKERS_MAX_CHECKPOINT_AGE", 3600)));
/// Don't fast-forward an index less than ten seconds forward so we don't
/// amplify every commit into another write when the system is under heavy load.
pub static DATABASE_WORKERS_POLL_INTERVAL: LazyLock<Duration> =
LazyLock::new(|| Duration::from_secs(env_config("DATABASE_WORKERS_POLL_INTERVAL", 20)));
/// When the persisted table summary is within this threshold of the current
/// timestamp, we'll tell the committer to process any remaining writes and
/// finish the bootstrap.
pub static TABLE_SUMMARY_BOOTSTRAP_RECENT_THRESHOLD: LazyLock<Duration> = LazyLock::new(|| {
Duration::from_secs(env_config(
"TABLE_SUMMARY_BOOTSTRAP_RECENT_THRESHOLD_SECS",
10,
))
});
/// The minimum time to retain the WriteLog, note that we will never retain for
/// less, even if WRITE_LOG_SOFT_MAX_SIZE_BYTES is exceeded.
pub static WRITE_LOG_MIN_RETENTION_SECS: LazyLock<Duration> =
LazyLock::new(|| Duration::from_secs(env_config("WRITE_LOG_MIN_RETENTION_SECS", 30)));
/// The maximum time to retain the WriteLog, note that the WriteLog might be
/// trimmed sooner if it size exceeds WRITE_LOG_MAX_SIZE_BYTES.
pub static WRITE_LOG_MAX_RETENTION_SECS: LazyLock<Duration> =
LazyLock::new(|| Duration::from_secs(env_config("WRITE_LOG_MAX_RETENTION_SECS", 300)));
/// The maximum size of the write log. Notes:
/// - the write log will be trimmed based on WRITE_LOG_MAX_RETENTION_SECS, and
/// thus it might never reach this size.
/// - the write log always retains at least WRITE_LOG_MIN_RETENTION_SECS, and
/// thus we could exceed this limit. The reason we do that is to allow some
/// minimum buffer for queries to refresh after execution.
pub static WRITE_LOG_SOFT_MAX_SIZE_BYTES: LazyLock<usize> =
LazyLock::new(|| env_config("WRITE_LOG_SOFT_MAX_SIZE_BYTES", 50 * 1024 * 1024));
/// How frequently system tables are cleaned up.
pub static SYSTEM_TABLE_CLEANUP_FREQUENCY: LazyLock<Duration> = LazyLock::new(|| {
Duration::from_secs(env_config(
"SYSTEM_TABLE_CLEANUP_FREQUENCY_SECONDS",
30 * 60,
))
});
/// Number of rows fetched and potentially deleted in a single transaction.
pub static SYSTEM_TABLE_CLEANUP_CHUNK_SIZE: LazyLock<usize> =
LazyLock::new(|| env_config("SYSTEM_TABLE_CLEANUP_CHUNK_SIZE", 64));
/// Maximum number of rows deleted per second.
/// This should not exceed the maximum rate that retention can process
/// tombstones, which is about 300.
pub static SYSTEM_TABLE_ROWS_PER_SECOND: LazyLock<NonZeroU32> = LazyLock::new(|| {
env_config(
"SYSTEM_TABLE_CLEANUP_ROWS_PER_SECOND",
NonZeroU32::new(256).unwrap(),
)
});
/// We can potentially reduce this window by changing
/// clients to track how long they have been open and throw an alert after
/// too many days. See go/idempotent-mutations
/// Zero indicates that there is no maximum.
pub static MAX_SESSION_CLEANUP_DURATION: LazyLock<Option<Duration>> = LazyLock::new(|| {
let hours = env_config("MAX_SESSION_CLEANUP_DURATION_HOURS", 2 * 7 * 24);
if hours > 0 {
Some(Duration::from_secs(60 * 60 * hours))
} else {
None
}
});
/// Number of concurrent commits to use for deleting session requests.
pub static SESSION_CLEANUP_DELETE_CONCURRENCY: LazyLock<usize> =
LazyLock::new(|| env_config("SESSION_CLEANUP_DELETE_CONCURRENCY", 2));
/// Snapshots that expired more than this number of days ago are purged
/// from storage.
pub static MAX_EXPIRED_SNAPSHOT_AGE: LazyLock<Duration> = LazyLock::new(|| {
let days = env_config("MAX_EXPIRED_SNAPSHOT_AGE_DAYS", 30);
Duration::from_days(days)
});
/// Number of chunks processed per second when calculating table summaries.
pub static TABLE_SUMMARY_CHUNKS_PER_SECOND: LazyLock<NonZeroU32> = LazyLock::new(|| {
env_config(
"TABLE_SUMMARY_CHUNKS_PER_SECOND",
NonZeroU32::new(1000).unwrap(),
)
});
/// Size at which a vector index will be queued for snapshotting vector indexes.
pub static VECTOR_INDEX_SIZE_SOFT_LIMIT: LazyLock<usize> =
LazyLock::new(|| env_config("VECTOR_INDEX_SIZE_SOFT_LIMIT", 30 * (1 << 20))); // 30 MiB
/// Max number of threads used to build a disk index.
pub static VECTOR_INDEX_THREADS: LazyLock<usize> =
LazyLock::new(|| env_config("VECTOR_INDEX_THREADS", 4));
/// Configures the vector and search index workers' rate limit on pages
/// processed per second. This is the default rate limit for anything a user
/// might be waiting on. It's initialized high enough that it effectively does
/// not rate limit. We keep the knob so that in an emergency we can reduce it.
///
/// NOTE: This number is multiplied with DEFAULT_DOCUMENTS_PAGE_SIZE, do not
/// set this value to such a larg number that doing so will overflow!
pub static SEARCH_WORKER_PAGES_PER_SECOND: LazyLock<NonZeroU32> = LazyLock::new(|| {
env_config(
"SEARCH_WORKER_PAGES_PER_SECOND",
NonZeroU32::new(1000).unwrap(),
)
});
/// Configures the vector and search index workers' rate limit on pages
/// processed per second for non-user facing rebuilds (well mostly non-user
/// facing, a user facing backfill might get stuck behind one of these).
///
/// The default is low so that an inadvertent metadata change or a deliberate
/// index backfill does not cause a thundering herd.
///
/// NOTE: This number is multiplied with DEFAULT_DOCUMENTS_PAGE_SIZE, do not
/// set this value to such a larg number that doing so will overflow!
pub static SEARCH_WORKER_PASSIVE_PAGES_PER_SECOND: LazyLock<NonZeroU32> = LazyLock::new(|| {
env_config(
"SEARCH_WORKER_PASSIVE_PAGES_PER_SECOND",
NonZeroU32::new(10).unwrap(),
)
});
/// Default page size (in number of docuemnts) used when loading documents from
/// the database for building a vector index.
pub static VECTOR_INDEX_WORKER_PAGE_SIZE: LazyLock<usize> =
LazyLock::new(|| env_config("VECTOR_INDEX_WORKER_PAGE_SIZE", 128));
/// Timeout on "user time" spent during a UDF.
pub static DATABASE_UDF_USER_TIMEOUT: LazyLock<Duration> =
LazyLock::new(|| Duration::from_secs(env_config("DATABASE_UDF_USER_TIMEOUT_SECONDS", 1)));
/// Timeout on the "system time" during a UDF -- i.e. syscalls.
// The user limits are not very tight, which requires us to have a high
// syscall timeout. When the database is healthy, we should never have UDF
// run into SYSTEM_TIMEOUT. UDFs are allowed to do up to 4096 unique queries,
// which at 1.6ms average, will take 6.4 seconds to run in sequence. We should
// aim to lower the SYSTEM_TIMEOUT limit over time by adding real parallelism
// and adding limit on number of `awaits` which is lower than the number of
// queries.
pub static DATABASE_UDF_SYSTEM_TIMEOUT: LazyLock<Duration> =
LazyLock::new(|| Duration::from_secs(env_config("DATABASE_UDF_SYSTEM_TIMEOUT_SECONDS", 15)));
/// Timeout on the time it takes to analyze code during a push.
pub static ISOLATE_ANALYZE_USER_TIMEOUT: LazyLock<Duration> =
LazyLock::new(|| Duration::from_secs(env_config("ISOLATE_ANALYZE_USER_TIMEOUT_SECONDS", 2)));
/// Increasing the size of the queue helps us deal with bursty requests. This is
/// a CoDel queue [https://queue.acm.org/detail.cfm?id=2209336], which will
/// switch from FIFO to LIFO queue when overloaded, in order to process as much
/// as possible and avoid a congestion collapse. The primary downside of
/// increase this is memory usage from the UDF arguments.
pub static ISOLATE_QUEUE_SIZE: LazyLock<usize> =
LazyLock::new(|| env_config("ISOLATE_QUEUE_SIZE", 2000));
/// The size of the pending commits in the committer queue. This is a FIFO
/// queue, so if the queue is too large, we run into a risk of all requests
/// waiting too long and no requests going through during overload. The size of
/// each commit request is also typically larger than a isolate request. For
/// time being, allow 128 slots, which is the maximum number of isolate threads
/// in any process.
pub static COMMITTER_QUEUE_SIZE: LazyLock<usize> =
LazyLock::new(|| env_config("COMMITTER_QUEUE_SIZE", 128));
/// 0 -> default (number of cores)
pub static V8_THREADS: LazyLock<u32> = LazyLock::new(|| env_config("V8_THREADS", 0));
/// If false, each UDF runs in its own isolate with its own heap.
/// If true, each UDF runs in the same isolate in its own context, sharing a
/// heap.
pub static REUSE_ISOLATES: LazyLock<bool> = LazyLock::new(|| env_config("REUSE_ISOLATES", true));
/// Duration in seconds before an idle isolate is recreated
pub static ISOLATE_IDLE_TIMEOUT: LazyLock<Duration> =
LazyLock::new(|| Duration::from_secs(env_config("ISOLATE_IDLE_TIMEOUT_SECONDS", 600)));
/// The maximum amount of time an isolate can be used before being recreated.
pub static ISOLATE_MAX_LIFETIME: LazyLock<Duration> =
LazyLock::new(|| Duration::from_secs(env_config("ISOLATE_MAX_LIFETIME_SECONDS", 60 * 60)));
/// System timeout for V8 actions.
/// This doesn't count most syscalls, but it does count module loading.
pub static V8_ACTION_SYSTEM_TIMEOUT: LazyLock<Duration> =
LazyLock::new(|| Duration::from_secs(env_config("V8_ACTION_SYSTEM_TIMEOUT_SECONDS", 5 * 60)));
/// The maximum amount of time
pub static APPLICATION_FUNCTION_RUNNER_SEMAPHORE_TIMEOUT: LazyLock<Duration> =
LazyLock::new(|| {
Duration::from_millis(env_config(
"APPLICATION_FUNCTION_RUNNER_SEMAPHORE_TIMEOUT",
5000,
))
});
/// Default max function concurrency limit for basic plan instances.
/// This value is used as the default for all APPLICATION_MAX_CONCURRENT
/// constants and is also used to determine if an instance is on a pro plan.
pub static DEFAULT_APPLICATION_MAX_FUNCTION_CONCURRENCY: usize = 16;
/// The maximum number of queries that can be run concurrently by an
/// application.
///
/// This is a per backend limit applied before FunctionRunner implementations.
///
/// The value here may be overridden by big brain.
pub static APPLICATION_MAX_CONCURRENT_QUERIES: LazyLock<usize> = LazyLock::new(|| {
env_config(
"APPLICATION_MAX_CONCURRENT_QUERIES",
DEFAULT_APPLICATION_MAX_FUNCTION_CONCURRENCY,
)
});
/// The maximum number of mutations that can be run concurrently by an
/// application.
///
/// This is a per backend limit applied before FunctionRunner implementations.
///
/// The value here may be overridden by big brain.
pub static APPLICATION_MAX_CONCURRENT_MUTATIONS: LazyLock<usize> = LazyLock::new(|| {
env_config(
"APPLICATION_MAX_CONCURRENT_MUTATIONS",
DEFAULT_APPLICATION_MAX_FUNCTION_CONCURRENCY,
)
});
/// The maximum number of v8 actions that can be run concurrently by an
/// application.
///
/// This is a higher level limit applied before FunctionRunner implementations.
///
/// This does NOT apply to:
/// 1. Http actions
/// 2. Node actions
///
/// Node actions are limited by the APPLICATION_MAX_CONCURRENT_NODE_ACTIONS
/// knob. Http actions are limited by APPLICATION_MAX_CONCURRENT_HTTP_ACTIONS
/// knob.
///
/// The value here may be overridden by big brain.
pub static APPLICATION_MAX_CONCURRENT_V8_ACTIONS: LazyLock<usize> = LazyLock::new(|| {
env_config(
"APPLICATION_MAX_CONCURRENT_V8_ACTIONS",
DEFAULT_APPLICATION_MAX_FUNCTION_CONCURRENCY,
)
});
/// The maximum number of node actions that can be run concurrently by an
/// application
///
/// Node actions are not sent through FunctionRunner implementations, so this is
/// a limit on the number of actions sent to AWS. AWS also has a global maximum
/// number of total concurrent actions across all backends. If we hit the AWS
/// limit, we'll see 429 error responses for node actions.
///
/// The value here may be overridden by big brain.
pub static APPLICATION_MAX_CONCURRENT_NODE_ACTIONS: LazyLock<usize> = LazyLock::new(|| {
env_config(
"APPLICATION_MAX_CONCURRENT_NODE_ACTIONS",
DEFAULT_APPLICATION_MAX_FUNCTION_CONCURRENCY,
)
});
/// Number of threads to execute V8 actions.
///
/// Http actions are not sent through FunctionRunner implementations. This is a
/// maximum on the number of http actions that will be executed in process in a
/// particular backend.
///
/// The value here may be overridden by big brain.
pub static APPLICATION_MAX_CONCURRENT_HTTP_ACTIONS: LazyLock<usize> = LazyLock::new(|| {
env_config(
"APPLICATION_MAX_CONCURRENT_HTTP_ACTIONS",
if cfg!(any(test, feature = "testing")) {
2
} else {
DEFAULT_APPLICATION_MAX_FUNCTION_CONCURRENCY
},
)
});
/// The maximum number of concurrent package uploads during
/// `/api/deploy2/start_push`.
pub static APPLICATION_MAX_CONCURRENT_UPLOADS: LazyLock<usize> =
LazyLock::new(|| env_config("APPLICATION_MAX_CONCURRENT_UPLOADS", 4));
/// Set a 64MB limit on the heap size.
pub static ISOLATE_MAX_USER_HEAP_SIZE: LazyLock<usize> =
LazyLock::new(|| env_config("ISOLATE_MAX_USER_HEAP_SIZE", 1 << 26));
/// Allow for some objects to persist between contexts, not necessarily created
/// by the UDF.
pub static ISOLATE_MAX_HEAP_EXTRA_SIZE: LazyLock<usize> =
LazyLock::new(|| env_config("ISOLATE_MAX_HEAP_EXTRA_SIZE", 1 << 25));
/// Set a separate 64MB limit on ArrayBuffer allocations.
pub static ISOLATE_MAX_ARRAY_BUFFER_TOTAL_SIZE: LazyLock<usize> =
LazyLock::new(|| env_config("ISOLATE_MAX_ARRAY_BUFFER_TOTAL_SIZE", 1 << 26));
/// Chunk sizes: 1, 2, 3, ..., MAX_DYNAMIC_SMART_CHUNK_SIZE incrementing by 1.
/// These chunk sizes allow small (common) batches to be handled in a single
/// chunk, while limiting the size of a chunk (don't overload the db), and
/// keeping the number of distinct queries small (for query plan caching).
pub static MYSQL_MAX_DYNAMIC_SMART_CHUNK_SIZE: LazyLock<usize> =
LazyLock::new(|| env_config("MYSQL_MAX_DYNAMIC_SMART_CHUNK_SIZE", 8));
/// More chunks sizes: 1, 2, 4, 8, 16, 32, ..., MYSQL_CHUNK_SIZE doubling.
/// Max packet size is 16MiB.
pub static MYSQL_MAX_CHUNK_BYTES: LazyLock<usize> =
LazyLock::new(|| env_config("MYSQL_MAX_CHUNK_BYTES", 10 << 20));
/// Timeout for all operations on MySQL connections
pub static MYSQL_TIMEOUT: LazyLock<u64> = LazyLock::new(|| env_config("MYSQL_TIMEOUT_SECONDS", 30));
/// Maximum number of connections to MySQL
pub static MYSQL_MAX_CONNECTIONS: LazyLock<usize> =
LazyLock::new(|| env_config("MYSQL_MAX_CONNECTIONS", 128));
/// Minimum number of rows to read from MySQL in a single query.
pub static MYSQL_MIN_QUERY_BATCH_SIZE: LazyLock<usize> =
LazyLock::new(|| env_config("MYSQL_MIN_QUERY_BATCH_SIZE", 1));
/// Maximum number of rows to read from MySQL in a single query.
pub static MYSQL_MAX_QUERY_BATCH_SIZE: LazyLock<usize> =
LazyLock::new(|| env_config("MYSQL_MAX_QUERY_BATCH_SIZE", 5000));
/// We dynamically increase the batch size up to this threshold if client keeps
/// fetching more results. This helps correct for tombstones, long prefixes and
/// wrong client size estimates.
pub static MYSQL_MAX_QUERY_DYNAMIC_BATCH_SIZE: LazyLock<usize> =
LazyLock::new(|| env_config("MYSQL_MAX_QUERY_DYNAMIC_BATCH_SIZE", 8));
/// Close a connection after it has been idle for some time. RDS proxy closes
/// connections after idle_client_timeout in mysql.tf, which should be
/// configured to be higher than this.
///
/// This is overridden to 30 min in funrun
pub static MYSQL_INACTIVE_CONNECTION_LIFETIME: LazyLock<Duration> = LazyLock::new(|| {
Duration::from_secs(env_config("MYSQL_INACTIVE_CONNECTION_LIFETIME_SECS", 90))
});
/// Force recycles a database connections after this period. RDS proxy pins
/// connections if the SQL query which exceeded the 16384 byte limit. Having a
/// hard limit on connection lifetime helps us reduce pinning and improve
/// connection reuse.
///
/// This is overridden to 30 min in funrun
pub static MYSQL_MAX_CONNECTION_LIFETIME: LazyLock<Duration> =
LazyLock::new(|| Duration::from_secs(env_config("MYSQL_MAX_CONNECTION_LIFETIME_SECS", 600)));
/// How many rows we fetch for retention and prev rev fetches (used for
/// TableIterator)
pub static MYSQL_CHUNK_SIZE: LazyLock<usize> =
LazyLock::new(|| env_config("MYSQL_CHUNK_SIZE", 128));
/// Maximum number of connections to Postgres
pub static POSTGRES_MAX_CONNECTIONS: LazyLock<usize> =
LazyLock::new(|| env_config("POSTGRES_MAX_CONNECTIONS", 128));
/// Maximum number of cached statements per Postgres connection
pub static POSTGRES_MAX_CACHED_STATEMENTS: LazyLock<NonZeroUsize> = LazyLock::new(|| {
env_config(
"POSTGRES_MAX_CACHED_STATEMENTS",
NonZeroUsize::new(128).unwrap(),
)
});
/// Close Postgres connections after being idle for this long.
pub static POSTGRES_INACTIVE_CONNECTION_LIFETIME: LazyLock<Duration> = LazyLock::new(|| {
Duration::from_secs(env_config("POSTGRES_INACTIVE_CONNECTION_LIFETIME_SECS", 90))
});
/// How many actions "ops" (e.g. syscalls) can execute concurrently.
pub static MAX_CONCURRENT_ACTION_OPS: LazyLock<usize> =
LazyLock::new(|| env_config("MAX_CONCURRENT_ACTION_OPS", 8));
/// Maximum count of transitions within the web socket server message buffer.
/// When this limit is reached, the web socket worker will temporary stop
/// computing and sending transition messages to the client.
pub static SYNC_MAX_SEND_TRANSITION_COUNT: LazyLock<usize> =
LazyLock::new(|| env_config("SYNC_MAX_SEND_TRANSITION_COUNT", 2));
/// Max Axiom sink attributes. This is a knob just in case a user actually hits
/// the limit but has an Enterprise Axiom plan that lets them use more than the
/// limit we've configured.
pub static AXIOM_MAX_ATTRIBUTES: LazyLock<usize> =
LazyLock::new(|| env_config("AXIOM_MAX_ATTRIBUTES", 1024));
/// If a qdrant Segment's estimated byte is is <= this threshold, we'll build a
/// plain index without HNSW. The default value is 2.5x qdrants. We hope this is
/// ok due to payload filtering, compaction and aiming for 30mb indexes via
/// VECTOR_INDEX_SIZE_SOFT_LIMIT. This is > VECTOR_INDEX_SIZE_SOFT_LIMIT so that
/// have some wiggle room if we build a slightly larger than expected segment.
pub static MULTI_SEGMENT_FULL_SCAN_THRESHOLD_KB: LazyLock<usize> =
LazyLock::new(|| env_config("MULTI_SEGMENT_FULL_SCAN_THRESHOLD_KB", 50_000));
/// The maximum size that we will compact any given set of segments to.
/// Default to using 3 segments per 1.1 million vectors.
pub static SEGMENT_MAX_SIZE_BYTES: LazyLock<u64> =
LazyLock::new(|| env_config("SEGMENT_MAX_SIZE_BYTES", (1100000 * 2048 * 4) / 3_u64));
/// The minimum number of segments we will compact in one pass.
pub static MIN_COMPACTION_SEGMENTS: LazyLock<u64> =
LazyLock::new(|| env_config("MIN_COMPACTION_SEGMENTS", 3));
/// The maximum number of segments to compact in one request.
pub static MAX_COMPACTION_SEGMENTS: LazyLock<usize> =
LazyLock::new(|| env_config("MAX_COMPACTION_SEGMENTS", 10));
/// The maximum percentage of a Segment that can be deleted before we will
/// recompact that segment to remove deleted vectors
/// This number must be between 0 and 1.
pub static MAX_SEGMENT_DELETED_PERCENTAGE: LazyLock<f64> =
LazyLock::new(|| env_config("MAX_SEGMENT_DELETED_PERCENTAGE", 0.2));
/// Whether to run queries, mutations, HTTP actions, and v8 actions in Funrun
/// (true) or InProcessFunctionRunner (false).
pub static UDF_USE_FUNRUN: LazyLock<bool> = LazyLock::new(|| env_config("UDF_USE_FUNRUN", true));
/// The amount of time to wait for the primary request to finish before starting
/// a second backup request when running a vector search.
pub static VECTOR_BACKUP_REQUEST_DELAY_MILLIS: LazyLock<Duration> =
LazyLock::new(|| Duration::from_millis(env_config("VECTOR_BACKUP_REQUEST_DELAY_MILLIS", 30)));
/// Whether to use prepared statements or not in Persistence.
pub static DATABASE_USE_PREPARED_STATEMENTS: LazyLock<bool> =
LazyLock::new(|| env_config("DATABASE_USE_PREPARED_STATEMENTS", false));
/// The amount of time to allow for downloading a single file in the archive
/// cache on searchlight before timing out. If a fetch times out, no progress is
/// made and a subsequent request for the same file will start from the
/// beginning. If this number is too low relative to our disk throughput,
/// archive sizes and concurrent fetches, it can cause congestion collapse when
/// searchlight is restarted.
pub static ARCHIVE_FETCH_TIMEOUT_SECONDS: LazyLock<Duration> =
LazyLock::new(|| Duration::from_secs(env_config("ARCHIVE_FETCH_TIMEOUT_SECONDS", 150)));
/// The total number of modules across all versions that will be held in memory
/// at once.
pub static MODULE_CACHE_MAX_SIZE_BYTES: LazyLock<u64> =
LazyLock::new(|| env_config("MODULE_CACHE_MAX_SIZE_BYTES", 100_000_000));
/// The maximum number of concurrent module fetches we'll allow.
pub static MODULE_CACHE_MAX_CONCURRENCY: LazyLock<usize> =
LazyLock::new(|| env_config("MODULE_CACHE_MAX_CONCURRENCY", 10));
/// The maximum size of the in memory index cache in Funrun in bytes.
pub static FUNRUN_INDEX_CACHE_SIZE: LazyLock<u64> =
LazyLock::new(|| env_config("FUNRUN_INDEX_CACHE_SIZE", 50_000_000)); // 50 MB
/// The maximum number of concurrent index cache requests in Funrun.
pub static FUNRUN_INDEX_CACHE_CONCURRENCY: LazyLock<usize> =
LazyLock::new(|| env_config("FUNRUN_INDEX_CACHE_CONCURRENCY", 100));
/// The maximum size of the module cache in Funrun in bytes.
pub static FUNRUN_MODULE_CACHE_SIZE: LazyLock<u64> =
LazyLock::new(|| env_config("FUNRUN_MODULE_CACHE_SIZE", 250_000_000));
/// The maximum number of concurrent module cache requests in Funrun.
pub static FUNRUN_MODULE_MAX_CONCURRENCY: LazyLock<usize> =
LazyLock::new(|| env_config("FUNRUN_MODULE_MAX_CONCURRENCY", 100));
/// The maximum size of the module code cache in Funrun in bytes.
pub static FUNRUN_CODE_CACHE_SIZE: LazyLock<u64> =
LazyLock::new(|| env_config("FUNRUN_CODE_CACHE_SIZE", 500_000_000));
/// The maximum number of fetch clients Funrun would create.
pub static FUNRUN_FETCH_CLIENT_CACHE_SIZE: LazyLock<usize> =
LazyLock::new(|| env_config("FUNRUN_FETCH_CLIENT_CACHE_SIZE", 100));
/// The maximum number of concurrent requests a single client can make to a
/// single Funrun server.
/// NOTE: When changing this value, ensure that the following parameters
/// for pro satisfy this inequality. This makes sure that if one funrun instance
/// is down, we still have enough capacity to satisfy requests if a backend is
/// maxing out its concurrent function call limits.
///
/// max_concurrent_queries +
/// max_concurrent_mutations + max_concurrent_v8_actions +
/// max_concurrent_http_actions < (number of funrun nodes - 1) *
/// FUNRUN_CLIENT_MAX_REQUESTS_PER_UPSTREAM
pub static FUNRUN_CLIENT_MAX_REQUESTS_PER_UPSTREAM: LazyLock<usize> =
LazyLock::new(|| env_config("FUNRUN_CLIENT_MAX_REQUESTS_PER_UPSTREAM", 20));
/// The maximum number of retries a Funrun client will perform. The client only
/// retries overloaded errors.
pub static FUNRUN_CLIENT_MAX_RETRIES: LazyLock<usize> =
LazyLock::new(|| env_config("FUNRUN_CLIENT_MAX_RETRIES", 4));
/// Value between 1 and 100 representing the percent of a the shared scheduler's
/// worker pool that a single client can use.
pub static FUNRUN_SCHEDULER_MAX_PERCENT_PER_CLIENT: LazyLock<usize> =
LazyLock::new(|| env_config("FUNRUN_SCHEDULER_MAX_PERCENT_PER_CLIENT", 50));
/// Name of the service to discover for when connecting to Funrun (e.g.
/// funrun-default, funrun-staging, etc.)
pub static FUNRUN_CLUSTER_NAME: LazyLock<String> =
LazyLock::new(|| env_config("FUNRUN_CLUSTER_NAME", String::from("funrun-default")));
/// Name of the service to discover for when connecting to Searchlight. (e.g.
/// searchlight-default, searchlight-staging, etc.)
// cluster is created.
pub static SEARCHLIGHT_CLUSTER_NAME: LazyLock<String> = LazyLock::new(|| {
env_config(
"SEARCHLIGHT_CLUSTER_NAME",
String::from("searchlight-default"),
)
});
/// The maximum number of CPU cores that can be used simultaneously by the
/// isolates. Zero means no limit.
pub static FUNRUN_ISOLATE_ACTIVE_THREADS: LazyLock<usize> =
LazyLock::new(|| env_config("FUNRUN_ISOLATE_ACTIVE_THREADS", 0));
/// The maximum length of time to wait to start running a function (when the
/// FUNRUN_ISOLATE_ACTIVE_THREADS limit is reached).
pub static FUNRUN_INITIAL_PERMIT_TIMEOUT: LazyLock<Duration> =
LazyLock::new(|| Duration::from_millis(env_config("FUNRUN_INITIAL_PERMIT_TIMEOUT_MS", 100)));
/// How long to splay deploying AWS Lambdas due to changes in the backend. This
/// knob doesn't delay deploys that are required due to the user pushing new
/// node actions. Only affects deploys on startup triggered by changes to
/// backend/node-executor code.
///
/// AWS has a rate limit of 15/s on their APIs, so we need this to be roughly
/// large enough to be on the same scale as N/s where N is the number of
/// instances with lambdas.
pub static AWS_LAMBDA_DEPLOY_SPLAY_SECONDS: LazyLock<Duration> =
LazyLock::new(|| Duration::from_secs(env_config("AWS_LAMBDA_DEPLOY_SPLAY_SECONDS", 1800)));
/// The maximum number of requests to send using a single AWS Lambda client.
/// Empirical tests have shown that AWS servers allows up to 128 concurrent
/// streams over a single http2 connection.
pub static AWS_LAMBDA_CLIENT_MAX_CONCURRENT_REQUESTS: LazyLock<usize> =
LazyLock::new(|| env_config("AWS_LAMBDA_MAX_CONCURRENT_STREAMS_PER_CONNECTION", 100));
/// The maximum number of times to retry analyze requests for node actions.
pub static NODE_ANALYZE_MAX_RETRIES: LazyLock<usize> =
LazyLock::new(|| env_config("NODE_ANALYZE_MAX_RETRIES", 3));
/// The number of seconds backend should wait for requests to drain before
/// shutting down after SIGINT.
pub static BACKEND_REQUEST_DRAIN_TIMEOUT: LazyLock<Duration> =
LazyLock::new(|| Duration::from_secs(env_config("BACKEND_REQUEST_DRAIN_TIMEOUT", 15)));
/// The kinesis firehose name for streaming usage metrics to the data
// large body of water.
pub static BACKEND_USAGE_FIREHOSE_NAME: LazyLock<Option<String>> = LazyLock::new(|| {
let result = env_config(
"BACKEND_USAGE_FIREHOSE_NAME",
prod_override("", "cvx-firehose-usage-prod").to_string(),
);
if !result.is_empty() {
Some(result)
} else {
None
}
});
/// The number of events we can accumulate in the buffer that's used to send
/// events from our business logic to our firehose client.
///
/// We use try_send to send data over a channel, so it's possible we send faster
/// than the receiving thread can read the data. The buffer helps mitigate that.
///
/// Each item in the buffer is a Vec of events. If we assume that each event is
/// 100 bytes, each Vec might have 100 events and we keep 2000 of these, then
/// worst case our usage is ~100MB.
pub static FIREHOSE_BUFFER_SIZE_COUNT: LazyLock<usize> =
LazyLock::new(|| env_config("FIREHOSE_BUFFER_SIZE_COUNT", 2000));
/// The maximum amount of data we'll populate in a single AWS Kinesis Firehose
/// Record.
///
/// Each Record in AWS has a maximum size of 1000 KiB. PutRecordBatch (which
/// we'll use if a single list of events exceeds this buffer size) has a maximum
/// limit of 4 MiB of records. This controls only the size of the Record, not
/// the size of the batch. We do not currently make any attempt to control the
/// batch size.
pub static FIREHOSE_MAX_BATCH_SIZE_BYTES: LazyLock<usize> =
LazyLock::new(|| env_config("FIREHOSE_MAX_BATCH_SIZE_BYTES", 900 * 1024));
/// The amount of time we'll allow firehose data to sit in memory while it
/// remains under our buffer size. Once this timeout is exceeded for the oldest
/// buffered record, we'll send our entire buffer.
pub static FIREHOSE_TIMEOUT: LazyLock<Duration> =
LazyLock::new(|| Duration::from_secs(env_config("FIREHOSE_TIMEOUT", 60)));
/// The initial backoff time for index workers when a failure occurs.
pub static INDEX_WORKERS_INITIAL_BACKOFF: LazyLock<Duration> =
LazyLock::new(|| Duration::from_millis(env_config("INDEX_WORKERS_INITIAL_BACKOFF", 500)));
/// The maximum backoff time for index workers when a failure occurs.
pub static INDEX_WORKERS_MAX_BACKOFF: LazyLock<Duration> =
LazyLock::new(|| Duration::from_millis(env_config("INDEX_WORKERS_MAX_BACKOFF", 30 * 1000)));
/// The maximum backoff time for search index flusher workers when a failure
/// occurs. This shouldn't be set too high because flushes are required for
/// write throughput.
pub static SEARCH_INDEX_FLUSHER_MAX_BACKOFF: LazyLock<Duration> =
LazyLock::new(|| Duration::from_secs(env_config("SEARCH_INDEX_FLUSHER_MAX_BACKOFF", 30)));
/// The maximum backoff time for search compactor workers when a failure occurs.
/// This can be set relatively high because compaction is not a critical
/// operation. If compaction fails, latency for searches may increase from
/// searching too many segments. Setting this too low can overwhelm searchlight
/// on persistent failures.
pub static SEARCH_COMPACTOR_MAX_BACKOFF: LazyLock<Duration> =
LazyLock::new(|| Duration::from_secs(env_config("SEARCH_COMPACTOR_MAX_BACKOFF", 10 * 60)));
/// The initial backoff time for search compactor workers when a failure occurs.
/// This can be set relatively high because compaction is not a critical
/// operation. If compaction fails, latency for searches may increase from
/// searching too many segments. Setting this too low can overwhelm searchlight
/// on persistent failures.
pub static SEARCH_COMPACTOR_INITIAL_BACKOFF: LazyLock<Duration> =
LazyLock::new(|| Duration::from_secs(env_config("SEARCH_COMPACTOR_INITIAL_BACKOFF", 60)));
/// The maximum size for Funrun run function request messages. The user-level
/// limit is 16MiB for args (as counted by ConvexValue::size), but the actual
/// wire encoding can be 3-4x that, so we set a higher limit here.
///
/// Note that analyze has a much higher limit because it includes user source
/// code.
pub static MAX_FUNRUN_RUN_FUNCTION_REQUEST_MESSAGE_SIZE: LazyLock<usize> = LazyLock::new(|| {
env_config(
"MAX_FUNRUN_RUN_FUNCTION_REQUEST_MESSAGE_SIZE",
1 << 26, // 64MiB
)
});
/// The maximum size for Funrun run function response messages. 8MiB for reads +
/// 16 MiB for writes + 16MiB for function result + 8MiB for log lines and a 8
/// MiB buffer for the smaller fields.
pub static MAX_FUNRUN_RUN_FUNCTION_RESPONSE_MESSAGE_SIZE: LazyLock<usize> =
LazyLock::new(|| env_config("MAX_FUNRUN_RUN_FUNCTION_RESPONSE_MESSAGE_SIZE", 1 << 26)); // 64 MiB
/// The maximum size for Backend HTTP and GRPC action callbacks. This is 8MiB
/// for path and args, plus a generous buffer for the smaller fields This should
/// also be enough for vector and text search callbacks.
pub static MAX_BACKEND_RPC_REQUEST_SIZE: LazyLock<usize> =
LazyLock::new(|| env_config("MAX_BACKEND_RPC_REQUEST_SIZE", 1 << 25)); // 32 MiB
/// The maximum size for Backend HTTP and GRPC action callbacks.
/// This needs to contain 8MiB for function result + 4MiB for log lines + 4MiB
/// for smaller fields.
pub static MAX_BACKEND_RPC_RESPONSE_SIZE: LazyLock<usize> =
LazyLock::new(|| env_config("MAX_BACKEND_RPC_RESPONSE_SIZE", 1 << 25)); // 32 MiB
/// The maximum size of byte chunks used when transmitting HTTP request/response
/// bodies as part of HTTP Actions.
pub static MAX_BACKEND_RPC_HTTP_CHUNK_SIZE: LazyLock<usize> =
LazyLock::new(|| env_config("MAX_BACKEND_RPC_RESPONSE_SIZE", 1 << 23)); // 8 MiB
/// The maximum size for requests to the backend public API. Must be at least 16
/// MiB for function arguments.
pub static MAX_BACKEND_PUBLIC_API_REQUEST_SIZE: LazyLock<usize> =
LazyLock::new(|| env_config("MAX_BACKEND_PUBLIC_API_REQUEST_SIZE", (1 << 24) + 2000)); // 16 MiB
/// The maximum size for GRPC responses from searchlight.
pub static MAX_SEARCHLIGHT_RESPONSE_SIZE: LazyLock<usize> =
LazyLock::new(|| env_config("MAX_SEARCHLIGHT_RESPONSE_SIZE", 1 << 23)); // 8 MiB
/// The maximum size for GRPC responses from searchlight.
pub static MAX_SEARCHLIGHT_REQUEST_SIZE: LazyLock<usize> =
LazyLock::new(|| env_config("MAX_SEARCHLIGHT_REQUEST_SIZE", 1 << 23)); // 8 MiB
/// Background database workers wake up periodically, check to see if something
/// has changed, then either go back to sleep or do work. Most workers determine
/// if something has changed at least in part by comparing the number of commits
/// since the last time they woke up. This works perfectly if there's exactly
/// one worker. But we actually have N workers, so if each of them wakes up and
/// does work whenever there's a new commit, they're all constantly doing work
/// because of each other.
///
/// So a simple way to avoid this is to require there to at least N commits
/// rather than >=1 commit. As long as N is greater than the number of workers,
/// each worker will wake up, commit once, then go back to sleep until something
/// other than the other workers happens.
///
/// We must also ensure that workers advance periodically to ensure that we can
/// run document retention in the future. Database times are bumped periodically
/// even if no writes occur. So any worker that checks this should always have
/// some maximum period of time after which they checkpoint unconditionally.
pub static DATABASE_WORKERS_MIN_COMMITS: LazyLock<usize> =
LazyLock::new(|| env_config("DATABASE_WORKERS_MIN_COMMITS", 100));
/// The TableSummaryWorker must checkpoint every
/// [`DATABASE_WORKERS_MAX_CHECKPOINT_AGE`] seconds even if nothing has changed.
/// However, to prevent all instances from checkpointing at the same time, we'll
/// add a jitter of up to ±TABLE_SUMMARY_AGE_JITTER_SECONDS.
///
/// Note: the configured value is capped at
/// `DATABASE_WORKERS_MAX_CHECKPOINT_AGE/2`.
pub static TABLE_SUMMARY_AGE_JITTER_SECONDS: LazyLock<f32> =
LazyLock::new(|| env_config("TABLE_SUMMARY_AGE_JITTER_SECONDS", 900.0));
/// HTTP requests to backend will time out after this duration has passed.
///
/// See https://docs.rs/tower-http/0.5.0/tower_http/timeout/struct.TimeoutLayer.html
pub static HTTP_SERVER_TIMEOUT_DURATION: LazyLock<Duration> =
LazyLock::new(|| Duration::from_secs(env_config("HTTP_SERVER_TIMEOUT_SECONDS", 300)));
/// The limit on the request size to /push_config.
// Schema and code bundle pushes must be less than this.
pub static MAX_PUSH_BYTES: LazyLock<usize> =
LazyLock::new(|| env_config("MAX_PUSH_BYTES", 200_000_000));
/// The limit on the request size to /echo. Limits requests to 128MiB to help
/// mitigate DDoS attacks.
pub static MAX_ECHO_BYTES: LazyLock<usize> =
LazyLock::new(|| env_config("MAX_ECHO_BYTES", 128 * 1024 * 1024));
/// The limit on the number of user modules in a push bundle.
pub static MAX_USER_MODULES: LazyLock<usize> =
LazyLock::new(|| env_config("MAX_USER_MODULES", 4096));
/// Percentage of request traces that should sampled.
///
/// Sampling config is a JSON object with the following format:
/// {
/// instanceOverrides: {
/// instanceName1: [{routeRegexp1: fraction1}, {routeRegexp2:
/// fraction2}, ...], instanceName2: [{routeRegexp1: fraction1},
/// {routeRegexp2: fraction2}, ...], ...
/// },
/// routeOverrides: [{routeRegexp: fraction}, ...],
/// defaultFraction: fraction
/// }
///
/// (see also the doc comment on SamplingConfigJson).
///
/// These apply in order -- the instance overrides take precedence over route
/// overrides, which take precedence over the default fraction.
///
/// When in doubt, write out a test case to verify the behavior in
/// fastrace_helpers.rs.
///
/// It's often easiest to craft the JSON in a repl (e.g. node, browser console),
/// and then stringify it at the end.
///
/// See `fastrace_helpers.rs` for more examples
pub static REQUEST_TRACE_SAMPLE_CONFIG: LazyLock<SamplingConfig> = LazyLock::new(|| {
env_config(
"REQUEST_TRACE_SAMPLE_CONFIG",
prod_override(
SamplingConfig::default(),
r#"{"defaultFraction":0.00001,"routeOverrides":[{"routeRegexp":"/api/push_config","fraction":0.1}, {"routeRegexp":"conductor/load-instance","fraction":0.01}]}"#
.parse()
.unwrap(),
),
)
});
/// Size of the cache for access token authentication
pub static AUTH_CACHE_SIZE: LazyLock<usize> = LazyLock::new(|| env_config("AUTH_CACHE_SIZE", 1000));
/// Length of time an entry to the access authentication cache is valid
pub static AUTH_CACHE_TTL_SECONDS: LazyLock<u64> =
LazyLock::new(|| env_config("AUTH_CACHE_TTL_SECONDS", 60));
/// Request body limit for airbyte streaming import requests
pub static AIRBYTE_STREAMING_IMPORT_REQUEST_SIZE_LIMIT: LazyLock<usize> = LazyLock::new(|| {
env_config(
"AIRBYTE_STREAMING_IMPORT_REQUEST_SIZE_LIMIT",
10 * (2 << 20),
)
});
/// The maximum number of backends to keep open connections to.
pub static USHER_BACKEND_CLIENTS_CACHE_SIZE: LazyLock<u64> =
LazyLock::new(|| env_config("USHER_BACKEND_CLIENTS_CACHE_SIZE", 5000));
/// The maximum number of concurrent streams over a single tonic channel.
/// Providing a limit helps us not run into any implementation limits.
pub static USHER_MAX_CONCURRENT_STREAMS_PER_CHANNEL: LazyLock<usize> =
LazyLock::new(|| env_config("USHER_MAX_CONCURRENT_STREAMS_PER_CHANNEL", 500));
/// Like FUNCTION_MAX_ARGS_SIZE - but while still serialized as JSON. Needs to
/// be larger. Also needs to fit within the GRPC limits we configure with
/// MAX_BACKEND_RPC_REQUEST_SIZE,
pub static USHER_MAX_JSON_ARGS_SIZE: LazyLock<usize> =
LazyLock::new(|| env_config("USHER_MAX_JSON_ARGS_SIZE", 30 * 1024 * 1024)); // 30MB
/// Batch size for migration that rewrites virtual tables.
pub static MIGRATION_REWRITE_BATCH_SIZE: LazyLock<usize> =
LazyLock::new(|| env_config("MIGRATION_REWRITE_BATCH_SIZE", 100));
/// If an import is taking longer than a day, it's a problem (and our fault).
/// But the customer is probably no longer waiting so we should fail the import.
/// If an import takes more than a week, the file may be deleted from S3.
pub static MAX_IMPORT_AGE: LazyLock<Duration> =
LazyLock::new(|| Duration::from_secs(env_config("MAX_IMPORT_AGE_SECONDS", 7 * 24 * 60 * 60)));
/// Max staleness in seconds of a partition loader result before we allow
/// refreshing. If a request tries to update the partition loader and this
/// duration has not passed since the last refresh, a stale value will be used.
pub static PARTITION_LOADER_MAX_STALE_SECS: LazyLock<Duration> =
LazyLock::new(|| Duration::from_secs(env_config("PARTITION_LOADER_MAX_STALE_SECS", 1)));
/// Seconds to wait before timing out when trying to acquire the lock for
/// claiming an instance in big-brain.
pub static CLAIM_INSTANCE_TIMEOUT_SECS: LazyLock<Duration> =
LazyLock::new(|| Duration::from_secs(env_config("CLAIM_INSTANCE_TIMEOUT_SECS", 10)));
/// The maximum number of bytes to buffer in an multipart upload.
/// There may be stricter limits imposed by the storage provider, but this is
/// the target max size for the buffer to protect against memory exhaustion.
/// The maximum number of parts is 10000, so this imposes a max file size, which
/// defaults to 10000 * 100MiB = 1TB. When reducing this knob, make sure the
/// maximum file size can fit a snapshot export of the instance.
/// Storage uploads can run in parallel, and the buffers get cloned during
/// upload, so make sure this knob times ~8 can fit in memory.
///
/// Defaults to 100MiB.
pub static STORAGE_MAX_INTERMEDIATE_PART_SIZE: LazyLock<usize> =
LazyLock::new(|| env_config("STORAGE_MAX_INTERMEDIATE_PART_SIZE", 100 * (1 << 20)));
/// Minimum number of milliseconds a commit needs to take to send traces to
/// honeycomb.
pub static COMMIT_TRACE_THRESHOLD: LazyLock<Duration> =
LazyLock::new(|| Duration::from_millis(env_config("COMMIT_TRACE_THRESHOLD", 500)));
/// How many instances a Conductor will try to simultaneously load (on startup,
/// or when it discovers new instances) Going too high means that the Conductor
/// may be unable to serve requests for already-loaded instances in a timely
/// manner, or that we may exhaust CPU on the physical host
pub static INSTANCE_LOADER_CONCURRENCY: LazyLock<usize> =
LazyLock::new(|| env_config("INSTANCE_LOADER_CONCURRENCY", 16));
/// Whether or not to use a rate limiter when loading instances
pub static INSTANCE_LOADER_USE_RATE_LIMITER: LazyLock<bool> =
LazyLock::new(|| env_config("INSTANCE_LOADER_USE_RATE_LIMITER", true));
/// The number of instances that can be loaded per second when the rate limiter
/// is in use. The default value of 4 means that for a Conductor with 5000
/// instances, we'd take about 20 minutes to load all instances with infinite
/// concurrency.
pub static INSTANCE_LOADER_INSTANCES_PER_SECOND: LazyLock<NonZeroU32> = LazyLock::new(|| {
env_config(
"INSTANCE_LOADER_INSTANCES_PER_SECOND",
NonZeroU32::new(4).unwrap(),
)
});
/// The max number of storage files that can be fetched concurrently during
/// export. Concurrency is also limited by `EXPORT_MAX_INFLIGHT_PREFETCH_BYTES`.
pub static EXPORT_STORAGE_GET_CONCURRENCY: LazyLock<usize> =
LazyLock::new(|| env_config("EXPORT_STORAGE_GET_CONCURRENCY", 128).max(1));
/// The max number of bytes that can be prefetched concurrently from storage
/// files during export.
///
/// Files larger than this will be not be fetched concurrently with other files.
pub static EXPORT_MAX_INFLIGHT_PREFETCH_BYTES: LazyLock<usize> = LazyLock::new(|| {
env_config(
"EXPORT_MAX_INFLIGHT_PREFETCH_BYTES",
32 * 1024 * 1024, // 32MiB
)
.clamp(1, u32::MAX as usize)
});
/// The page size to the table iterator in the export worker.
pub static EXPORT_WORKER_PAGE_SIZE: LazyLock<usize> =
LazyLock::new(|| env_config("EXPORT_WORKER_PAGE_SIZE", 1000));
/// Whether or not a service should propagate all upstream traces or perform its
/// own sampling
pub static PROPAGATE_UPSTREAM_TRACES: LazyLock<bool> =
LazyLock::new(|| env_config("PROPAGATE_UPSTREAM_TRACES", true));
/// The maximum allowed age of /list_snapshot's timestamp.
pub static LIST_SNAPSHOT_MAX_AGE_SECS: LazyLock<Duration> = LazyLock::new(|| {
Duration::from_secs(env_config("LIST_SNAPSHOT_MAX_AGE_SECS", 30 * 24 * 60 * 60))
});
/// The length of the SubscriptionsWorker's input queue.
/// If the worker starts to lag then subscriptions will fail once this many
/// messages are unread.
pub static SUBSCRIPTIONS_WORKER_QUEUE_SIZE: LazyLock<usize> =
LazyLock::new(|| env_config("SUBSCRIPTIONS_WORKER_QUEUE_SIZE", 10000));
/// Time to wait before scheduling update queries in the sync worker after a
/// search query fails because indexes are bootstrapping.
pub static SEARCH_INDEXES_UNAVAILABLE_RETRY_DELAY: LazyLock<Duration> =
LazyLock::new(|| Duration::from_secs(env_config("SEARCH_INDEXES_UNAVAILABLE_RETRY_DELAY", 3)));
/// The maximum number of subscriptions that can be invalidated immediately. If
/// there are more, they will be splayed out.
pub static SUBSCRIPTION_INVALIDATION_DELAY_THRESHOLD: LazyLock<usize> =
LazyLock::new(|| env_config("SUBSCRIPTION_INVALIDATION_DELAY_THRESHOLD", 200));
/// How much to splay subscription invalidations. More precisely, this is the
/// number used to multiply by the number of subscriptions that need to be
/// invalidated to determine the delay before invalidating them.
pub static SUBSCRIPTION_INVALIDATION_DELAY_MULTIPLIER: LazyLock<u64> =
LazyLock::new(|| env_config("SUBSCRIPTION_INVALIDATION_DELAY_MULTIPLIER", 5));
/// When processing a single write log entry takes longer than that time, log
/// extra detail.
pub static SUBSCRIPTION_PROCESS_LOG_ENTRY_TRACING_THRESHOLD: LazyLock<u64> =
LazyLock::new(|| env_config("SUBSCRIPTION_PROCESS_LOG_ENTRY_TRACING_THRESHOLD", 2));
/// When advancing the write log takes longer than this amount, log extra
/// details.
pub static SUBSCRIPTION_ADVANCE_LOG_TRACING_THRESHOLD: LazyLock<u64> =
LazyLock::new(|| env_config("SUBSCRIPTION_ADVANCE_LOG_TRACING_THRESHOLD", 10));
/// How many concurrent index backfill threads to run concurrently.
pub static INDEX_BACKFILL_CONCURRENCY: LazyLock<usize> =
LazyLock::new(|| env_config("INDEX_BACKFILL_CONCURRENCY", 8));
/// The max size of the global HTTP cache, in bytes. This is only used for
/// getting auth metadata right now.
pub static HTTP_CACHE_SIZE: LazyLock<u64> =
LazyLock::new(|| env_config("HTTP_CACHE_SIZE", 16 * 1024 * 1024));