{
"id": "s3",
"schema": {
"type": "object",
"title": "",
"required": ["bucket"],
"properties": {
"type": {
"type": "string",
"enum": ["s3"],
"flag": {
"prefix": "collector",
"modes": ["single", "worker"]
}
},
"outputName": {
"type": "string",
"title": "Auto-populate from",
"description": "Name of the predefined Destination that will be used to auto-populate Collector settings"
},
"bucket": {
"type": "string",
"title": "S3 bucket",
"minLength": 1,
"description": "S3 Bucket from which to collect data",
"jsExpression": "warning",
"s3BucketName": true
},
"parquetChunkSizeMB": {
"type": "number",
"title": "Parquet chunk size limit (MB)",
"description": "Maximum file size for each Parquet chunk",
"default": 5,
"maximum": 100,
"minimum": 1
},
"parquetChunkDownloadTimeout": {
"type": "number",
"title": "Parquet chunk download timeout (seconds)",
"description": "Maximum time allowed for downloading a Parquet chunk. Processing will stop if a chunk cannot be downloaded within the time specified.",
"default": 600,
"maximum": 3600,
"minimum": 1
},
"region": {
"type": "string",
"title": "Region",
"description": "Region from which to retrieve data"
},
"path": {
"type": "string",
"title": "Path",
"description": "Directory where data will be collected. Templating (such as 'myDir/${datacenter}/${host}/${app}/') and time-based tokens (such as 'myOtherDir/${_time:%Y}/${_time:%m}/${_time:%d}/') are supported. Can be a constant (enclosed in quotes) or a JavaScript expression.",
"minLength": 1,
"jsExpression": "warning",
"templated": true
},
"partitioningScheme": {
"type": "string",
"title": "Partitioning scheme",
"description": "Partitioning scheme used for this dataset. Using a known scheme like DDSS enables more efficient data reading and retrieval.",
"default": "none",
"enum": ["none", "ddss"],
"enumNames": ["Defined in Path", "DDSS"]
},
"extractors": {
"type": "array",
"uniqueFields": ["key"],
"title": "Path extractors",
"additionalProperties": false,
"items": {
"type": "object",
"required": ["key", "expression"],
"properties": {
"key": {
"type": "string",
"title": "Token",
"description": "A token from the template path, such as epoch"
},
"expression": {
"type": "string",
"title": "Extractor Expression",
"description": "JavaScript expression that receives token under \"value\" variable, and evaluates to populate event fields. Example: {date: new Date(+value*1000)}"
}
}
},
"description": "Allows using template tokens as context for expressions that enrich discovery results. For example, given a template /path/${epoch}, an extractor under key \"epoch\" with an expression {date: new Date(+value*1000)}, will enrich discovery results with a human readable \"date\" field."
},
"awsAuthenticationMethod": {
"type": "string",
"title": "Authentication method",
"description": "AWS authentication method. Choose Auto to use IAM roles.",
"default": "auto",
"enum": ["auto", "manual", "secret"],
"enumNames": ["Auto", "Manual", "Secret Key pair"]
},
"endpoint": {
"type": "string",
"title": "Endpoint",
"description": "Must point to an S3-compatible endpoint. If empty, defaults to an AWS region-specific endpoint. "
},
"signatureVersion": {
"type": "string",
"title": "Signature version",
"description": "Signature version to use for signing S3 requests",
"default": "v4",
"enum": ["v2", "v4"]
},
"enableAssumeRole": {
"type": "boolean",
"title": "Enable Assume Role",
"description": "Use AssumeRole credentials",
"default": false
},
"assumeRoleArn": {
"type": "string",
"title": "AssumeRole ARN",
"description": "Amazon Resource Name (ARN) of the role to assume",
"pattern": "^arn:",
"minLength": 20
},
"assumeRoleExternalId": {
"type": "string",
"title": "External ID",
"description": "External ID to use when assuming role"
},
"durationSeconds": {
"type": "number",
"title": "Duration (seconds)",
"description": "Duration of the Assumed Role's session, in seconds. Minimum is 900 (15 minutes), default is 3600 (1 hour), and maximum is 43200 (12 hours).",
"minimum": 900,
"maximum": 43200,
"default": 3600
},
"maxBatchSize": {
"type": "number",
"title": "Batch size limit (objects)",
"description": "Maximum number of metadata objects to batch before recording as results",
"minimum": 1,
"default": 10
},
"recurse": {},
"reuseConnections": {
"type": "boolean",
"title": "Reuse connections",
"description": "Reuse connections between requests to improve performance",
"default": true
},
"rejectUnauthorized": {
"type": "boolean",
"title": "Reject unauthorized certificates",
"description": "Reject certificates that cannot be verified against a valid CA (such as a self-signed certificate)",
"default": true
},
"verifyPermissions": {
"type": "boolean",
"title": "Verify bucket permissions",
"description": "Disable if you can access files within the bucket but not the bucket itself. Resolves errors of the form \"discover task initialization failed...error: Forbidden\".",
"default": true
},
"disableTimeFilter": {
"type": "boolean",
"title": "Disable time filter",
"description": "Disable Collector event time filtering when a date range is specified",
"default": false
}
},
"dependencies": {
"partitioningScheme": {
"oneOf": [
{
"properties": {
"partitioningScheme": {
"enum": ["ddss"]
}
}
},
{
"properties": {
"partitioningScheme": {
"enum": ["none"]
},
"recurse": {
"type": "boolean",
"title": "Recursive",
"description": "Traverse and include files from subdirectories. Leave this option enabled to ensure that all nested directories are searched and their contents collected.",
"default": true
}
}
}
]
},
"awsAuthenticationMethod": {
"oneOf": [
{
"properties": {
"awsAuthenticationMethod": {
"enum": ["auto"]
}
}
},
{
"properties": {
"awsAuthenticationMethod": {
"enum": ["manual"]
},
"awsApiKey": {
"type": "string",
"title": "Access key",
"jsExpression": "warning",
"description": "Access key. If not present, will fall back to env.AWS_ACCESS_KEY_ID, or to the metadata endpoint for IAM creds. Optional when running on AWS. This value can be a constant or a JavaScript expression."
},
"awsSecretKey": {
"type": "string",
"title": "Secret key",
"description": "Secret key. If not present, will fall back to env.AWS_SECRET_ACCESS_KEY, or to the metadata endpoint for IAM creds. Optional when running on AWS. This value can be a constant or a JavaScript expression.",
"jsExpression": "warning",
"encrypt": true
}
}
},
{
"properties": {
"awsAuthenticationMethod": {
"enum": ["secret"]
},
"awsSecret": {
"type": "string",
"title": "Secret key pair",
"secret": {
"target": ["awsApiKey", "awsSecretKey"],
"type": "keypair"
},
"description": "Select or create a stored secret that references AWS access key and secret key."
}
}
}
]
}
}
}
}