changedOutput schema / (root)
Previous value: -nullNew value: +{
+ "properties": {
+ "agent": {
+ "type": "string"
+ },
+ "blocked": {
+ "description": "The response refused the crawler (403, 429 or similar)",
+ "type": "boolean"
+ },
+ "chain": {
+ "description": "Redirect hops",
+ "items": {
+ "properties": {
+ "status": {
+ "type": "integer"
+ },
+ "url": {
+ "type": "string"
+ }
+ },
+ "type": "object"
+ },
+ "type": "array"
+ },
+ "challenge": {
+ "description": "A bot-challenge page rather than the site",
+ "type": "boolean"
+ },
+ "content_type": {
+ "type": "string"
+ },
+ "domain": {
+ "type": "string"
+ },
+ "error": {
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "extract": {
+ "description": "The text this crawler would index",
+ "type": "string"
+ },
+ "final_url": {
+ "type": "string"
+ },
+ "headers": {
+ "additionalProperties": {
+ "type": "string"
+ },
+ "type": "object"
+ },
+ "label": {
+ "description": "The crawler as its operator names it",
+ "type": "string"
+ },
+ "ms": {
+ "type": "integer"
+ },
+ "status": {
+ "description": "HTTP status the crawler received",
+ "type": [
+ "integer",
+ "null"
+ ]
+ },
+ "title": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "domain",
+ "agent"
+ ],
+ "type": "object"
+}