make dev

make install-claude-config

docker run --rm \
  --entrypoint /bin/bash \
  -v "$(pwd)/crawler-config:/app/config" \
  --network host \
  docker.elastic.co/integrations/crawler:latest \
  -c "bin/crawler crawl config/test-crawler.yml"

POST /_security/api_key
{
  "name": "crawler-search-labs",
  "role_descriptors": {
    "crawler-search-labs-role": {
      "cluster": ["monitor"],
      "indices": [
        {
          "names": ["search-labs-posts"],
          "privileges": ["all"]
        }
      ]
    }
  },
  "metadata": {
    "application": "crawler"
  }
}

PUT search-labs-posts

PUT search-labs-posts/_mappings
{
  "properties": {
    "body": {
      "type": "text",
      "copy_to": "semantic_body"
    },
    "semantic_body": {
      "type": "semantic_text",
      "inference_id": ".elser-2-elasticsearch"
    }
  }
}

docker run --rm \
  --entrypoint /bin/bash \
  -v "$(pwd)/crawler-config:/app/config" \
  --network host \
  docker.elastic.co/integrations/crawler:latest \
  -c "bin/crawler crawl config/elastic-search-labs-crawler.yml"

GET search-labs-posts/_count