@arizeai/phoenix-mcp

{ "cells": [ { "cell_type": "code", "execution_count": null, "id": "initial_id", "metadata": {}, "outputs": [], "source": [ "%pip install -Uqqq arize-phoenix-client arize-phoenix-otel anthropic requests beautifulsoup4 openinference-instrumentation-anthropic" ] }, { "cell_type": "code", "execution_count": null, "id": "923081d1", "metadata": {}, "outputs": [], "source": [ "import os\n", "from getpass import getpass\n", "from itertools import chain\n", "from pprint import pp\n", "from textwrap import dedent\n", "\n", "import pandas as pd\n", "import requests\n", "from anthropic import Anthropic\n", "from anthropic.types.message_create_params import MessageCreateParamsBase\n", "from bs4 import BeautifulSoup\n", "from IPython.display import HTML, display\n", "from openinference.instrumentation.anthropic import AnthropicInstrumentor\n", "\n", "from phoenix.client import Client\n", "from phoenix.client.types import PromptVersion\n", "from phoenix.otel import register" ] }, { "cell_type": "code", "execution_count": null, "id": "2c190492", "metadata": {}, "outputs": [], "source": [ "if not os.getenv(\"ANTHROPIC_API_KEY\"):\n", " os.environ[\"ANTHROPIC_API_KEY\"] = getpass(\"Anthropic API key: \")" ] }, { "cell_type": "code", "execution_count": null, "id": "e94030bd", "metadata": {}, "outputs": [], "source": [ "tracer_provider = register()\n", "AnthropicInstrumentor().instrument(tracer_provider=tracer_provider)" ] }, { "cell_type": "markdown", "id": "ce9b317ee68c93ca", "metadata": {}, "source": [ "Install and start a local instance of Phoenix if needed." ] }, { "cell_type": "code", "execution_count": null, "id": "06a78b27", "metadata": {}, "outputs": [], "source": [ "%pip install -Uqqq arize-phoenix\n", "import phoenix as px\n", "\n", "px.launch_app()" ] }, { "cell_type": "markdown", "id": "f459e468", "metadata": {}, "source": [ "# Text Generation" ] }, { "cell_type": "markdown", "id": "2eb0dbdb", "metadata": {}, "source": [ "## Quick Start" ] }, { "cell_type": "markdown", "id": "ae5a3607", "metadata": {}, "source": [ "Here's a simple LLM invocation." ] }, { "cell_type": "code", "execution_count": null, "id": "3c923162", "metadata": {}, "outputs": [], "source": [ "params = MessageCreateParamsBase(\n", " model=\"claude-3-5-haiku-latest\",\n", " max_tokens=128,\n", " temperature=0,\n", " system=\"You're a coding poet\",\n", " messages=[{\"role\": \"user\", \"content\": \"Write a haiku about recursion in programming.\"}],\n", ")\n", "resp = Anthropic().messages.create(**params)\n", "print(resp.content[0].text)" ] }, { "cell_type": "markdown", "id": "d8cf828f", "metadata": {}, "source": [ "We can save the prompt in Phoenix." ] }, { "cell_type": "code", "execution_count": null, "id": "023da3ef", "metadata": {}, "outputs": [], "source": [ "# prompt identifier should contain only alphanumeric characters, hyphens or underscores\n", "prompt_identifier = \"haiku-recursion\"\n", "\n", "prompt = Client().prompts.create(\n", " name=prompt_identifier,\n", " prompt_description=\"Haiku about recursion in programming\",\n", " version=PromptVersion.from_anthropic(params),\n", ")" ] }, { "cell_type": "markdown", "id": "1245d993", "metadata": {}, "source": [ "We can fetch the prompt from Phoenix." ] }, { "cell_type": "code", "execution_count": null, "id": "d0a0876e", "metadata": {}, "outputs": [], "source": [ "prompt = Client().prompts.get(prompt_identifier=prompt_identifier)\n", "resp = Anthropic().messages.create(**prompt.format())\n", "print(resp.content[0].text)" ] }, { "cell_type": "markdown", "id": "11ad4765", "metadata": {}, "source": [ "## Building a moderation filter" ] }, { "cell_type": "markdown", "id": "a95119d0", "metadata": {}, "source": [ "Base on [this example](https://colab.research.google.com/github/anthropics/anthropic-cookbook/blob/09c00ae4694b1a754dac26202ec13c1190a54a52/misc/building_moderation_filter.ipynb#scrollTo=S22qtucNQaPs&line=7&uniqifier=1) from Anthropic cookbook" ] }, { "cell_type": "code", "execution_count": null, "id": "68e9c71e", "metadata": {}, "outputs": [], "source": [ "content = \"\"\"\\\n", " You are a content moderation expert tasked with categorizing user-generated text based on the following guidelines:\n", "\n", " {{guidelines}}\n", "\n", " Here is the user-generated text to categorize:\n", " <user_text>{{user_text}}</user_text>\n", "\n", " Based on the guidelines above, classify this text as either ALLOW or BLOCK. Return nothing else.\\\n", "\"\"\"\n", "\n", "params = MessageCreateParamsBase(\n", " model=\"claude-3-5-haiku-latest\",\n", " max_tokens=10,\n", " messages=[{\"role\": \"user\", \"content\": dedent(content)}],\n", ")" ] }, { "cell_type": "markdown", "id": "68ffc98b", "metadata": {}, "source": [ "Save prompt in Phoenix" ] }, { "cell_type": "code", "execution_count": null, "id": "1a9e1698", "metadata": {}, "outputs": [], "source": [ "# prompt identifier should contain only alphanumeric characters, hyphens or underscores\n", "prompt_identifier = \"content-moderation\"\n", "\n", "prompt = Client().prompts.create(\n", " name=prompt_identifier,\n", " prompt_description=\"Content moderation task\",\n", " version=PromptVersion.from_anthropic(params),\n", ")" ] }, { "cell_type": "markdown", "id": "2d7edd43", "metadata": {}, "source": [ "Fetch prompt and apply to data." ] }, { "cell_type": "code", "execution_count": null, "id": "b1da419d", "metadata": {}, "outputs": [], "source": [ "example_guidelines = \"\"\"\\\n", " BLOCK CATEGORY:\n", " - Promoting violence, illegal activities, or hate speech\n", " - Explicit sexual content\n", " - Harmful misinformation or conspiracy theories\n", "\n", " ALLOW CATEGORY:\n", " - Most other content is allowed, as long as it is not explicitly disallowed\\\n", "\"\"\"\n", "\n", "user_comments = [\n", " \"This movie was great, I really enjoyed it. The main actor really killed it!\",\n", " \"Delete this post now or you better hide. I am coming after you and your family.\",\n", " \"Stay away from the 5G cellphones!! They are using 5G to control you.\",\n", " \"Thanks for the helpful information!\",\n", "]\n", "\n", "prompt = Client().prompts.get(prompt_identifier=prompt_identifier)\n", "\n", "for comment in user_comments:\n", " variables = {\"user_text\": comment, \"guidelines\": example_guidelines}\n", " response = Anthropic().messages.create(**prompt.format(variables=variables))\n", " print(f\"User comment: {comment}\")\n", " print(f\"Model response: {response.content[0].text}\")\n", " print()" ] }, { "cell_type": "markdown", "id": "9033089e", "metadata": {}, "source": [ "# Tool Use" ] }, { "cell_type": "markdown", "id": "8e6218ea", "metadata": {}, "source": [ "## Article Summarization" ] }, { "cell_type": "markdown", "id": "4291e22b", "metadata": {}, "source": [ "Based on [this example](https://colab.research.google.com/github/anthropics/anthropic-cookbook/blob/09c00ae4694b1a754dac26202ec13c1190a54a52/tool_use/extracting_structured_json.ipynb#scrollTo=dcrGrM9G9CNO&line=2&uniqifier=1) from Anthropic cookbook" ] }, { "cell_type": "code", "execution_count": null, "id": "afde437a", "metadata": {}, "outputs": [], "source": [ "tools = [\n", " {\n", " \"name\": \"print_summary\",\n", " \"description\": \"Prints a summary of the article.\",\n", " \"input_schema\": {\n", " \"type\": \"object\",\n", " \"properties\": {\n", " \"author\": {\"type\": \"string\", \"description\": \"Name of the article author\"},\n", " \"topics\": {\n", " \"type\": \"array\",\n", " \"items\": {\"type\": \"string\"},\n", " \"description\": 'Array of topics, e.g. [\"tech\", \"politics\"]. Should be as specific as possible, and can overlap.',\n", " },\n", " \"summary\": {\n", " \"type\": \"string\",\n", " \"description\": \"Summary of the article. One or two paragraphs max.\",\n", " },\n", " \"coherence\": {\n", " \"type\": \"integer\",\n", " \"description\": \"Coherence of the article's key points, 0-100 (inclusive)\",\n", " },\n", " \"persuasion\": {\n", " \"type\": \"number\",\n", " \"description\": \"Article's persuasion score, 0.0-1.0 (inclusive)\",\n", " },\n", " },\n", " \"required\": [\"author\", \"topics\", \"summary\", \"coherence\", \"persuasion\", \"counterpoint\"],\n", " },\n", " }\n", "]\n", "\n", "content = \"\"\"\\\n", " <article>\n", " {{article}}\n", " </article>\n", "\n", " Use the `print_summary` tool.\\\n", "\"\"\"\n", "\n", "params = MessageCreateParamsBase(\n", " model=\"claude-3-5-haiku-latest\",\n", " max_tokens=4096,\n", " tools=tools,\n", " tool_choice={\"type\": \"tool\", \"name\": \"print_summary\"},\n", " messages=[{\"role\": \"user\", \"content\": dedent(content)}],\n", ")" ] }, { "cell_type": "markdown", "id": "88ce3dc0", "metadata": {}, "source": [ "Save prompt in Phoenix." ] }, { "cell_type": "code", "execution_count": null, "id": "ce1e4ea9", "metadata": {}, "outputs": [], "source": [ "# prompt identifier should contain only alphanumeric characters, hyphens or underscores\n", "prompt_identifier = \"article-summary\"\n", "\n", "prompt = Client().prompts.create(\n", " name=prompt_identifier,\n", " prompt_description=\"Article summary\",\n", " version=PromptVersion.from_anthropic(params),\n", ")" ] }, { "cell_type": "markdown", "id": "879dea55", "metadata": {}, "source": [ "Download articles." ] }, { "cell_type": "code", "execution_count": null, "id": "a62f45d3", "metadata": {}, "outputs": [], "source": [ "articles = []\n", "for item in (\"third-party-testing\", \"alignment-faking\"):\n", " response = requests.get(f\"https://www.anthropic.com/news/{item}\")\n", " soup = BeautifulSoup(response.text, \"html.parser\")\n", " articles.append({\"article\": \" \".join([p.text for p in soup.find_all(\"p\")])})" ] }, { "cell_type": "markdown", "id": "8689c9b7", "metadata": {}, "source": [ "Fetch prompt from Phoenix and apply to data." ] }, { "cell_type": "code", "execution_count": null, "id": "7f3b7247", "metadata": {}, "outputs": [], "source": [ "prompt = Client().prompts.get(prompt_identifier=prompt_identifier)\n", "\n", "\n", "def summarize(input: dict[str, str]):\n", " response = Anthropic().messages.create(**prompt.format(variables=input))\n", " for content in response.content:\n", " if content.type == \"tool_use\":\n", " yield content.input\n", "\n", "\n", "res = pd.json_normalize(chain.from_iterable(map(summarize, articles)))\n", "display(HTML(res.to_html()))" ] }, { "cell_type": "markdown", "id": "5e39893e", "metadata": {}, "source": [ "## Text classification" ] }, { "cell_type": "markdown", "id": "16617f1a", "metadata": {}, "source": [ "Base on [this example](https://colab.research.google.com/github/anthropics/anthropic-cookbook/blob/09c00ae4694b1a754dac26202ec13c1190a54a52/tool_use/extracting_structured_json.ipynb#scrollTo=QAIe78WiDN5B&line=13&uniqifier=1) from Anthropic cookbook." ] }, { "cell_type": "code", "execution_count": null, "id": "57a7049f", "metadata": {}, "outputs": [], "source": [ "tools = [\n", " {\n", " \"name\": \"print_classification\",\n", " \"description\": \"Prints the classification results.\",\n", " \"input_schema\": {\n", " \"type\": \"object\",\n", " \"properties\": {\n", " \"categories\": {\n", " \"type\": \"array\",\n", " \"items\": {\n", " \"type\": \"object\",\n", " \"properties\": {\n", " \"name\": {\"type\": \"string\", \"description\": \"The category name.\"},\n", " \"score\": {\n", " \"type\": \"number\",\n", " \"description\": \"The classification score for the category, ranging from 0.0 to 1.0.\",\n", " },\n", " },\n", " \"required\": [\"name\", \"score\"],\n", " },\n", " }\n", " },\n", " \"required\": [\"categories\"],\n", " },\n", " }\n", "]\n", "\n", "content = \"\"\"\\\n", " <document>\n", " {{text}}\n", " </document>\n", "\n", " Use the print_classification tool. The categories can be Politics, Sports, Technology, Entertainment, Business.\\\n", "\"\"\"\n", "\n", "# https://docs.anthropic.com/en/api/messages\n", "params = MessageCreateParamsBase(\n", " model=\"claude-3-5-haiku-latest\",\n", " max_tokens=4096,\n", " tools=tools,\n", " tool_choice={\"type\": \"tool\", \"name\": \"print_classification\"},\n", " messages=[{\"role\": \"user\", \"content\": dedent(content)}],\n", ")" ] }, { "cell_type": "markdown", "id": "65550ab3", "metadata": {}, "source": [ "Save prompt in Phoenix." ] }, { "cell_type": "code", "execution_count": null, "id": "9bd10ac1", "metadata": {}, "outputs": [], "source": [ "# prompt identifier should contain only alphanumeric characters, hyphens or underscores\n", "prompt_identifier = \"document-classification\"\n", "\n", "prompt = Client().prompts.create(\n", " name=prompt_identifier,\n", " prompt_description=\"Document classification\",\n", " version=PromptVersion.from_anthropic(params),\n", ")" ] }, { "cell_type": "markdown", "id": "7d3c9e92", "metadata": {}, "source": [ "Fetch prompt from Phoenix and apply to data." ] }, { "cell_type": "code", "execution_count": null, "id": "f7f41ec7", "metadata": {}, "outputs": [], "source": [ "prompt = Client().prompts.get(prompt_identifier=prompt_identifier)\n", "\n", "variables = {\n", " \"text\": \"The new quantum computing breakthrough could revolutionize the tech industry.\"\n", "}\n", "response = Anthropic().messages.create(**prompt.format(variables=variables))\n", "for content in response.content:\n", " if content.type == \"tool_use\":\n", " pp(content.input)\n", " break" ] }, { "cell_type": "markdown", "id": "fa64d271", "metadata": {}, "source": [ "## Working with unknown keys" ] }, { "cell_type": "markdown", "id": "ab35ac9e", "metadata": {}, "source": [ "Based on [this example](https://colab.research.google.com/github/anthropics/anthropic-cookbook/blob/09c00ae4694b1a754dac26202ec13c1190a54a52/tool_use/extracting_structured_json.ipynb#scrollTo=QTTcOArJATAu&line=24&uniqifier=1) from Anthropic cookbook." ] }, { "cell_type": "code", "execution_count": null, "id": "028d9cf1", "metadata": {}, "outputs": [], "source": [ "tools = [\n", " {\n", " \"name\": \"print_all_characteristics\",\n", " \"description\": \"Prints all characteristics which are provided.\",\n", " \"input_schema\": {\"type\": \"object\", \"additionalProperties\": True},\n", " }\n", "]\n", "\n", "content = \"\"\"\\\n", " Given a description of a character, your task is to extract all the characteristics of the character and print them using the print_all_characteristics tool.\n", "\n", " The print_all_characteristics tool takes an arbitrary number of inputs where the key is the characteristic name and the value is the characteristic value (age: 28 or eye_color: green).\n", "\n", " <description>\n", " {{desc}}\n", " </description>\n", "\n", " Now use the print_all_characteristics tool.\\\n", "\"\"\"\n", "\n", "params = MessageCreateParamsBase(\n", " model=\"claude-3-5-haiku-latest\",\n", " max_tokens=4096,\n", " tools=tools,\n", " tool_choice={\"type\": \"tool\", \"name\": \"print_all_characteristics\"},\n", " messages=[{\"role\": \"user\", \"content\": dedent(content)}],\n", ")" ] }, { "cell_type": "markdown", "id": "daa63889", "metadata": {}, "source": [ "Save prompt in Phoenix." ] }, { "cell_type": "code", "execution_count": null, "id": "3333291d", "metadata": {}, "outputs": [], "source": [ "# prompt identifier should contain only alphanumeric characters, hyphens or underscores\n", "prompt_identifier = \"character-characteristics\"\n", "\n", "prompt = Client().prompts.create(\n", " name=prompt_identifier,\n", " prompt_description=\"Character characteristics\",\n", " version=PromptVersion.from_anthropic(params),\n", ")" ] }, { "cell_type": "markdown", "id": "b9b5c28b", "metadata": {}, "source": [ "Fetch prompt from Phoenix and apply to data." ] }, { "cell_type": "code", "execution_count": null, "id": "d597ac17", "metadata": {}, "outputs": [], "source": [ "prompt = Client().prompts.get(prompt_identifier=prompt_identifier)\n", "\n", "variables = {\n", " \"desc\": \"The man is tall, with a beard and a scar on his left cheek. He has a deep voice and wears a black leather jacket.\"\n", "}\n", "response = Anthropic().messages.create(**prompt.format(variables=variables))\n", "for content in response.content:\n", " if content.type == \"tool_use\":\n", " pp(content.input)\n", " break" ] } ], "metadata": { "language_info": { "name": "python" } }, "nbformat": 4, "nbformat_minor": 5 }