Constitutional Prompting#
Using techniques from Constitutional AI to generate effective prompts for an AI acting as a Copy Editor.
We’ll be using this Prompt class throughout:
# %load ./prompt.py
import os
import anthropic
from enum import Enum
from dotenv import load_dotenv
load_dotenv()
anthropic_client = anthropic.Client(os.getenv('ANTHROPIC_API_KEY'))
class Model(Enum):
claude_v1_0 = "claude-v1.0"
claude_v1_2 = "claude-v1.2"
claude_v1_3 = "claude-v1.3"
claude_v1_3_100k = "claude-v1.3-100k"
claude_v1_latest = "claude-v1"
claude_v1_latest_100k = "claude-v1-100k"
claude_instant_v1_0 = "claude-instant-v1.0"
claude_instant_v1_1 = "claude-instant-v1.1"
claude_instant_v1_1_100k = "claude-instant-v1.1-100k"
claude_instant_v1_latest = "claude-instant-v1"
claude_instant_v1_latest_100k = "claude-instant-v1-100k"
class Prompt:
def __init__(
self,
human_message: str,
model: Model = Model.claude_v1_latest,
temp_0_1: float = 0.5,
max_tokens_to_sample: int = 1024,
assistant_prefix: str = None,
response_prefix: str = None
):
self.human_message = human_message
self.model = model
self.temp_0_1 = temp_0_1
self.max_tokens_to_sample = max_tokens_to_sample
self.assistant_prefix = assistant_prefix
self.response_prefix = response_prefix
@property
def prompt(self) -> str:
prompt = f"{anthropic.HUMAN_PROMPT} {self.human_message}{anthropic.AI_PROMPT}"
if self.assistant_prefix:
prompt += f" {self.assistant_prefix}"
return prompt
def get_response(self) -> str:
response = anthropic_client.completion(
prompt=self.prompt,
stop_sequences=[anthropic.HUMAN_PROMPT],
model=self.model.value,
max_tokens_to_sample=self.max_tokens_to_sample,
temperature=self.temp_0_1,
)
text = response['completion']
if self.response_prefix:
# This is often useful to clean up after we've used `assistant_prefix` e.g. to start a numbered list of items.
text = f"{self.response_prefix}{text}"
return text
And here are the prompts and constitution rules we’ll be using. Don’t worry about understanding them yet, we’ll go through them step-by-step after this code block.
model = Model.claude_v1_latest
num_samples = 5
# These are best practices for prompts (aka instructions) intended for a Copy Editor AI agent:
constitution_rules = [
"Instructions must explicitly describe the tone that the editor adopts.",
"Instructions must explicitly state the number of words that the editor's written response should be, and the number of words should be appropriate for the task.",
"Instructions must explicitly reference the article content at the beginning, for example \"Here is an article:\n\n<article>{{ARTICLE_CONTENT}}</article>\n\n...\".",
"Instructions which are complicated should be broken down into numbered subtasks.",]
constitution_str = "\n".join(map(lambda x: f"{x[0]+1}. {x[1]}", enumerate(constitution_rules)))
def get_concepts_prompt(n: int) -> Prompt:
return Prompt(
model=model,
temp_0_1=0.7,
human_message=f"Generate a list of {n} tasks that a Copy Editor might do as part of their job, given a written article. The output of each completed task should be written text. Write each task on a separate, numbered line.",
assistant_prefix=f"Here are {n} tasks:\n\n1.",
response_prefix="1.",
)
def get_naive_response_prompt(concept: str) -> Prompt:
return Prompt(
model=model,
temp_0_1=0.3,
human_message=f"""I have an AI agent which acts as a Copy Editor and I want it to complete the following task:
<task>
{concept}
</task>
The agent only responds with written text. Write a concise instruction for the agent, asking it to complete this task. Don't include any preamble, just respond directly with the instruction for the agent.""",
)
def get_naive_response_prompt_with_constitution(concept: str) -> Prompt:
return Prompt(
model=model,
temp_0_1=0.3,
human_message=f"""I have an AI agent which acts as a Copy Editor and I want it to complete the following task:
<task>
{concept}
</task>
The agent only responds with written text. Write a concise instruction for the agent, asking it to complete this task. The instruction must abide by the following rules:
<rules>
{constitution_str}
</rules>
Don't include any preamble, just respond directly with the instruction for the agent.""",
)
def get_critique_prompt(naive_response: str) -> Prompt:
return Prompt(
model=model,
temp_0_1=0.5,
human_message=f"""Here is an instruction to an AI agent which acts as a Copy Editor:
<instruction>
{naive_response}
</instruction>
There may be some problems with this instruction. In particular, the instruction must abide by the following rules:
<rules>
{constitution_str}
</rules>
List each rule that the instruction breaks. State the rule verbatim, then describe how the instruction breaks the rule.
For example, if the instruction breaks rule 1, you would write:
The instruction breaks the following rules:
1.
<rule>{constitution_rules[0]}</rule>
<reason>[Reason for breaking rule 1]</reason>""",
assistant_prefix="The instruction breaks the following rules:\n\n"
)
def get_rewrite_prompt(naive_response: str, critique: str) -> Prompt:
return Prompt(
model=model,
temp_0_1=0.5,
human_message=f"""Here is an instruction to an AI agent which acts as a Copy Editor:
<instruction>
{naive_response}
</instruction>
The instruction is supposed to follow certain rules, but it breaks them as follows:
<issues>
{critique}
</issues>
Rewrite the instruction to address these issues. Do not enclose your answer in <instruction> tags.""",
assistant_prefix="Here is the rewritten instruction:\n\n",
)
Step 0: Generate concepts#
Before we get started on trying to generate prompts, let’s first think of some tasks we might want a Copy Editor to carry out. We’ll run a simple prompt to get a list of example tasks:
concepts_prompt = get_concepts_prompt(num_samples)
print(f"PROMPT:\n\n===\n{concepts_prompt.prompt}\n===\n")
concepts_string = concepts_prompt.get_response()
print(f"TASKS:\n\n===\n{concepts_string}\n===\n")
PROMPT:
===
Human: Generate a list of 5 tasks that a Copy Editor might do as part of their job, given a written article. The output of each completed task should be written text. Write each task on a separate, numbered line.
Assistant: Here are 5 tasks:
1.
===
TASKS:
===
1. Check for proper spelling, grammar, and punctuation.
2. Ensure consistent and proper style (e.g. AP style).
3. Verify facts and citations are accurate.
4. Check for proper flow, clarity, and coherence.
5. Query the author about any unclear or ambiguous information.
===
Now let’s get these items into a list:
import re
def extract_items(text):
items = []
lines = text.split('\n')
for line in lines:
stripped = line.strip()
if stripped and re.match(r'^\d+\.\s', stripped):
items.append(re.sub(r'^\d+\.\s', '', stripped))
return items
concepts = extract_items(concepts_string)
if len(concepts) != num_samples:
raise Exception(f"Expected {num_samples} concepts, but got {len(concepts)}")
else:
print(f"Got {len(concepts)} concepts")
print(concepts)
Got 5 concepts
['Check for proper spelling, grammar, and punctuation.', 'Ensure consistent and proper style (e.g. AP style).', 'Verify facts and citations are accurate.', 'Check for proper flow, clarity, and coherence.', 'Query the author about any unclear or ambiguous information.']
Step 1: Generate naive responses#
First we’ll try our hand at asking the LLM to write prompts for us, given a task, and without much guidance.
Terminology - here we’re prompting an LLM to generate prompts, which is confusing. We’ll use the word “prompt” to refer to the initial input, and “response” to refer to the output
print(f"EXAMPLE PROMPT:\n\n===\n{get_naive_response_prompt(concepts[0]).prompt}\n===\n")
EXAMPLE PROMPT:
===
Human: I have an AI agent which acts as a Copy Editor and I want it to complete the following task:
<task>
Check for proper spelling, grammar, and punctuation.
</task>
The agent only responds with written text. Write a concise instruction for the agent, asking it to complete this task. Don't include any preamble, just respond directly with the instruction for the agent.
Assistant:
===
Let’s execute a prompt like the above, for each of our Copy Editor tasks:
# Anthropic does not allow concurrent requests, so we have to do this sequentially:
naive_responses = []
for concept in concepts:
naive_responses.append(get_naive_response_prompt(concept).get_response())
tasksResponsesString = "\n\n\n===\n\n\n".join(
[f"TASK:\n{task}\n\nNAIVE RESPONSE:\n{response}" for task, response in zip(concepts, naive_responses)])
print(tasksResponsesString)
TASK:
Check for proper spelling, grammar, and punctuation.
NAIVE RESPONSE:
Check the spelling, grammar, and punctuation in the provided text.
===
TASK:
Ensure consistent and proper style (e.g. AP style).
NAIVE RESPONSE:
Apply AP style guidelines to ensure consistent and proper style.
===
TASK:
Verify facts and citations are accurate.
NAIVE RESPONSE:
Verify that all facts and citations in the text are accurate and from reputable sources.
===
TASK:
Check for proper flow, clarity, and coherence.
NAIVE RESPONSE:
Review and revise the text for flow, clarity, coherence, and consistency.
===
TASK:
Query the author about any unclear or ambiguous information.
NAIVE RESPONSE:
Query the author for clarification on any information that is unclear or could be interpreted in multiple ways.
Step 2: Critique#
As you can see, the naive responses above are quite similar to the original wordings of the tasks, sometimes with a bit more detail. Let’s see how closely they follow the constitution rules we defined i.e. best practices for prompt design.
print(f"EXAMPLE PROMPT:\n\n===\n{get_critique_prompt(naive_responses[0]).prompt}\n===\n")
EXAMPLE PROMPT:
===
Human: Here is an instruction to an AI agent which acts as a Copy Editor:
<instruction>
Check the spelling, grammar, and punctuation in the provided text.
</instruction>
There may be some problems with this instruction. In particular, the instruction must abide by the following rules:
<rules>
1. Instructions must explicitly describe the tone that the editor adopts.
2. Instructions must explicitly state the number of words that the editor's written response should be, and the number of words should be appropriate for the task.
3. Instructions must explicitly reference the article content at the beginning, for example "Here is an article:
<article>{{ARTICLE_CONTENT}}</article>
...".
4. Instructions which are complicated should be broken down into numbered subtasks.
</rules>
List each rule that the instruction breaks. State the rule verbatim, then describe how the instruction breaks the rule.
For example, if the instruction breaks rule 1, you would write:
The instruction breaks the following rules:
1.
<rule>Instructions must explicitly describe the tone that the editor adopts.</rule>
<reason>[Reason for breaking rule 1]</reason>
Assistant: The instruction breaks the following rules:
===
Let’s execute a critique prompt like this on each of the naive responses:
critiques = []
for naive_response in naive_responses:
critiques.append(get_critique_prompt(naive_response).get_response())
responseCritiqueString = "\n\n\n===\n\n\n".join(
[f"NAIVE RESPONSE:\n{response}\n\nCRITIQUE:\n{critique}" for response, critique in zip(naive_responses, critiques)])
print(responseCritiqueString)
NAIVE RESPONSE:
Check the spelling, grammar, and punctuation in the provided text.
CRITIQUE:
1.
<rule>Instructions must explicitly describe the tone that the editor adopts.</rule>
<reason>The instruction does not specify the tone (formal, informal, etc.) that the editor should adopt when editing the text.</reason>
2.
<rule>Instructions must explicitly state the number of words that the editor's written response should be, and the number of words should be appropriate for the task.</rule>
<reason>The instruction does not specify the length of the editor's written response.</reason>
3.
<rule>Instructions must explicitly reference the article content at the beginning, for example "Here is an article:
<article>{{ARTICLE_CONTENT}}</article>
...".</rule>
<reason>The instruction does not explicitly reference the article content that the editor is supposed to edit.</reason>
4.
<rule>Instructions which are complicated should be broken down into numbered subtasks.</rule>
<reason>The instruction is a single broad task that could be broken down into subtasks to make it less complicated, but it is not broken down.</reason>
===
NAIVE RESPONSE:
Apply AP style guidelines to ensure consistent and proper style.
CRITIQUE:
1. <rule>Instructions must explicitly describe the tone that the editor adopts.</rule>
<reason> The instruction does not specify the tone the editor should adopt. It only says to apply AP style guidelines, but does not specify if the tone should be formal, informal, etc. </reason>
2. <rule>Instructions must explicitly state the number of words that the editor's written response should be, and the number of words should be appropriate for the task.</rule>
<reason> The instruction does not specify the number of words the editor's response should be. </reason>
3. <rule>Instructions must explicitly reference the article content at the beginning, for example "Here is an article:
<article>{{ARTICLE_CONTENT}}</article>
...". </rule>
<reason> The instruction does not reference any article content. It only provides a general instruction without specifying what content the instruction applies to. </reason>
4. <rule>Instructions which are complicated should be broken down into numbered subtasks.</rule>
<reason> The instruction is a single broad instruction without any numbered subtasks to break it down, even though applying AP style guidelines thoroughly would likely require multiple subtasks. </reason>
===
NAIVE RESPONSE:
Verify that all facts and citations in the text are accurate and from reputable sources.
CRITIQUE:
1. <rule>Instructions must explicitly describe the tone that the editor adopts.</rule>
<reason>The instruction does not specify the tone the editor should adopt.</reason>
2. <rule>Instructions must explicitly state the number of words that the editor's written response should be, and the number of words should be appropriate for the task.</rule>
<reason>The instruction does not specify the number of words for the editor's response.</reason>
3. <rule>Instructions must explicitly reference the article content at the beginning, for example "Here is an article:
<article>{{ARTICLE_CONTENT}}</article>
...".</rule>
<reason>The instruction does not reference an article at the beginning.</reason>
4. <rule>Instructions which are complicated should be broken down into numbered subtasks.</rule>
<reason>The instruction is a single complex task and is not broken down into numbered subtasks.</reason>
===
NAIVE RESPONSE:
Review and revise the text for flow, clarity, coherence, and consistency.
CRITIQUE:
1.
<rule>Instructions must explicitly describe the tone that the editor adopts.</rule>
<reason>The instruction does not specify the tone the editor should adopt when revising the text.</reason>
2.
<rule>Instructions must explicitly state the number of words that the editor's written response should be, and the number of words should be appropriate for the task.</rule>
<reason>The instruction does not specify the number of words the editor's response should be.</reason>
3.
<rule>Instructions must explicitly reference the article content at the beginning, for example "Here is an article:
<article>{{ARTICLE_CONTENT}}</article>
...".</rule>
<reason>The instruction does not reference any article content.</reason>
4.
<rule>Instructions which are complicated should be broken down into numbered subtasks.</rule>
<reason>The instruction is a single broad task that is not broken down into numbered subtasks.</reason>
===
NAIVE RESPONSE:
Query the author for clarification on any information that is unclear or could be interpreted in multiple ways.
CRITIQUE:
1.
<rule>Instructions must explicitly describe the tone that the editor adopts.</rule>
<reason>The instruction does not specify the tone the editor should adopt when querying the author.</reason>
2.
<rule>Instructions must explicitly state the number of words that the editor's written response should be, and the number of words should be appropriate for the task.</rule>
<reason>The instruction does not specify the number of words the editor's response should be.</reason>
3.
<rule>Instructions must explicitly reference the article content at the beginning, for example "Here is an article:
<article>{{ARTICLE_CONTENT}}</article>
...".</rule>
<reason>The instruction does not reference any article content.</reason>
4.
<rule>Instructions which are complicated should be broken down into numbered subtasks.</rule>
<reason>The instruction is a single broad task and is not broken down into numbered subtasks.</reason>
Step 3: Rewrite#
So the naive responses broke every rule in our constitution. Let’s rewrite the responses, using the critiques above as context for our rewriter.
Here’s what an example rewrite prompt looks like:
print(f"EXAMPLE PROMPT:\n\n===\n{get_rewrite_prompt(naive_responses[0], critiques[0]).prompt}\n===\n")
EXAMPLE PROMPT:
===
Human: Here is an instruction to an AI agent which acts as a Copy Editor:
<instruction>
Check the spelling, grammar, and punctuation in the provided text.
</instruction>
The instruction is supposed to follow certain rules, but it breaks them as follows:
<issues>
1.
<rule>Instructions must explicitly describe the tone that the editor adopts.</rule>
<reason>The instruction does not specify the tone (formal, informal, etc.) that the editor should adopt when editing the text.</reason>
2.
<rule>Instructions must explicitly state the number of words that the editor's written response should be, and the number of words should be appropriate for the task.</rule>
<reason>The instruction does not specify the length of the editor's written response.</reason>
3.
<rule>Instructions must explicitly reference the article content at the beginning, for example "Here is an article:
<article>{{ARTICLE_CONTENT}}</article>
...".</rule>
<reason>The instruction does not explicitly reference the article content that the editor is supposed to edit.</reason>
4.
<rule>Instructions which are complicated should be broken down into numbered subtasks.</rule>
<reason>The instruction is a single broad task that could be broken down into subtasks to make it less complicated, but it is not broken down.</reason>
</issues>
Rewrite the instruction to address these issues. Do not enclose your answer in <instruction> tags.
Assistant: Here is the rewritten instruction:
===
Now let’s execute the rewrite prompt on each of the naive responses (with the relevant critique inserted):
rewrites = []
for i, (naive_response, critique) in enumerate(zip(naive_responses, critiques)):
rewrites.append(get_rewrite_prompt(naive_response, critique).get_response())
rewriteString = "\n\n\n===\n\n\n".join(
[f"NAIVE RESPONSE:\n{response}\n\nREWRITE:\n{rewrite}" for response, rewrite in zip(naive_responses, rewrites)])
print(rewriteString)
NAIVE RESPONSE:
Check the spelling, grammar, and punctuation in the provided text.
REWRITE:
Here is an article:
<article>
The quik broun fox jumpd ovr the lazy dog.
</article>
1. Edit the spelling, grammar, and punctuation in the provided article in a formal tone.
2. Your written response should be 3-5 sentences.
3. Check that all words are spelled correctly.
4. Ensure that proper nouns are capitalized.
5. Check that verbs are conjugated correctly.
6. Ensure that punctuation like periods, commas, and quotation marks are used correctly.
===
NAIVE RESPONSE:
Apply AP style guidelines to ensure consistent and proper style.
REWRITE:
Here is an article:
<article>{{ARTICLE_CONTENT}}</article>
Apply AP style guidelines to ensure consistent and proper style, using a formal tone. Respond with 200-300 words.
1. Ensure consistent spelling, punctuation, and grammar.
2. Verify proper use of AP style for titles, abbreviations, numbers, etc.
3. Check that the overall structure and flow of the article is coherent and compelling. Move or reword sentences as needed.
4. Ensure the lead paragraph effectively draws the reader in and that the conclusion is impactful.
5. Double check that all facts stated in the article are accurate and that any quotes are correctly attributed.
6. Flag any remaining issues with the content or flow and provide suggestions for improvement.
===
NAIVE RESPONSE:
Verify that all facts and citations in the text are accurate and from reputable sources.
REWRITE:
Here is an article:
{{ARTICLE_CONTENT}}
1. Adopt a formal and objective tone in your response.
2. Verify that all facts and citations in the article are accurate and from reputable sources in 100-200 words.
3. Ensure that any claims made in the article are supported by evidence from reputable sources.
4. Flag any potential issues with the content, flow, or structure of the article. Provide specific examples and suggestions for improvement.
5. Check that the article is coherent, logically structured, and flows well from one idea to the next. Suggest any reorganization needed.
6. Ensure consistent and appropriate style and formatting throughout the article. Note any needed changes to style, grammar, spelling, or punctuation.
7. Provide an overall assessment of the accuracy, coherence, and quality of the content in 3-4 sentences.
===
NAIVE RESPONSE:
Review and revise the text for flow, clarity, coherence, and consistency.
REWRITE:
Here is an article:
<article>{{ARTICLE_CONTENT}}</article>
Review and revise the following article in a formal and objective tone in 150-200 words:
1. Ensure the flow and coherence of ideas.
2. Check for clarity and consistency in language and style.
3. Revise the structure and organization as needed to improve the flow and coherence.
===
NAIVE RESPONSE:
Query the author for clarification on any information that is unclear or could be interpreted in multiple ways.
REWRITE:
Here is an article:
<article>{{ARTICLE_CONTENT}}</article>
1. In a polite and constructive tone, query the author for clarification on any information in the article that is unclear or could be interpreted in multiple ways. Responses should be 3 to 5 sentences (15 to 25 words) in length.
2. For any claims that seem unsupported or speculative, politely ask the author to provide additional evidence or reword the claim to be less definitive. Responses should again be 3 to 5 sentences (15 to 25 words) in length.
3. Point out any spelling, grammar, or punctuation errors you notice and suggest corrections to the author. Responses can be 1 to 2 sentences (5 to 10 words) in length.
4. Ask the author if they would like you to reorganize or restructure any parts of the article for clarity or flow. Explain your suggested changes and why you think they would improve the article. Responses should be 3 to 5 sentences (15 to 25 words) in length.
5. Request an updated draft from the author and review it to ensure all issues have been addressed. Respond with any final questions or comments. Responses should be 2 to 3 sentences (10 to 15 words) in length.
Step 4: Training a model#
The rewritten responses are much better, generally following every best practice in the constitution rules. These responses, at scale, could serve as useful data for training a model to be good at writing prompts. Let’s mimic this concept using few-shot prompting. We’ll use the first n-1 tasks + rewrites from above as examples, and we’ll try to generate a decent response off the bat for nth task, without referencing the constitution rules. Here’s the prompt we’ll use:
few_shot_prompt = ''
for i, (concept, rewrite) in enumerate(zip(concepts, rewrites)):
few_shot_prompt += f"{anthropic.HUMAN_PROMPT} {get_naive_response_prompt(concept).human_message}"
few_shot_prompt += f"{anthropic.AI_PROMPT}"
if i < len(concepts) - 1:
few_shot_prompt += f" {rewrite}"
print(few_shot_prompt)
Human: I have an AI agent which acts as a Copy Editor and I want it to complete the following task:
<task>
Check for proper spelling, grammar, and punctuation.
</task>
The agent only responds with written text. Write a concise instruction for the agent, asking it to complete this task. Don't include any preamble, just respond directly with the instruction for the agent.
Assistant:
Here is an article:
<article>
The quik broun fox jumpd ovr the lazy dog.
</article>
1. Edit the spelling, grammar, and punctuation in the provided article in a formal tone.
2. Your written response should be 3-5 sentences.
3. Check that all words are spelled correctly.
4. Ensure that proper nouns are capitalized.
5. Check that verbs are conjugated correctly.
6. Ensure that punctuation like periods, commas, and quotation marks are used correctly.
Human: I have an AI agent which acts as a Copy Editor and I want it to complete the following task:
<task>
Ensure consistent and proper style (e.g. AP style).
</task>
The agent only responds with written text. Write a concise instruction for the agent, asking it to complete this task. Don't include any preamble, just respond directly with the instruction for the agent.
Assistant:
Here is an article:
<article>{{ARTICLE_CONTENT}}</article>
Apply AP style guidelines to ensure consistent and proper style, using a formal tone. Respond with 200-300 words.
1. Ensure consistent spelling, punctuation, and grammar.
2. Verify proper use of AP style for titles, abbreviations, numbers, etc.
3. Check that the overall structure and flow of the article is coherent and compelling. Move or reword sentences as needed.
4. Ensure the lead paragraph effectively draws the reader in and that the conclusion is impactful.
5. Double check that all facts stated in the article are accurate and that any quotes are correctly attributed.
6. Flag any remaining issues with the content or flow and provide suggestions for improvement.
Human: I have an AI agent which acts as a Copy Editor and I want it to complete the following task:
<task>
Verify facts and citations are accurate.
</task>
The agent only responds with written text. Write a concise instruction for the agent, asking it to complete this task. Don't include any preamble, just respond directly with the instruction for the agent.
Assistant:
Here is an article:
{{ARTICLE_CONTENT}}
1. Adopt a formal and objective tone in your response.
2. Verify that all facts and citations in the article are accurate and from reputable sources in 100-200 words.
3. Ensure that any claims made in the article are supported by evidence from reputable sources.
4. Flag any potential issues with the content, flow, or structure of the article. Provide specific examples and suggestions for improvement.
5. Check that the article is coherent, logically structured, and flows well from one idea to the next. Suggest any reorganization needed.
6. Ensure consistent and appropriate style and formatting throughout the article. Note any needed changes to style, grammar, spelling, or punctuation.
7. Provide an overall assessment of the accuracy, coherence, and quality of the content in 3-4 sentences.
Human: I have an AI agent which acts as a Copy Editor and I want it to complete the following task:
<task>
Check for proper flow, clarity, and coherence.
</task>
The agent only responds with written text. Write a concise instruction for the agent, asking it to complete this task. Don't include any preamble, just respond directly with the instruction for the agent.
Assistant:
Here is an article:
<article>{{ARTICLE_CONTENT}}</article>
Review and revise the following article in a formal and objective tone in 150-200 words:
1. Ensure the flow and coherence of ideas.
2. Check for clarity and consistency in language and style.
3. Revise the structure and organization as needed to improve the flow and coherence.
Human: I have an AI agent which acts as a Copy Editor and I want it to complete the following task:
<task>
Query the author about any unclear or ambiguous information.
</task>
The agent only responds with written text. Write a concise instruction for the agent, asking it to complete this task. Don't include any preamble, just respond directly with the instruction for the agent.
Assistant:
Now let’s execute the few-shot prompt:
few_shot_response = anthropic_client.completion(
prompt=few_shot_prompt,
stop_sequences=[anthropic.HUMAN_PROMPT],
model=model.value,
max_tokens_to_sample=1024,
temperature=0.5,
)
print(f"TASK:\n\n===\n{concepts[len(concepts) -1]}\n===\n")
print(f"FEW-SHOT RESPONSE:\n\n===\n{few_shot_response['completion']}\n===\n")
TASK:
===
Query the author about any unclear or ambiguous information.
===
FEW-SHOT RESPONSE:
===
Here is an article:
<article>{{ARTICLE_CONTENT}}</article>
1. Review the article and identify any unclear, ambiguous or contradictory information in 3-5 sentences.
2. Pose specific questions to the author to clarify the identified issues. Request additional details or examples where needed.
3. Ask follow up questions as needed to ensure you fully understand the key ideas and arguments presented in the article.
4. Note any suggestions you have for revising or reorganizing the content to improve clarity and flow. Provide 3-4 examples.
5. Summarize the key themes and main takeaways from the article in your own words to confirm your understanding with the author. Keep your summary to 5-7 sentences.
6. Thank the author for clarifying any points of confusion and for the opportunity to provide feedback. Convey your appreciation for the discussion in 2-3 sentences.
===
This looks much better than our original cold attempts. Let’s see if the critical LLM agrees:
print(f"FEW-SHOT CRITIQUE:\n\n===\n{get_critique_prompt(few_shot_response['completion']).get_response()}\n===\n")
FEW-SHOT CRITIQUE:
===
1. <rule>Instructions must explicitly describe the tone that the editor adopts.</rule>
<reason>The instruction does not specify the tone the editor should adopt in the response, such as formal, informal, polite, etc.</reason>
2. <rule>Instructions must explicitly state the number of words that the editor's written response should be, and the number of words should be appropriate for the task.</rule>
<reason>The instruction does not specify the word count for the editor's written response.</reason>
3. <rule>Instructions must explicitly reference the article content at the beginning, for example "Here is an article:
<article>{{ARTICLE_CONTENT}}</article>
...".</rule>
<reason>The instruction satisfies this rule.</reason>
4. <rule>Instructions which are complicated should be broken down into numbered subtasks.</rule>
<reason>The instruction satisfies this rule by breaking down the task into numbered steps.</reason>
===