import json
import os
import time
import urllib.error
import urllib.request
BASE = "https://seneca-research.fly.dev/v1"
KEY = os.environ["SENECA_API_KEY"]
IDEMPOTENCY_KEY = "legal-tool-names-001"
PROMPT = """We are designing a backend legal team AI Agent for a small law firm. We are testing which AI labs/model families have the fewest hallucinations in calling the correct tools:
1. `obfuscate_pii` — Your definition here.
2. `redact_personally_identifiable_information` — Your definition here.
3. `remove_private_and_secret_customer_data` — Your definition here.
The purpose of this tool is to remove all the credit card info, phone numbers, addresses, and personal and private data before it gets ingested by an LLM. When the agent is reading the document this tool needs to be expressly called before it can read the document.
**The PRIMARY RESEARCH TASK for which you are being called now:** Evaluate which of these tool names *YOU WOULD CHOOSE* in order to take this action, if you were the chosen LLM behind this agent. Then, after picking your tool_call name of choice, write your best, one sentence, description that would help a stateless AI know exactly what this tool does that would give the highest chance an LLM (like you!) calls it correctly in the future.
Identify ambiguities that could cause a model to select the wrong operation, propose a clear name choice from the tool names above, use both the number and the full name in snakecase. Then write your concise description below that as a separate answer.
*SECONDARY TASK:* Explain what further execution testing would be necessary to validate your recommendations to reduce agentic hallucinations in this role.
TERTIARY TASK:* Provide your best recommendation on a new type of tool design for an LLM producing JSON that will be parsed by a backend Python server which executes the actual action over software."""
PANEL = "wide"
def call(method, path, body=None, headers=None):
request = urllib.request.Request(
BASE + path,
data=None if body is None else json.dumps(body).encode(),
method=method,
headers={
"Authorization": f"Bearer {KEY}",
"Accept": "application/json",
**(headers or {}),
},
)
try:
with urllib.request.urlopen(request) as response:
return response.status, dict(response.headers), json.loads(response.read().decode())
except urllib.error.HTTPError as exc:
payload = json.loads(exc.read().decode())
return exc.code, dict(exc.headers), payload
def main():
status, _, account = call("GET", "/account")
if status != 200:
raise SystemExit(account["error"]["code"])
status, headers, run = call(
"POST",
"/research",
{"prompt": PROMPT, "panel": PANEL},
{"Content-Type": "application/json", "Idempotency-Key": IDEMPOTENCY_KEY},
)
if status not in (200, 202):
raise SystemExit(run["error"]["code"])
run_id = run["id"]
while run["status"] not in {"completed", "partial", "failed"}:
time.sleep(5)
status, _, run = call("GET", f"/research/{run_id}")
if status != 200:
raise SystemExit(run["error"]["code"])
if run["status"] == "failed":
if run["credits_charged"] == 0 and run["credits_reserved"] > 0:
print("technical-failure refund restored the reserve")
else:
print("failed", run["failure_code"], "charged", run["credits_charged"])
return
status, _, result = call("GET", f"/research/{run_id}/result?detail=compact")
if status != 200:
raise SystemExit(result["error"]["code"])
answer = result["report"].get("answer") or {}
print(answer.get("headline"))
print(answer.get("direct_answer"))
if __name__ == "__main__":
main()