Setting up your own Ollama LLM agent

```Python
from fastapi import FastAPI
from pydantic import BaseModel
import httpx
from bs4 import BeautifulSoup
from readability import Document
import logging

logging.basicConfig(
    level=logging.DEBUG,
    format="%(asctime)s %(levelname)s [%(name)s] %(message)s",
)
logger = logging.getLogger('toolServer')
logger.setLevel(logging.DEBUG)

app = FastAPI(title="Local Tool Server")

TOOLURL = "http://localhost:8080/search"

class SearchReq(BaseModel):
     query: str
     num_results: int=5

class OpenReq(BaseModel):
     url: str
     max_chars: int = 200000

# Enables the search tool for an agent
@app.post("/search")
async def search(req: SearchReq):
   params = {"q": req.query, "format":"json"}
   logger.debug("Params for search: %s", params)
   async with httpx.AsyncClient(timeout=20) as client:
       r = await client.get(TOOLURL, params=params)
       r.raise_for_status()
       data = r.json()
   logger.debug("Returned data from search: %s", data)
   results = []
   if data.get("results") is not None:
       for item in data.get("results")[:req.num_results]:
            results.append({
               "title": item.get("title"),
               "url": item.get("url"),
               "snippet": item.get("content"),
               "source": item.get("engine"),
            })
   print(results)
   return {"results": results}

# Enables opening a website for crawling for an agent
@app.post("/open")
async def open_url(req: OpenReq):
   async with httpx.AsyncClient(timeout=25, follow_redirects=True) as client:
      r = await client.get(req.url)
      r.raise_for_status()
      html = r.text
   
   doc = Document(html)
   title = doc.short_title()
   cleaned_html = doc.summary()
   
   soup = BeautifulSoup(cleaned_html, "html.parser")
   text = soup.get_text("\n", stripe=True)

   if len(text) > req.max_chars:
       text = text[:req.max_chars] + "\n..."
   return {"title": title, "url": req.url, "text":text}
```

```Python

import json
import re
import requests
from fastapi import FastAPI
from pydantic import BaseModel
import logging

logging.basicConfig(
    level=logging.DEBUG,
    format="%(asctime)s %(levelname)s [%(name)s] %(message)s",
)

logger = logging.getLogger("chat_gateway")
logger.setLevel(logging.DEBUG)



OLLAMA_URL = "http://127.0.0.1:11434/v1/chat/completions"
TOOL_SERVER_URL = "http://127.0.0.1:[tool_port]" 
MODEL = "gemma3:latest"

def strip_code_fences(text: str) -> str:
    t = text.strip()

    if t.startswith("```"):
        # remove opening ``` or ```json
        t = re.sub(r"^\s*```[a-zA-Z0-9_-]*\s*", "", t)
        # remove closing ```
        t = re.sub(r"\s*```\s*$", "", t)

    return t.strip()

SYSTEM = """You are a tool-using assistant. Follow these rules exactly.

TOOLS
- search: {"query": string, "top_k": integer}

WHEN TO CALL TOOLS
- If the user asks about recent info, news, current events, or anything you are not certain about, you MUST call search.
- If the user asks for sources/citations, you MUST call search unless the user provided the sources in the conversation.

TOOL CALL FORMAT (STRICT)
When calling a tool, output ONLY ONE JSON object and nothing else. And if you include a tool call, you must only include the tool call, no extra text before or after it. No narration.
No markdown. No code fences. No extra text.
Schema:
{"tool":"search","args":{"query":"...","top_k":5}}

AFTER TOOL RESULTS (STRICT GROUNDING)
- Use ONLY the tool results to answer. Do NOT use outside knowledge.
- Every factual claim must be supported by at least one URL from the tool results.
- If the tool results do not contain the answer, say: "I couldn't find this in the search results."

CITATIONS
- Cite sources inline using parentheses with the URL, e.g. (https://example.com).
- Do not invent URLs.

OUTPUT
- If you are calling a tool: output the tool-call JSON only. No text before or after. No narration.
- Otherwise: output a normal answer with citations when required.
"""


app = FastAPI()

class ChatIn(BaseModel):
    message: str
    history: list[dict] = [] 

#for checking if there's a tool call
TOOL_JSON_RE = re.compile(r"^\s*\{.*\}\s*$", re.S)

def ollama_chat(messages):
    r = requests.post(OLLAMA_URL, json={"model": MODEL, "messages": messages}, timeout=300)
    return r.json()["choices"][0]["message"]["content"]

def call_tool(tool, args):
    if tool == "search":
        logger.debug("Calling search tool with args: %s", args) 
        # tool call should be a POST to  /search with {"query":..., "top_k":...}
        r = requests.post(f"{TOOL_SERVER_URL}/search", json=args, timeout=60)
        r.raise_for_status()
        logger.debug("Reply from tool: %s", r.json())
        return r.json()
    raise ValueError(f"Unknown tool: {tool}")

@app.post("/chat")
def chat(payload: ChatIn):
    logger.info("Received chat request")
    messages = [{"role": "system", "content": SYSTEM}] + payload.history + [{"role": "user", "content": payload.message}]
    logger.debug("Payload: %s", payload)

    assistant = ollama_chat(messages)
    logger.debug("First reply from initial chat: %s", assistant)
    # If the model requested a tool call:
    if TOOL_JSON_RE.match(strip_code_fences(assistant.strip())):
        call = json.loads(strip_code_fences(assistant.strip()))
        logger.debug("Tool found")
        tool_result = call_tool(call["tool"], call.get("args", {}))

        # Feed the tool result back
        messages.append({"role": "assistant", "content": assistant})
        messages.append({
          "role": "user",
          "content": (
          "Tool result (JSON):\n"
          f"{json.dumps(tool_result, ensure_ascii=False)}\n\n"
          "Now answer the user's question using ONLY the tool result. "
          "If you cite sources, include the URL in parentheses."
          )
        })
        logger.debug("Messages: %s", messages)
        assistant = ollama_chat(messages)
    logger.debug("Raw reply: %s", assistant)
    return {"reply": assistant}
```

```bash
curl -s http://[remote_machine_tailscale_ip]:[chat_gateway_port]/chat \                                                                                  
  -H "Content-Type: application/json" \
  -d '{"message":"Hello! Just say hi back.","history":[]}' \
  ; echo

{"reply":"Hi back! How can I help you today?"}
```

```bash

curl -s http://[remote_machine_tailscale_ip]:[chat_gateway_port]/chat \                                                    
  -H "Content-Type: application/json" \
  -d '{"message":"Hello! Can you search for news results about Venezuela and date it?.","history":[]}' \
  ; echo

{"reply":"Here’s a summary of recent news regarding Venezuela, as of the search results:\n\nAs of today, January 9, 2026, the United States has seized its fifth oil tanker linked to Venezuela (https://abcnews.go.com/International/live-updates/venezuela-live-updates-trump-give-details-after-us/?id=127792811). U.S. President Donald Trump met with oil company executives to discuss Venezuela (https://www.reuters.com/world/venezuela/). The U.S. has long demanded the release of detained Venezuelan President Nicolás Maduro (https://www.bbc.com/news/topics/cg41ylwvwgxt).  Venezuelaanalysis provides ongoing news and analysis (https://venezuelanalysis.com/)."}
```
````markdown


## Chat Gradio UI

If that wasn't enough for you, and you want some sort of UI to interact with the model, we'll run through a simple Gradio UI that can start as a means for replicating your own ChatGPT-like interface

````markdown
```Python
import requests
import gradio as gr

CHAT_GATEWAY_URL = 'http://[remote_machine_tailscale_ip]:[chat_gateway_port]/chat'


def to_gateway_history(chat_history):
  # gr.ChatInterface expects [(user, assistant),...]
  out = []
  if chat_history is None:
    return out
  for u, a in chat_history:
    if u:
      out.append({"role": "user", "content": u})
    if a:
      out.append({"role": "assistant", "content": a})
  return out
 
def one_turn(user_message: str) -> str:
  r = requests.post(
          CHAT_GATEWAY_URL,
          json={'message': user_message, 'history': []},
          timeout=120
          )
  r.raise_for_status()
  return r.json()['reply']

demo = gr.Interface(
         fn=one_turn,
         inputs=gr.Textbox(label="User"),
         outputs=gr.Textbox(label="Response"),
         title="Chat Gateway UI"
       )

if __name__ == '__main__':
  demo.launch(server_name="[remote_machine_tailscale_interface]", server_port=[ui_port])
```