//! Bounded HTTP or HTTPS downloader for model-selected public pages.
use std::time::Duration;
use reqwest::{
Client,
header::{ACCEPT, ACCEPT_ENCODING, CONTENT_ENCODING, CONTENT_LENGTH, CONTENT_TYPE, USER_AGENT},
};
use serde_json::{Map, Value};
use url::Url;
use crate::config::WebConfig;
use super::{ToolResult, ToolScope};
#[derive(Clone)]
pub(super) struct FetchWebPageTool
{
client: Client,
config: WebConfig,
}
impl FetchWebPageTool
{
pub(super) fn new(config: WebConfig) -> Result<Self, reqwest::Error>
{
let client = Client::builder()
.no_proxy()
.connect_timeout(Duration::from_secs(config.connect_timeout_seconds))
.read_timeout(Duration::from_secs(config.read_timeout_seconds))
.timeout(Duration::from_secs(config.total_timeout_seconds))
.redirect(reqwest::redirect::Policy::custom(move |attempt|
{
if attempt.previous().len() > config.max_redirects as usize
|| !matches!(attempt.url().scheme(), "http" | "https")
{
attempt.stop()
}
else
{
attempt.follow()
}
}))
.build()?;
Ok(Self { client, config })
}
pub(super) async fn execute(&self, arguments: Value, scope: &mut ToolScope) -> ToolResult
{
let Some(object) = arguments.as_object() else
{
return ToolResult::failure("invalid_arguments", "Tool arguments must be an object.");
};
if object.len() != 1 || !object.contains_key("url")
{
return ToolResult::failure("invalid_arguments", "Only a URL argument is accepted.");
}
let Some(raw_url) = object.get("url").and_then(Value::as_str) else
{
return ToolResult::failure("invalid_arguments", "A URL string is required.");
};
let Some(url) = validate_url(raw_url) else
{
return ToolResult::failure("invalid_url", "Only valid HTTP or HTTPS URLs without credentials are allowed.");
};
let response = match self.client.get(url)
.header(USER_AGENT, &self.config.user_agent)
.header(ACCEPT, "text/html, text/plain;q=0.9")
.header(ACCEPT_ENCODING, "identity")
.send().await
{
Ok(response) if response.status().is_success() => response,
Ok(response) => return ToolResult::failure("http_error", format!("The page returned HTTP {}.", response.status().as_u16())),
Err(_) => return ToolResult::failure("timeout", "The page could not be downloaded within the configured limit."),
};
if response.headers().get(CONTENT_ENCODING).is_some_and(|value| value != "identity")
{
return ToolResult::failure("unsupported_content_encoding", "The page used an unsupported content encoding.");
}
if response.headers().get(CONTENT_LENGTH)
.and_then(|value| value.to_str().ok())
.and_then(|value| value.parse::<usize>().ok())
.is_some_and(|size| size > self.config.max_download_bytes)
{
return ToolResult::failure("download_too_large", "The page exceeded the configured download-size limit.");
}
let final_url = response.url().clone();
let content_type = response.headers().get(CONTENT_TYPE)
.and_then(|value| value.to_str().ok()).unwrap_or("")
.split(';').next().unwrap_or("").trim().to_ascii_lowercase();
if content_type != "text/html" && content_type != "text/plain"
{
return ToolResult::failure("unsupported_content_type", "Only HTML and plain-text pages are supported.");
}
let mut bytes = Vec::new();
let mut response = response;
loop
{
match response.chunk().await
{
Ok(Some(chunk)) =>
{
if chunk.len() > self.config.max_download_bytes.saturating_sub(bytes.len())
{
return ToolResult::failure("download_too_large", "The page exceeded the configured download-size limit.");
}
bytes.extend_from_slice(&chunk);
}
Ok(None) => break,
Err(_) => return ToolResult::failure("timeout", "The page download did not complete."),
}
}
let title = if content_type == "text/html" { html_title(&bytes) } else { final_url.to_string() };
scope.sources.register(final_url.clone(), title.clone());
let artifact_id = match scope.artifacts.insert(content_type.clone(), bytes.clone(), final_url.clone())
{
Ok(identifier) => identifier,
Err(error) => return error,
};
let mut metadata = Map::new();
metadata.insert("artifact_id".to_owned(), Value::String(artifact_id));
metadata.insert("final_url".to_owned(), Value::String(final_url.to_string()));
metadata.insert("content_type".to_owned(), Value::String(content_type.clone()));
metadata.insert("byte_count".to_owned(), Value::from(bytes.len()));
metadata.insert("title".to_owned(), Value::String(title));
let content = if content_type == "text/plain"
{
String::from_utf8_lossy(&bytes).chars()
.take(self.config.max_tool_output_characters as usize).collect()
}
else
{
"HTML was stored as an artifact. Use html_to_markdown to read it.".to_owned()
};
ToolResult::success("fetch_ok", content, metadata)
}
}
fn validate_url(value: &str) -> Option<Url>
{
if value.chars().any(char::is_control) || value.contains('%') && !valid_percent_escapes(value)
{
return None;
}
let mut url = Url::parse(value).ok()?;
if !matches!(url.scheme(), "http" | "https") || url.host_str().is_none()
|| !url.username().is_empty() || url.password().is_some()
{
return None;
}
url.set_fragment(None);
Some(url)
}
fn valid_percent_escapes(value: &str) -> bool
{
let bytes = value.as_bytes();
let mut index = 0;
while index < bytes.len()
{
if bytes[index] == b'%'
{
if index + 2 >= bytes.len() || !bytes[index + 1].is_ascii_hexdigit()
|| !bytes[index + 2].is_ascii_hexdigit()
{
return false;
}
index += 3;
}
else
{
index += 1;
}
}
true
}
fn html_title(bytes: &[u8]) -> String
{
let html = String::from_utf8_lossy(bytes);
let lower = html.to_ascii_lowercase();
let Some(start) = lower.find("<title") else { return "Web page".to_owned(); };
let Some(content_start) = lower[start..].find('>').map(|index| start + index + 1) else { return "Web page".to_owned(); };
let Some(end) = lower[content_start..].find("</title>").map(|index| content_start + index) else { return "Web page".to_owned(); };
html[content_start..end].chars().take(200).collect()
}
#[cfg(test)]
mod tests
{
use super::validate_url;
#[test]
fn accepts_only_public_http_urls()
{
assert!(validate_url("https://example.test/a#fragment").is_some());
assert!(validate_url("file:///etc/passwd").is_none());
assert!(validate_url("ftp://example.test/file").is_none());
assert!(validate_url("data:text/plain,hello").is_none());
assert!(validate_url("gopher://example.test/1").is_none());
assert!(validate_url("//example.test/path").is_none());
assert!(validate_url("https://user:password@example.test").is_none());
assert!(validate_url("https://example.test/%zz").is_none());
assert!(validate_url("https://example.test/\npath").is_none());
}
}