diff --git a/Cargo.toml b/Cargo.toml index 56ba45b..921fe90 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,7 @@ name = "agent-search" version = "0.9.0" edition = "2021" -description = "Unified multi-provider search CLI for AI agents — 13 providers, 13 modes, email verification, one binary" +description = "Unified multi-provider search CLI for AI agents — 14 providers, 13 modes, email verification, one binary" license = "MIT" repository = "https://github.com/paperfoot/search-cli" homepage = "https://github.com/paperfoot/search-cli" diff --git a/README.md b/README.md index 06e9cc2..4abebc7 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ # Search CLI — Web Search for AI Agents -**One binary, 13 providers, 13 modes, rank-fused results. The web search tool your AI agent is missing.** +**One binary, 14 providers, 13 modes, rank-fused results. The web search tool your AI agent is missing.**
@@ -19,7 +19,7 @@ --- -A single Rust binary that aggregates Brave, Serper, Exa, Linkup, Jina, Firecrawl, Tavily, SerpApi, Perplexity, Parallel, xAI, and more into one search interface. Built for AI agents from day one: structured JSON, semantic exit codes, self-describing `agent-info`, reciprocal rank fusion across providers, and a `usage` command that reports remaining API credits. +A single Rust binary that aggregates Brave, Serper, Exa, Linkup, Jina, You.com, Firecrawl, Tavily, SerpApi, Perplexity, Parallel, xAI, and more into one search interface. Built for AI agents from day one: structured JSON, semantic exit codes, self-describing `agent-info`, reciprocal rank fusion across providers, and a `usage` command that reports remaining API credits. [Install](#install) | [How It Works](#how-it-works) | [Features](#features) | [Providers](#providers) | [Contributing](#contributing) @@ -125,8 +125,8 @@ same routing registry the engine uses. | Mode | Use when | `-q` is | Providers used | |------|----------|---------|----------------| -| `general` | Any web lookup not covered below (default) | query | Parallel + Brave + Serper + Exa + Jina + Linkup + Tavily + Perplexity | -| `news` | Current events; add `-f day`/`-f week` | query | Parallel + Brave + Serper + Linkup + Tavily + Perplexity (news endpoints) | +| `general` | Any web lookup not covered below (default) | query | Parallel + Brave + Serper + Exa + Jina + Linkup + You.com + Tavily + Perplexity | +| `news` | Current events; add `-f day`/`-f week` | query | Parallel + Brave + Serper + Linkup + You.com + Tavily + Perplexity (news endpoints) | | `academic` | Papers/studies by topic (semantic + web) | query | Exa + Serper + Tavily + Perplexity | | `scholar` | Google Scholar records: citations, PDFs | query | Serper + SerpApi | | `deep` | Max coverage; waits for all providers — use `-c 30` | query | Parallel + Brave (web + LLM Context) + Serper + Exa + Linkup + Tavily + Perplexity + xAI | @@ -226,6 +226,7 @@ search "query" 2>/dev/null # suppress diagnostics | **[Exa](https://exa.ai/)** | Neural/semantic search, category filters | Research papers, people search, similar sites | | **[Jina](https://jina.ai/)** | Fast URL-to-markdown, 500 RPM free tier | Reading article content, quick extraction | | **[Linkup](https://www.linkup.so/)** | High-accuracy agent search (leads the SimpleQA benchmark) | Factual lookups where accuracy matters most | +| **You.com** | Current web + news snippets | Fast general search and news-grounded results | | **[Firecrawl](https://firecrawl.dev/)** | JavaScript rendering, structured extraction | Dynamic pages, SPAs, data extraction | | **[Tavily](https://tavily.com/)** | General + deep search, research-focused | Broad coverage, research queries | | **[SerpApi](https://serpapi.com/)** | Many engines: Google, Bing, YouTube, Baidu | Multi-engine coverage; only provider with a real balance API | diff --git a/src/cli.rs b/src/cli.rs index 4e4e8bb..3dbeb85 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -6,7 +6,7 @@ use clap::{Parser, Subcommand}; name = "search", version, about = "Agent-friendly multi-provider search CLI", - long_about = "Aggregates 13 search providers across 13 explicit search modes.\n\ + long_about = "Aggregates 14 search providers across 13 explicit search modes.\n\ You choose the mode (-m) and/or providers (-p); the CLI does not guess\n\ intent. Run `search agent-info` for the machine-readable capability map.\n\ Outputs colored tables for humans, JSON when piped to other tools.\n\n\ @@ -17,6 +17,7 @@ use clap::{Parser, Subcommand}; exa Neural/semantic search, LinkedIn people, find-similar\n \ jina Fast web search + URL-to-markdown reader\n \ linkup High-accuracy agent search (SimpleQA leader)\n \ + youcom You.com web search + news snippets\n \ firecrawl JS-rendered page scraping + structured extraction\n \ tavily RAG-oriented search: general, news, academic, deep\n \ serpapi Many engines (Google, Bing, YouTube, Baidu, Scholar)\n \ @@ -182,7 +183,7 @@ pub struct SearchArgs { #[arg(short, long)] pub count: Option, - /// Use only specific providers (comma-separated: parallel,brave,serper,exa,jina,linkup,firecrawl,tavily,serpapi,perplexity,browserless,stealth,xai) + /// Use only specific providers (comma-separated: parallel,brave,serper,exa,jina,linkup,youcom,firecrawl,tavily,serpapi,perplexity,browserless,stealth,xai) #[arg(short, long, value_delimiter = ',')] pub providers: Option>, diff --git a/src/config.rs b/src/config.rs index 0652561..49f9134 100644 --- a/src/config.rs +++ b/src/config.rs @@ -48,6 +48,8 @@ pub struct ApiKeys { #[serde(default)] pub linkup: String, #[serde(default)] + pub youcom: String, + #[serde(default)] pub firecrawl: String, #[serde(default)] pub tavily: String, @@ -86,6 +88,7 @@ impl Default for AppConfig { exa: String::new(), jina: String::new(), linkup: String::new(), + youcom: String::new(), firecrawl: String::new(), tavily: String::new(), serpapi: String::new(), @@ -151,6 +154,7 @@ pub const PROVIDER_KEYS: &[&str] = &[ "exa", "jina", "linkup", + "youcom", "firecrawl", "tavily", "serpapi", @@ -206,6 +210,7 @@ pub fn config_show(config: &AppConfig) { ("exa", &config.keys.exa, "EXA_API_KEY"), ("jina", &config.keys.jina, "JINA_API_KEY"), ("linkup", &config.keys.linkup, "LINKUP_API_KEY"), + ("youcom", &config.keys.youcom, "YDC_API_KEY"), ("firecrawl", &config.keys.firecrawl, "FIRECRAWL_API_KEY"), ("tavily", &config.keys.tavily, "TAVILY_API_KEY"), ("serpapi", &config.keys.serpapi, "SERPAPI_API_KEY"), @@ -477,6 +482,12 @@ pub fn config_check(config: &AppConfig) { "LINKUP_API_KEY", "High-accuracy agent search (SimpleQA leader)", ), + ( + "youcom", + &config.keys.youcom, + "YDC_API_KEY", + "Web search and news snippets", + ), ( "firecrawl", &config.keys.firecrawl, diff --git a/src/errors.rs b/src/errors.rs index 4b4b214..9daf696 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -146,11 +146,7 @@ impl SearchError { pub fn suggestion(&self) -> Option { match self { - Self::AuthMissing { provider } => Some(format!( - "Set {}_API_KEY env var, or: echo YOUR_KEY | search config set keys.{} -", - provider.to_uppercase(), - provider - )), + Self::AuthMissing { provider } => Some(auth_missing_suggestion(provider)), Self::NoProviders(mode) => Some(format!( "No providers configured for mode '{}'. Run: search config check", mode @@ -187,6 +183,18 @@ impl SearchError { } } +fn auth_missing_suggestion(provider: &str) -> String { + match provider { + "youcom" => "Set YDC_API_KEY env var, or: echo YOUR_KEY | search config set keys.youcom -" + .to_string(), + _ => format!( + "Set {}_API_KEY env var, or: echo YOUR_KEY | search config set keys.{} -", + provider.to_uppercase(), + provider + ), + } +} + /// Scrub credential values from user-visible strings. Transport errors can /// embed full request URLs (SerpApi authenticates via `?api_key=` in the /// query string), and provider error bodies sometimes echo the caller's key. diff --git a/src/main.rs b/src/main.rs index abfdffe..492fd40 100644 --- a/src/main.rs +++ b/src/main.rs @@ -94,6 +94,7 @@ async fn main() { "google.serper.dev:443", "api.exa.ai:443", "api.jina.ai:443", + "ydc-index.io:443", "api.tavily.com:443", "api.perplexity.ai:443", ]; @@ -192,6 +193,7 @@ async fn main() { "https://api.search.brave.com/res/v1/web/search", "https://google.serper.dev/search", "https://api.exa.ai/search", + "https://ydc-index.io/v1/search", ]; for url in urls { let _ = app_c.client.head(url).send().await; @@ -537,7 +539,7 @@ async fn run(cli: Cli, ctx: &Ctx, app: Arc) -> Result { if ctx.is_json() { // Use the same resolver as `config check` (is_configured -> - // resolve_key) so env-only keys count and all 12 providers + // resolve_key) so env-only keys count and all providers // are covered — the old hardcoded list missed parallel + // stealth and ignored env vars. let all = providers::build_providers(&app); @@ -689,7 +691,7 @@ async fn run(cli: Cli, ctx: &Ctx, app: Arc) -> Result) -> Result) -> Vec> { Box::new(exa::Exa::new(ctx.clone())), Box::new(jina::Jina::new(ctx.clone())), Box::new(linkup::Linkup::new(ctx.clone())), + Box::new(youcom::YouCom::new(ctx.clone())), #[cfg(feature = "stealth")] Box::new(stealth::Stealth::new(ctx.clone())), Box::new(firecrawl::Firecrawl::new(ctx.clone())), diff --git a/src/providers/youcom.rs b/src/providers/youcom.rs new file mode 100644 index 0000000..768babf --- /dev/null +++ b/src/providers/youcom.rs @@ -0,0 +1,287 @@ +use crate::context::AppContext; +use crate::errors::SearchError; +use crate::types::{SearchOpts, SearchResult}; +use async_trait::async_trait; +use serde::Deserialize; +use serde_json::json; +use std::sync::Arc; +use std::time::Duration; + +pub struct YouCom { + ctx: Arc, +} + +impl YouCom { + pub fn new(ctx: Arc) -> Self { + Self { ctx } + } + + fn api_key(&self) -> String { + super::resolve_key(&self.ctx.config.keys.youcom, "YDC_API_KEY") + } +} + +#[derive(Debug, Deserialize)] +struct YouResponse { + results: Option, +} + +#[derive(Debug, Deserialize)] +struct YouResults { + web: Option>, + news: Option>, +} + +#[derive(Debug, Deserialize)] +struct YouResult { + title: Option, + url: Option, + description: Option, + snippets: Option>, + page_age: Option, + thumbnail_url: Option, +} + +fn build_body(query: &str, count: usize, opts: &SearchOpts) -> serde_json::Value { + let mut body = json!({ + "query": query, + "count": count.min(100).max(1), + }); + + if !opts.include_domains.is_empty() { + body["include_domains"] = json!(opts.include_domains); + } + if !opts.exclude_domains.is_empty() { + body["exclude_domains"] = json!(opts.exclude_domains); + } + if let Some(freshness) = &opts.freshness { + body["freshness"] = json!(freshness); + } + if let Some(country) = &opts.country { + body["country"] = json!(country.to_uppercase()); + } + if let Some(lang) = &opts.lang { + body["language"] = json!(lang.to_uppercase()); + } + + body +} + +fn join_snippets(description: Option, snippets: Option>) -> String { + let mut out = description.unwrap_or_default(); + let joined = snippets + .unwrap_or_default() + .into_iter() + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) + .collect::>() + .join("\n"); + if !joined.is_empty() { + if !out.is_empty() { + out.push('\n'); + } + out.push_str(&joined); + } + out +} + +fn map_result(result: YouResult, source: &str) -> Option { + let url = result.url.unwrap_or_default(); + if url.is_empty() { + return None; + } + + Some(SearchResult { + title: result.title.unwrap_or_default(), + url, + snippet: join_snippets(result.description, result.snippets), + source: source.to_string(), + published: result.page_age, + image_url: result.thumbnail_url, + extra: None, + }) +} + +fn collect_results(resp: YouResponse, news_only: bool) -> Vec { + let sections = resp.results.unwrap_or(YouResults { + web: None, + news: None, + }); + + let mut out = Vec::new(); + let push_section = |results: Option>, source: &str, out: &mut Vec| { + if let Some(results) = results { + out.extend(results.into_iter().filter_map(|r| map_result(r, source))); + } + }; + + if news_only { + push_section(sections.news, "youcom_news", &mut out); + if out.is_empty() { + push_section(sections.web, "youcom", &mut out); + } + } else { + push_section(sections.web, "youcom", &mut out); + push_section(sections.news, "youcom_news", &mut out); + } + + out +} + +async fn search_impl( + ctx: &AppContext, + key: String, + query: &str, + count: usize, + opts: &SearchOpts, + news_only: bool, +) -> Result, SearchError> { + if key.is_empty() { + return Err(SearchError::AuthMissing { provider: "youcom" }); + } + + let body = build_body(query, count, opts); + super::retry_request(|| async { + let resp = ctx + .client + .post("https://ydc-index.io/v1/search") + .header("X-API-Key", key.as_str()) + .header("Content-Type", "application/json") + .json(&body) + .send() + .await?; + + let resp = super::ok_or_api_error(resp, "youcom").await?; + + let body_bytes = resp.bytes().await?; + let mut body_vec = body_bytes.to_vec(); + let parsed: YouResponse = + simd_json::from_slice(&mut body_vec).map_err(|e| SearchError::Api { + provider: "youcom", + code: "json_error", + status: None, + message: e.to_string(), + })?; + Ok(collect_results(parsed, news_only)) + }) + .await +} + +#[async_trait] +impl super::Provider for YouCom { + fn name(&self) -> &'static str { + "youcom" + } + + fn capabilities(&self) -> &[&'static str] { + &["general", "news", "deep"] + } + + fn env_keys(&self) -> &[&'static str] { + &["YDC_API_KEY", "SEARCH_KEYS_YOUCOM"] + } + + fn is_configured(&self) -> bool { + !self.api_key().is_empty() + } + + fn timeout(&self) -> Duration { + Duration::from_secs(15) + } + + async fn search( + &self, + query: &str, + count: usize, + opts: &SearchOpts, + ) -> Result, SearchError> { + search_impl(&self.ctx, self.api_key(), query, count, opts, false).await + } + + async fn search_news( + &self, + query: &str, + count: usize, + opts: &SearchOpts, + ) -> Result, SearchError> { + search_impl(&self.ctx, self.api_key(), query, count, opts, true).await + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn build_body_applies_limits_and_filters() { + let opts = SearchOpts { + include_domains: vec!["example.com".into()], + exclude_domains: vec!["spam.example".into()], + freshness: Some("week".into()), + country: Some("us".into()), + lang: Some("en".into()), + }; + + let body = build_body("rust search", 250, &opts); + assert_eq!(body["query"], "rust search"); + assert_eq!(body["count"], 100); + assert_eq!(body["include_domains"][0], "example.com"); + assert_eq!(body["exclude_domains"][0], "spam.example"); + assert_eq!(body["freshness"], "week"); + assert_eq!(body["country"], "US"); + assert_eq!(body["language"], "EN"); + } + + #[test] + fn maps_web_and_news_sections() { + let resp = YouResponse { + results: Some(YouResults { + web: Some(vec![YouResult { + title: Some("Web".into()), + url: Some("https://example.com/web".into()), + description: Some("web desc".into()), + snippets: Some(vec!["one".into(), "two".into()]), + page_age: Some("2026-08-31T00:00:00Z".into()), + thumbnail_url: Some("https://example.com/web.png".into()), + }]), + news: Some(vec![YouResult { + title: Some("News".into()), + url: Some("https://example.com/news".into()), + description: Some("news desc".into()), + snippets: Some(vec!["fresh".into()]), + page_age: Some("2026-08-31T01:00:00Z".into()), + thumbnail_url: None, + }]), + }), + }; + + let results = collect_results(resp, false); + assert_eq!(results.len(), 2); + assert_eq!(results[0].source, "youcom"); + assert_eq!(results[0].snippet, "web desc\none\ntwo"); + assert_eq!(results[0].image_url.as_deref(), Some("https://example.com/web.png")); + assert_eq!(results[1].source, "youcom_news"); + } + + #[test] + fn news_mode_falls_back_to_web_when_news_missing() { + let resp = YouResponse { + results: Some(YouResults { + web: Some(vec![YouResult { + title: Some("Fallback".into()), + url: Some("https://example.com/fallback".into()), + description: None, + snippets: Some(vec!["fallback".into()]), + page_age: None, + thumbnail_url: None, + }]), + news: None, + }), + }; + + let results = collect_results(resp, true); + assert_eq!(results.len(), 1); + assert_eq!(results[0].source, "youcom"); + assert_eq!(results[0].snippet, "fallback"); + } +} diff --git a/src/registry.rs b/src/registry.rs index 01fd765..5c1a920 100644 --- a/src/registry.rs +++ b/src/registry.rs @@ -69,6 +69,7 @@ pub const MODES: &[ModeSpec] = &[ "exa", "jina", "linkup", + "youcom", "tavily", "perplexity", ], @@ -79,7 +80,15 @@ pub const MODES: &[ModeSpec] = &[ mode: Mode::News, input: InputKind::Query, merge: MergeKind::Fused, - providers: &["parallel", "brave", "serper", "linkup", "tavily", "perplexity"], + providers: &[ + "parallel", + "brave", + "serper", + "linkup", + "youcom", + "tavily", + "perplexity", + ], description: "News-specific endpoints of the news-capable providers", when_to_use: "Current events and anything where recency dominates; combine with -f day or -f week.", }, @@ -109,6 +118,7 @@ pub const MODES: &[ModeSpec] = &[ "serper", "exa", "linkup", + "youcom", "tavily", "perplexity", "xai", @@ -205,7 +215,7 @@ pub fn filter_support(provider: &str) -> FilterSupport { let base = provider.split("_llm_").next().unwrap_or(provider); match base { "brave" | "serper" | "serpapi" | "tavily" | "exa" | "perplexity" | "parallel" - | "linkup" => FilterSupport { + | "linkup" | "youcom" => FilterSupport { freshness: true, domains: true, note: None, @@ -254,6 +264,7 @@ mod tests { "exa", "jina", "linkup", + "youcom", "firecrawl", "tavily", "serpapi", diff --git a/tests/integration.rs b/tests/integration.rs index 8d96fa7..67442bd 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -34,10 +34,11 @@ fn test_help_output() { .arg("--help") .assert() .success() - .stdout(predicate::str::contains("Aggregates 13 search providers")) + .stdout(predicate::str::contains("Aggregates 14 search providers")) .stdout(predicate::str::contains("brave")) .stdout(predicate::str::contains("serper")) - .stdout(predicate::str::contains("exa")); + .stdout(predicate::str::contains("exa")) + .stdout(predicate::str::contains("youcom")); } #[test] @@ -92,6 +93,7 @@ fn test_providers_json() { assert!(names.contains(&"serper")); assert!(names.contains(&"exa")); assert!(names.contains(&"jina")); + assert!(names.contains(&"youcom")); assert!(names.contains(&"firecrawl")); assert!(names.contains(&"tavily")); } @@ -106,6 +108,7 @@ fn test_config_check() { .stdout(predicate::str::contains("serper")) .stdout(predicate::str::contains("exa")) .stdout(predicate::str::contains("jina")) + .stdout(predicate::str::contains("youcom")) .stdout(predicate::str::contains("firecrawl")) .stdout(predicate::str::contains("tavily")); }