feat: add URL context support using lynx

Adds new URL context that allows including content of URLs in chat context
using lynx text browser. This enables users to reference external content
in their chat prompts.

Also improves documentation around optional dependencies and fixes curl
utility functions to work with optional parameters.

Signed-off-by: Tomas Slusny <slusnucky@gmail.com>

Author: deathbeam

README.md

@@ -46,10 +46,15 @@ Verify "[Copilot chat in the IDE](https://github.com/settings/copilot)" is enabl
 
 Optional:
 
-- tiktoken_core: `sudo luarocks install --lua-version 5.1 tiktoken_core`. Alternatively, download a pre-built binary from [lua-tiktoken releases](https://github.com/gptlang/lua-tiktoken/releases)
-- You can check your Lua PATH in Neovim by doing `:lua print(package.cpath)`. Save the binary as `tiktoken_core.so` in any of the given paths.
-
-> For Arch Linux user, you can install [`luajit-tiktoken-bin`](https://aur.archlinux.org/packages/luajit-tiktoken-bin) or [`lua51-tiktoken-bin`](https://aur.archlinux.org/packages/lua51-tiktoken-bin) from aur!
+- [tiktoken_core](https://github.com/gptlang/lua-tiktoken)
+  - Used for more accurate token counting
+  - Install via luarocks: `sudo luarocks install --lua-version 5.1 tiktoken_core`
+  - Alternatively, download a pre-built binary from [lua-tiktoken releases](https://github.com/gptlang/lua-tiktoken/releases). You can check your Lua PATH in Neovim by doing `:lua print(package.cpath)`. Save the binary as `tiktoken_core.so` in any of the given paths.
+  - For Arch Linux users, you can install [`luajit-tiktoken-bin`](https://aur.archlinux.org/packages/luajit-tiktoken-bin) or [`lua51-tiktoken-bin`](https://aur.archlinux.org/packages/lua51-tiktoken-bin) from aur
+- [lynx](https://lynx.invisible-island.net/)
+  - Used for fetching textual representation of URLs for `url` context
+  - For Arch Linux users, you can install [`lynx`](https://archlinux.org/packages/extra/x86_64/lynx) from the official repositories
+  - For other systems, use your package manager to install `lynx`. For windows use the installer provided from lynx site
 
 > [!WARNING]
 > If you are on neovim < 0.11.0, you also might want to add `noinsert` and `popup` to your `completeopt` to make the chat completion behave well.
@@ -277,6 +282,7 @@ Default contexts are:
 - `files` - Includes all non-hidden files in the current workspace in chat context. By default includes only filenames, includes also content with `full` input. Including all content can be slow on big workspaces so use with care. Supports input.
 - `git` - Includes current git diff in chat context (default unstaged). Supports input.
 - `register` - Includes content of specified register in chat context (default `+`, e.g system clipboard). Supports input.
+- `url` - Includes content of provided URL in chat context. Requires `lynx` (see requirements). Supports input.
 
 You can define custom contexts like this:
 
@@ -495,6 +501,9 @@ Also see [here](/lua/CopilotChat/config.lua):
     register = {
       -- see config.lua for implementation
     },
+    url = {
+      -- see config.lua for implementation
+    },
   },
 
   -- default prompts

lua/CopilotChat/config.lua

@@ -273,6 +273,20 @@ return {
         }
       end,
     },
+    url = {
+      description = 'Includes content of provided URL in chat context. Requires `lynx`. Supports input.',
+      input = function(callback)
+        vim.ui.input({
+          prompt = 'Enter URL> ',
+          default = 'https://',
+        }, callback)
+      end,
+      resolve = function(input)
+        return {
+          context.url(input),
+        }
+      end,
+    },
   },
 
   -- default prompts

lua/CopilotChat/context.lua

@@ -19,6 +19,7 @@ local async = require('plenary.async')
 local log = require('plenary.log')
 local utils = require('CopilotChat.utils')
 local file_cache = {}
+local url_cache = {}
 
 local M = {}
 
@@ -400,6 +401,27 @@ function M.buffer(bufnr)
   )
 end
 
+--- Get the content of an URL
+---@param url string
+---@return CopilotChat.context.embed?
+function M.url(url)
+  local content = url_cache[url]
+  if not content then
+    local out = utils.system({ 'lynx', '-dump', url })
+    if not out or out.code ~= 0 then
+      return nil
+    end
+    content = out.stdout
+    url_cache[url] = content
+  end
+
+  return {
+    content = content,
+    filename = url,
+    filetype = 'text',
+  }
+end
+
 --- Get current git diff
 ---@param type string?
 ---@param winnr number

lua/CopilotChat/health.lua

@@ -79,6 +79,15 @@ function M.check()
     ok('git: ' .. git_version)
   end
 
+  local lynx_version = run_command('lynx', '-version')
+  if lynx_version == false then
+    warn(
+      'lynx: missing, optional for fetching url contents. See "https://lynx.invisible-island.net/".'
+    )
+  else
+    ok('lynx: ' .. lynx_version)
+  end
+
   start('CopilotChat.nvim [dependencies]')
 
   if lualib_installed('plenary') then

lua/CopilotChat/utils.lua

@@ -267,11 +267,11 @@ end
 
 --- Send curl get request
 ---@param url string The url
----@param opts table The options
+---@param opts table? The options
 M.curl_get = async.wrap(function(url, opts, callback)
   curl.get(
     url,
-    vim.tbl_deep_extend('force', opts, {
+    vim.tbl_deep_extend('force', opts or {}, {
       callback = callback,
       on_error = function(err)
         err = M.make_string(err and err.stderr or err)
@@ -283,11 +283,11 @@ end, 3)
 
 --- Send curl post request
 ---@param url string The url
----@param opts table The options
+---@param opts table? The options
 M.curl_post = async.wrap(function(url, opts, callback)
   curl.post(
     url,
-    vim.tbl_deep_extend('force', opts, {
+    vim.tbl_deep_extend('force', opts or {}, {
       callback = callback,
       on_error = function(err)
         err = M.make_string(err and err.stderr or err)