feat(search)!: use ripgrep when available for file scanning (CopilotC-Nvim#953)

* feat(search)!: use ripgrep when available for file scanning

Adds ripgrep integration to improve search performance when scanning
directories. Falls back to the existing scandir implementation when
ripgrep is not available. Updates the configuration options to use
more consistent naming and adds health check for the ripgrep
dependency. Also cleans up installation instructions in README.

BREAKING CHANGE: Due to previous incorrect glob handling,
.lua patterns etc need to be specified as *.lua

Signed-off-by: Tomas Slusny <slusnucky@gmail.com>

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Signed-off-by: Tomas Slusny <slusnucky@gmail.com>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: deathbeam

README.md

@@ -46,13 +46,8 @@ CopilotChat.nvim is a Neovim plugin that brings GitHub Copilot Chat capabilities
   - Manual: Download from [lua-tiktoken releases](https://github.com/gptlang/lua-tiktoken/releases) and save as `tiktoken_core.so` in your Lua path
 
 - [git](https://git-scm.com/) - For git diff context features
-
-  - Arch Linux: Install from official repositories
-  - Other systems: Use system package manager or official installer
-
+- [ripgrep](https://github.com/BurntSushi/ripgrep) - For improved search performance
 - [lynx](https://lynx.invisible-island.net/) - For improved URL context features
-  - Arch Linux: Install from official repositories
-  - Other systems: Use system package manager or official installer
 
 ## Integration with pickers
 
@@ -331,7 +326,7 @@ Examples:
 ```markdown
 > #buffer
 > #buffer:2
-> #files:.lua
+> #files:\*.lua
 > #filenames
 > #git:staged
 > #url:https://example.com

lua/CopilotChat/config/contexts.lua

@@ -57,8 +57,6 @@ return {
     input = function(callback, source)
       local cwd = utils.win_cwd(source.winnr)
       local files = utils.scan_dir(cwd, {
-        add_dirs = false,
-        respect_gitignore = true,
         max_files = 1000,
       })
 
@@ -83,7 +81,7 @@ return {
     end,
     resolve = function(input, source)
       return context.files(source.winnr, true, {
-        search_pattern = input and utils.glob_to_regex(input),
+        glob = input,
       })
     end,
   },
@@ -97,7 +95,7 @@ return {
     end,
     resolve = function(input, source)
       return context.files(source.winnr, false, {
-        search_pattern = input and utils.glob_to_regex(input),
+        glob = input,
       })
     end,
   },

lua/CopilotChat/context.lua

@@ -72,6 +72,7 @@ local MIN_SYMBOL_SIMILARITY = 0.3
 local MIN_SEMANTIC_SIMILARITY = 0.4
 local MULTI_FILE_THRESHOLD = 5
 local MAX_FILES = 2500
+local MAX_DEPTH = 50
 
 --- Compute the cosine similarity between two vectors
 ---@param a table<number>
@@ -372,7 +373,7 @@ end
 --- Get list of all files in workspace
 ---@param winnr number?
 ---@param with_content boolean?
----@param search_options table?
+---@param search_options CopilotChat.utils.scan_dir_opts?
 ---@return table<CopilotChat.context.embed>
 function M.files(winnr, with_content, search_options)
   local cwd = utils.win_cwd(winnr)
@@ -382,9 +383,8 @@ function M.files(winnr, with_content, search_options)
   local files = utils.scan_dir(
     cwd,
     vim.tbl_extend('force', {
-      add_dirs = false,
-      respect_gitignore = true,
-      max_files = MAX_FILES,
+      max_count = MAX_FILES,
+      max_depth = MAX_DEPTH,
     }, search_options)
   )
 

lua/CopilotChat/health.lua

@@ -72,6 +72,15 @@ function M.check()
     ok('git: ' .. git_version)
   end
 
+  local rg_version = run_command('rg', '--version')
+  if rg_version == false then
+    warn(
+      'rg: missing, optional for improved search performance. See "https://github.com/BurntSushi/ripgrep".'
+    )
+  else
+    ok('rg: ' .. rg_version)
+  end
+
   local lynx_version = run_command('lynx', '-version')
   if lynx_version == false then
     warn(

lua/CopilotChat/utils.lua

@@ -394,17 +394,73 @@ M.curl_post = async.wrap(function(url, opts, callback)
   curl.post(url, args)
 end, 3)
 
+---@class CopilotChat.utils.scan_dir_opts
+---@field max_count number? The maximum number of files to scan
+---@field max_depth number? The maximum depth to scan
+---@field glob? string The glob pattern to match files
+---@field hidden? boolean Whether to include hidden files
+---@field no_ignore? boolean Whether to respect or ignore .gitignore
+
 --- Scan a directory
 ---@param path string The directory path
----@param opts table The options
+---@param opts CopilotChat.utils.scan_dir_opts The options
 ---@async
 M.scan_dir = async.wrap(function(path, opts, callback)
+  -- Use ripgrep if available
+  if vim.fn.executable('rg') == 1 then
+    local cmd = { 'rg' }
+
+    if opts.glob then
+      table.insert(cmd, '-g')
+      table.insert(cmd, opts.glob)
+    end
+
+    if opts.max_count then
+      table.insert(cmd, '-m')
+      table.insert(cmd, tostring(opts.max_count))
+    end
+
+    if opts.max_depth then
+      table.insert(cmd, '-d')
+      table.insert(cmd, tostring(opts.max_depth))
+    end
+
+    if opts.no_ignore then
+      table.insert(cmd, '--no-ignore')
+    end
+
+    if opts.hidden then
+      table.insert(cmd, '--hidden')
+    end
+
+    table.insert(cmd, '--files')
+    table.insert(cmd, path)
+
+    vim.system(cmd, { text = true }, function(result)
+      local files = {}
+      if result and result.code == 0 and result.stdout ~= '' then
+        files = vim.tbl_filter(function(file)
+          return file ~= ''
+        end, vim.split(result.stdout, '\n'))
+      end
+
+      callback(files)
+    end)
+
+    return
+  end
+
+  -- Fall back to scandir if rg is not available or fails
   scandir.scan_dir_async(
     path,
     vim.tbl_deep_extend('force', opts, {
+      depth = opts.max_depth,
+      add_dirs = false,
+      search_pattern = M.glob_to_pattern(opts.glob),
+      respect_gitignore = not opts.no_ignore,
       on_exit = function(files)
-        if opts.max_files then
-          files = vim.list_slice(files, 1, opts.max_files)
+        if opts.max_count then
+          files = vim.list_slice(files, 1, opts.max_count)
         end
         callback(files)
       end,
@@ -517,24 +573,135 @@ function M.empty(v)
 end
 
 --- Convert glob pattern to regex pattern
----@param glob string The glob pattern
----@return string?
-function M.glob_to_regex(glob)
-  if not glob or glob == '' then
-    return nil
+--- https://github.com/davidm/lua-glob-pattern/blob/master/lua/globtopattern.lua
+---@param g string The glob pattern
+---@return string
+function M.glob_to_pattern(g)
+  local p = '^' -- pattern being built
+  local i = 0 -- index in g
+  local c -- char at index i in g.
+
+  -- unescape glob char
+  local function unescape()
+    if c == '\\' then
+      i = i + 1
+      c = g:sub(i, i)
+      if c == '' then
+        p = '[^]'
+        return false
+      end
+    end
+    return true
   end
 
-  -- Escape regex special chars except * and ?
-  local pattern = glob:gsub('[%^%$%(%)%%%.%[%]%+%-]', '%%%1')
+  -- escape pattern char
+  local function escape(c)
+    return c:match('^%w$') and c or '%' .. c
+  end
 
-  -- Convert glob to regex pattern
-  pattern = pattern
-    :gsub('%*%*/%*', '.*') -- **/* -> .*
-    :gsub('%*%*', '.*') -- ** -> .*
-    :gsub('([^%.])%*', '%1[^/]*') -- * -> [^/]* (when not preceded by .)
-    :gsub('%?', '.') -- ? -> .
+  -- Convert tokens at end of charset.
+  local function charset_end()
+    while 1 do
+      if c == '' then
+        p = '[^]'
+        return false
+      elseif c == ']' then
+        p = p .. ']'
+        break
+      else
+        if not unescape() then
+          break
+        end
+        local c1 = c
+        i = i + 1
+        c = g:sub(i, i)
+        if c == '' then
+          p = '[^]'
+          return false
+        elseif c == '-' then
+          i = i + 1
+          c = g:sub(i, i)
+          if c == '' then
+            p = '[^]'
+            return false
+          elseif c == ']' then
+            p = p .. escape(c1) .. '%-]'
+            break
+          else
+            if not unescape() then
+              break
+            end
+            p = p .. escape(c1) .. '-' .. escape(c)
+          end
+        elseif c == ']' then
+          p = p .. escape(c1) .. ']'
+          break
+        else
+          p = p .. escape(c1)
+          i = i - 1 -- put back
+        end
+      end
+      i = i + 1
+      c = g:sub(i, i)
+    end
+    return true
+  end
 
-  return pattern .. '$'
+  -- Convert tokens in charset.
+  local function charset()
+    i = i + 1
+    c = g:sub(i, i)
+    if c == '' or c == ']' then
+      p = '[^]'
+      return false
+    elseif c == '^' or c == '!' then
+      i = i + 1
+      c = g:sub(i, i)
+      if c == ']' then
+        -- ignored
+      else
+        p = p .. '[^'
+        if not charset_end() then
+          return false
+        end
+      end
+    else
+      p = p .. '['
+      if not charset_end() then
+        return false
+      end
+    end
+    return true
+  end
+
+  -- Convert tokens.
+  while 1 do
+    i = i + 1
+    c = g:sub(i, i)
+    if c == '' then
+      p = p .. '$'
+      break
+    elseif c == '?' then
+      p = p .. '.'
+    elseif c == '*' then
+      p = p .. '.*'
+    elseif c == '[' then
+      if not charset() then
+        break
+      end
+    elseif c == '\\' then
+      i = i + 1
+      c = g:sub(i, i)
+      if c == '' then
+        p = p .. '\\$'
+        break
+      end
+      p = p .. escape(c)
+    else
+      p = p .. escape(c)
+    end
+  end
+  return p
 end
 
 ---@class CopilotChat.Diagnostic