refactor: migrate to uv async for file operations

Replace plenary async operations with vim.uv async APIs where possible to
improve performance and reduce dependencies. This includes:
- File operations now use vim.uv.fs_* functions
- Simplified files context provider to remove pattern matching
- Moved curl helpers to utils module
- Cleaned up tiktoken loading process

The file operations are still partly synchronous and marked with FIXME
comments for future async migration.

Author: deathbeam

README.md

@@ -275,7 +275,7 @@ Default contexts are:
 - `buffer` - Includes specified buffer in chat context (default current). Supports input.
 - `buffers` - Includes all buffers in chat context (default listed). Supports input.
 - `file` - Includes content of provided file in chat context. Supports input.
-- `files` - Includes all non-hidden filenames in the current workspace in chat context. Supports input.
+- `files` - Includes all non-hidden filenames in the current workspace in chat context.
 - `git` - Includes current git diff in chat context (default unstaged). Supports input.
 - `register` - Includes content of specified register in chat context (default `+`, e.g system clipboard). Supports input.
 

lua/CopilotChat/config.lua

@@ -228,15 +228,9 @@ return {
       end,
     },
     files = {
-      description = 'Includes all non-hidden filenames in the current workspace in chat context. Supports input.',
-      input = function(callback)
-        vim.ui.input({
-          prompt = 'Enter a file pattern> ',
-          default = '**/*',
-        }, callback)
-      end,
-      resolve = function(input, source)
-        return context.files(input, source.winnr)
+      description = 'Includes all non-hidden filenames in the current workspace in chat context.',
+      resolve = function(_, source)
+        return context.files(source.winnr)
       end,
     },
     git = {

lua/CopilotChat/context.lua

@@ -274,19 +274,14 @@ function M.outline(content, filename, ft)
 end
 
 --- Get list of all files in workspace
----@param pattern string?
 ---@param winnr number?
 ---@return table<CopilotChat.copilot.embed>
-function M.files(pattern, winnr)
+function M.files(winnr)
   local cwd = utils.win_cwd(winnr)
-  local search = cwd .. '/' .. (pattern or '**/*')
-  local files = vim.tbl_filter(function(file)
-    return vim.fn.isdirectory(file) == 0
-  end, vim.fn.glob(search, false, true))
-
-  if #files == 0 then
-    return {}
-  end
+  local files = utils.scan_dir(cwd, {
+    add_dirs = false,
+    respect_gitignore = true,
+  })
 
   local out = {}
 
@@ -315,17 +310,13 @@ end
 ---@param filename string
 ---@return CopilotChat.copilot.embed?
 function M.file(filename)
-  if vim.fn.filereadable(filename) ~= 1 then
-    return nil
-  end
-
-  local content = vim.fn.readfile(filename)
-  if not content or #content == 0 then
+  local content = utils.read_file(filename)
+  if not content then
     return nil
   end
 
   return {
-    content = table.concat(content, '\n'),
+    content = content,
     filename = vim.fn.fnamemodify(filename, ':p:.'),
     filetype = vim.filetype.match({ filename = filename }),
   }

lua/CopilotChat/copilot.lua

@@ -18,9 +18,7 @@
 ---@field model string?
 ---@field chunk_size number?
 
-local async = require('plenary.async')
 local log = require('plenary.log')
-local curl = require('plenary.curl')
 local prompts = require('CopilotChat.prompts')
 local tiktoken = require('CopilotChat.tiktoken')
 local utils = require('CopilotChat.utils')
@@ -49,32 +47,6 @@ local VERSION_HEADERS = {
   -- ['x-github-api-version'] = '2023-07-07',
 }
 
-local curl_get = async.wrap(function(url, opts, callback)
-  opts = vim.tbl_deep_extend('force', opts, {
-    callback = callback,
-    on_error = function(err)
-      err = err and err.stderr or vim.inspect(err)
-      callback(nil, err)
-    end,
-  })
-  curl.get(url, opts)
-end, 3)
-
-local curl_post = async.wrap(function(url, opts, callback)
-  opts = vim.tbl_deep_extend('force', opts, {
-    callback = callback,
-    on_error = function(err)
-      err = err and err.stderr or vim.inspect(err)
-      callback(nil, err)
-    end,
-  })
-  curl.post(url, opts)
-end, 3)
-
-local tiktoken_load = async.wrap(function(tokenizer, callback)
-  tiktoken.load(tokenizer, callback)
-end, 2)
-
 --- Get the github oauth cached token
 ---@return string|nil
 local function get_cached_token()
@@ -384,7 +356,7 @@ function Copilot:authenticate()
       ['accept'] = 'application/json',
     }, VERSION_HEADERS)
 
-    local response, err = curl_get(
+    local response, err = utils.curl_get(
       'https://api.github.com/copilot_internal/v2/token',
       vim.tbl_extend('force', self.request_args, {
         headers = headers,
@@ -427,7 +399,7 @@ function Copilot:fetch_models()
     return self.models
   end
 
-  local response, err = curl_get(
+  local response, err = utils.curl_get(
     'https://api.githubcopilot.com/models',
     vim.tbl_extend('force', self.request_args, {
       headers = self:authenticate(),
@@ -468,7 +440,7 @@ function Copilot:fetch_agents()
     return self.agents
   end
 
-  local response, err = curl_get(
+  local response, err = utils.curl_get(
     'https://api.githubcopilot.com/agents',
     vim.tbl_extend('force', self.request_args, {
       headers = self:authenticate(),
@@ -504,7 +476,7 @@ function Copilot:enable_policy(model)
     return
   end
 
-  local response, err = curl_post(
+  local response, err = utils.curl_post(
     'https://api.githubcopilot.com/models/' .. model .. '/policy',
     vim.tbl_extend('force', self.request_args, {
       headers = self:authenticate(),
@@ -565,7 +537,7 @@ function Copilot:ask(prompt, opts)
   local tokenizer = capabilities.tokenizer
   log.debug('Max tokens: ' .. max_tokens)
   log.debug('Tokenizer: ' .. tokenizer)
-  tiktoken_load(tokenizer)
+  tiktoken.load(tokenizer)
 
   local generated_messages = {}
   local selection_messages = generate_selection_messages(selection)
@@ -740,7 +712,7 @@ function Copilot:ask(prompt, opts)
     args.stream = stream_func
   end
 
-  local response, err = curl_post(url, args)
+  local response, err = utils.curl_post(url, args)
 
   if self.current_job ~= job_id then
     return nil, nil, nil
@@ -902,7 +874,7 @@ function Copilot:embed(inputs, opts)
   for i = 1, #uncached_embeddings, chunk_size do
     local chunk = vim.list_slice(uncached_embeddings, i, i + chunk_size - 1)
     local body = vim.json.encode(generate_embedding_request(chunk, model))
-    local response, err = curl_post(
+    local response, err = utils.curl_post(
       'https://api.githubcopilot.com/embeddings',
       vim.tbl_extend('force', self.request_args, {
         headers = self:authenticate(),

lua/CopilotChat/tiktoken.lua

@@ -1,79 +1,70 @@
+local async = require('plenary.async')
 local curl = require('plenary.curl')
 local log = require('plenary.log')
+local utils = require('CopilotChat.utils')
 local tiktoken_core = nil
 local current_tokenizer = nil
-
-local function get_cache_path(fname)
-  vim.fn.mkdir(tostring(vim.fn.stdpath('cache')), 'p')
-  return vim.fn.stdpath('cache') .. '/' .. fname
-end
-
-local function file_exists(name)
-  local f = io.open(name, 'r')
-  if f ~= nil then
-    io.close(f)
-    return true
-  else
-    return false
-  end
-end
+local cache_dir = vim.fn.stdpath('cache')
+vim.fn.mkdir(tostring(cache_dir), 'p')
 
 --- Load tiktoken data from cache or download it
-local function load_tiktoken_data(done, tokenizer)
+---@param tokenizer string The tokenizer to load
+---@param on_done fun(path: string) The callback to call when the data is loaded
+local function load_tiktoken_data(tokenizer, on_done)
   local tiktoken_url = 'https://openaipublic.blob.core.windows.net/encodings/'
     .. tokenizer
     .. '.tiktoken'
-  local cache_path = get_cache_path(tiktoken_url:match('.+/(.+)'))
+  local cache_path = cache_dir .. '/' .. tiktoken_url:match('.+/(.+)')
 
-  if file_exists(cache_path) then
-    done(cache_path)
+  if utils.file_exists(cache_path) then
+    on_done(cache_path)
     return
   end
 
   log.info('Downloading tiktoken data from ' .. tiktoken_url)
   curl.get(tiktoken_url, {
     output = cache_path,
     callback = function()
-      done(cache_path)
+      on_done(cache_path)
     end,
   })
 end
 
 local M = {}
 
-function M.load(tokenizer, on_done)
+--- Load the tiktoken module
+---@param tokenizer string The tokenizer to load
+M.load = async.wrap(function(tokenizer, callback)
   if tokenizer == current_tokenizer then
-    on_done()
+    callback()
     return
   end
 
   local ok, core = pcall(require, 'tiktoken_core')
   if not ok then
-    on_done()
+    callback()
     return
   end
 
-  vim.schedule(function()
-    load_tiktoken_data(
-      vim.schedule_wrap(function(path)
-        local special_tokens = {}
-        special_tokens['<|endoftext|>'] = 100257
-        special_tokens['<|fim_prefix|>'] = 100258
-        special_tokens['<|fim_middle|>'] = 100259
-        special_tokens['<|fim_suffix|>'] = 100260
-        special_tokens['<|endofprompt|>'] = 100276
-        local pat_str =
-          "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+"
-        core.new(path, special_tokens, pat_str)
-        tiktoken_core = core
-        current_tokenizer = tokenizer
-        on_done()
-      end),
-      tokenizer
-    )
+  load_tiktoken_data(tokenizer, function(path)
+    local special_tokens = {}
+    special_tokens['<|endoftext|>'] = 100257
+    special_tokens['<|fim_prefix|>'] = 100258
+    special_tokens['<|fim_middle|>'] = 100259
+    special_tokens['<|fim_suffix|>'] = 100260
+    special_tokens['<|endofprompt|>'] = 100276
+    local pat_str =
+      "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+"
+    core.new(path, special_tokens, pat_str)
+    tiktoken_core = core
+    current_tokenizer = tokenizer
+    callback()
   end)
-end
+end, 2)
 
+--- Encode a prompt
+---@param prompt string The prompt to encode
+---@return table?
 function M.encode(prompt)
   if not tiktoken_core then
     return nil
@@ -88,6 +79,9 @@ function M.encode(prompt)
   return tiktoken_core.encode(prompt)
 end
 
+--- Count the tokens in a prompt
+---@param prompt string The prompt to count
+---@return number
 function M.count(prompt)
   if not tiktoken_core then
     return math.ceil(#prompt * 0.5) -- Fallback to 1/2 character count

lua/CopilotChat/utils.lua

@@ -1,3 +1,7 @@
+local async = require('plenary.async')
+local curl = require('plenary.curl')
+local scandir = require('plenary.scandir')
+
 local M = {}
 M.timers = {}
 
@@ -212,7 +216,7 @@ end
 ---@param winnr number? The buffer number
 ---@return string
 function M.win_cwd(winnr)
-  if not winnr or not vim.api.nvim_win_is_valid(winnr) then
+  if not winnr then
     return '.'
   end
 
@@ -224,4 +228,68 @@ function M.win_cwd(winnr)
   return dir
 end
 
+--- Send curl get request
+---@param url string The url
+---@param opts table The options
+M.curl_get = async.wrap(function(url, opts, callback)
+  curl.get(
+    url,
+    vim.tbl_deep_extend('force', opts, {
+      callback = callback,
+      on_error = function(err)
+        err = err and err.stderr or vim.inspect(err)
+        callback(nil, err)
+      end,
+    })
+  )
+end, 3)
+
+--- Send curl post request
+---@param url string The url
+---@param opts table The options
+M.curl_post = async.wrap(function(url, opts, callback)
+  curl.post(
+    url,
+    vim.tbl_deep_extend('force', opts, {
+      callback = callback,
+      on_error = function(err)
+        err = err and err.stderr or vim.inspect(err)
+        callback(nil, err)
+      end,
+    })
+  )
+end, 3)
+
+--- Scan a directory
+--- FIXME: Make async
+M.scan_dir = scandir.scan_dir
+
+--- Check if a file exists
+--- FIXME: Make async
+---@param path string The file path
+M.file_exists = function(path)
+  local stat = vim.uv.fs_stat(path)
+  return stat ~= nil
+end
+
+--- Read a file
+--- FIXME: Make async
+---@param path string The file path
+M.read_file = function(path)
+  local fd = vim.uv.fs_open(path, 'r', 438)
+  if not fd then
+    return nil
+  end
+
+  local stat = vim.uv.fs_fstat(fd)
+  if not stat then
+    vim.uv.fs_close(fd)
+    return nil
+  end
+
+  local data = vim.uv.fs_read(fd, stat.size, 0)
+  vim.uv.fs_close(fd)
+  return data
+end
+
 return M