Update documentation

Author: intitni

Tool/Sources/LangChain/DocumentLoader/TextLoader.swift

@@ -1,7 +1,14 @@
 import AppKit
 import Foundation
 
+/// Load a text document from local file.
 public struct TextLoader: DocumentLoader {
+    enum MetadataKeys {
+        static let filename = "filename"
+        static let `extension` = "extension"
+        static let contentModificationDate = "contentModificationDate"
+    }
+    
     let url: URL
     let encoding: String.Encoding
     let options: [NSAttributedString.DocumentReadingOptionKey: Any]
@@ -26,9 +33,9 @@ public struct TextLoader: DocumentLoader {
         let modificationDate = try? url.resourceValues(forKeys: [.contentModificationDateKey])
             .contentModificationDate
         return [Document(pageContent: attributedString.string, metadata: [
-            "filename": .string(url.lastPathComponent),
-            "extension": .string(url.pathExtension),
-            "contentModificationDate": .number(
+            MetadataKeys.filename: .string(url.lastPathComponent),
+            MetadataKeys.extension: .string(url.pathExtension),
+            MetadataKeys.contentModificationDate: .number(
                 (modificationDate ?? Date()).timeIntervalSince1970
             ),
         ])]

Tool/Sources/LangChain/DocumentLoader/WebLoader.swift

@@ -3,13 +3,14 @@ import Logger
 import SwiftSoup
 import WebKit
 
+/// Load the body of a web page.
 public struct WebLoader: DocumentLoader {
     enum MetadataKeys {
         static let title = "title"
         static let url = "url"
         static let date = "date"
     }
-    
+
     var downloadHTML: (_ url: URL) async throws -> (url: URL, html: String) = { url in
         let html = try await WebScrapper().fetch(url: url)
         return (url, html)

Tool/Sources/LangChain/DocumentTransformer/RecursiveCharacterTextSplitter.swift

@@ -1,15 +1,23 @@
 import Foundation
 
+/// Implementation of splitting text that looks at characters.
+/// Recursively tries to split by different characters to find one that works.
 public class RecursiveCharacterTextSplitter: TextSplitter {
-    /**
-     Implementation of splitting text that looks at characters.
-     Recursively tries to split by different characters to find one that works.
-     */
-    public var separators: [String]
     public var chunkSize: Int
     public var chunkOverlap: Int
     public var lengthFunction: (String) -> Int
 
+    /// A list of separators to try. They will be used in order. Supports regular expressions.
+    public var separators: [String]
+
+    /// Create a new splitter
+    /// - Parameters:
+    ///    - separators: A list of separators to try. They will be used in order. Supports regular
+    /// expressions.
+    ///    - chunkSize: The maximum size of chunks. Don't use chunk size larger than 8191, because
+    /// length safe embedding is not implemented.
+    ///    - chunkOverlap: The maximum overlap between chunks.
+    ///    - lengthFunction: A function to compute the length of text.
     public init(
         separators: [String] = ["\n\n", "\n", " ", ""],
         chunkSize: Int = 4000,
@@ -23,6 +31,13 @@ public class RecursiveCharacterTextSplitter: TextSplitter {
         self.separators = separators
     }
 
+    // Create a new splitter
+    /// - Parameters:
+    ///    - separatorSet: A set of separators to try.
+    ///    - chunkSize: The maximum size of chunks. Don't use chunk size larger than 8191, because
+    /// length safe embedding is not implemented.
+    ///    - chunkOverlap: The maximum overlap between chunks.
+    ///    - lengthFunction: A function to compute the length of text.
     public init(
         separatorSet: TextSplitterSeparatorSet,
         chunkSize: Int = 4000,
@@ -55,7 +70,7 @@ public class RecursiveCharacterTextSplitter: TextSplitter {
         }
         var separator: String
         var nextSeparators: [String]
-        
+
         if let index = firstSeparatorIndex {
             separator = separators[index]
             if index < separators.endIndex - 1 {

Tool/Sources/LangChain/DocumentTransformer/TextSplitter.swift

@@ -1,9 +1,13 @@
 import Foundation
 import JSONRPC
 
+/// Split text into multiple components.
 public protocol TextSplitter: DocumentTransformer {
+    /// The maximum size of chunks.
     var chunkSize: Int { get }
+    /// The maximum overlap between chunks.
     var chunkOverlap: Int { get }
+    /// A function to compute the length of text.
     var lengthFunction: (String) -> Int { get }
 
     /// Split text into multiple components.

Tool/Sources/LangChain/Embedding/OpenAIEmbedding.swift

@@ -74,6 +74,8 @@ extension OpenAIEmbedding {
         }
     }
 
+    /// OpenAI's embedding API doesn't support embedding inputs longer than the max token.
+    /// https://github.com/openai/openai-cookbook/blob/main/examples/Embedding_long_inputs.ipynb
     func getLenSafeEmbeddings(
         documents: [Document]
     ) async throws -> [EmbeddedDocument] {
@@ -117,6 +119,7 @@ extension OpenAIEmbedding {
                                         .map(\.embedding)
                                 )
                             }
+                            
                             if shouldAverageLongEmbeddings {
                                 return (
                                     text.document,

Tool/Sources/Preferences/Types/OpenAIEmbeddingModel.swift

@@ -6,7 +6,7 @@ public extension OpenAIEmbeddingModel {
     var maxToken: Int {
         switch self {
         case .textEmbeddingAda002:
-            return 8192
+            return 8191
         }
     }
 }