Add Python base RecursiveCharacterTextSplitter

Author: intitni

Python/Sources/PythonResources/Export.swift

@@ -1,9 +1,21 @@
 import Foundation
 
+class BundleFinder {}
+
 let containingBundle: Bundle? = {
     if Bundle.main.path(forResource: "site-packages", ofType: nil) != nil {
         return Bundle.main
     }
+    
+    if Bundle.main.bundlePath.contains("Contents/Developer/Platforms") {
+        // unit tests
+        let bundle = Bundle(for: BundleFinder.self)
+        let path = bundle.bundleURL
+            .deletingLastPathComponent()
+            .appendingPathComponent("CopilotForXcodeExtensionService.app").path
+        return Bundle(path: path)
+    }
+    
     let path = Bundle.main.bundleURL
         .appendingPathComponent("Contents")
         .appendingPathComponent("Applications")

Tool/Sources/LangChain/DocumentTransformer/RecursiveCharacterTextSplitter.swift

@@ -0,0 +1,39 @@
+import Foundation
+import PythonHelper
+import PythonKit
+
+public struct RecursiveCharacterTextSplitter: TextSplitter {
+    public var separators: [String]
+    public var chunkSize: Int
+    public var chunkOverlap: Int
+    
+    public init(
+        separators: [String] = ["\n\n", "\n", " ", ""],
+        chunkSize: Int = 4000,
+        chunkOverlap: Int = 200
+    ) {
+        self.separators = separators
+        self.chunkSize = chunkSize
+        self.chunkOverlap = chunkOverlap
+    }
+
+    public func split(text: String) async throws -> [String] {
+        try await runPython {
+            let text_splitter = try Python.attemptImportOnPythonThread("langchain.text_splitter")
+            let PythonRecursiveCharacterTextSplitter = text_splitter.RecursiveCharacterTextSplitter
+            let splitter = PythonRecursiveCharacterTextSplitter(
+                separators: separators,
+                chunk_size: chunkSize,
+                chunk_overlap: chunkOverlap
+//                length_function: PythonFunction({ object in
+//                    if let string = String(object) { return string.count }
+//                    return 0
+//                })
+            )
+            let result = splitter.split_text(text)
+            guard let array = [String](result) else { return [] }
+            return array
+        }
+    }
+}
+

Tool/Tests/LangChainTests/TextSplitterTests/RecursiveCharacterTextSplitterTests.swift

@@ -0,0 +1,31 @@
+import PythonHelper
+import XCTest
+
+@testable import LangChain
+
+final class RecursiveCharacterTextSplitterTests: XCTestCase {
+    override func setUp() async throws {
+        try await super.setUp()
+        await initializePython()
+    }
+
+    func test_split_text() async throws {
+        let splitter = RecursiveCharacterTextSplitter(
+            separators: ["\n\n", "\n", " ", ""],
+            chunkSize: 100,
+            chunkOverlap: 20
+        )
+
+        let text = """
+        Madam Speaker, Madam Vice President, our First Lady and Second Gentleman. Members of Congress and the Cabinet. Justices of the Supreme Court. My fellow Americans.
+        """
+
+        let result = try await splitter.split(text: text)
+
+        XCTAssertEqual(result, [
+            "Madam Speaker, Madam Vice President, our First Lady and Second Gentleman. Members of Congress and",
+            "of Congress and the Cabinet. Justices of the Supreme Court. My fellow Americans."
+        ])
+    }
+}
+

Tool/Tests/OpenAIServiceTests/LimitMessagesTests.swift

@@ -101,7 +101,8 @@ private func runService(
     ))
     let memory = AutoManagedChatGPTMemory(
         systemPrompt: systemPrompt,
-        configuration: configuration
+        configuration: configuration,
+        functionProvider: NoChatGPTFunctionProvider()
     )
 
     for message in messages {