Add WebLoader

Author: intitni

Copilot for Xcode.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved

@@ -23,6 +23,7 @@ let package = Package(
         .package(url: "https://github.com/apple/swift-async-algorithms", from: "0.1.0"),
         .package(url: "https://github.com/pointfreeco/swift-parsing", from: "0.12.1"),
         .package(url: "https://github.com/ChimeHQ/JSONRPC", from: "0.6.0"),
+        .package(url: "https://github.com/scinfu/SwiftSoup.git", from: "2.6.0"),
     ],
     targets: [
         // MARK: - Helpers
@@ -44,6 +45,7 @@ let package = Package(
                 "PythonHelper",
                 .product(name: "PythonKit", package: "PythonKit"),
                 .product(name: "Parsing", package: "swift-parsing"),
+                .product(name: "SwiftSoup", package: "SwiftSoup"),
             ]
         ),
 

Tool/Package.swift

@@ -0,0 +1,59 @@
+import Foundation
+import Logger
+import SwiftSoup
+
+public struct WebLoader: DocumentLoader {
+    var downloadHTML: (_ url: URL) async throws -> (url: URL, html: String) = { url in
+        let session = URLSession.shared
+        let (data, _) = try await session.data(for: .init(url: url))
+        let html = String(data: data, encoding: .utf8) ?? ""
+        return (url, html)
+    }
+
+    public var urls: [URL]
+
+    public init(urls: [URL]) {
+        self.urls = urls
+    }
+
+    public init(url: URL) {
+        urls = [url]
+    }
+
+    public func load() async throws -> [Document] {
+        try await withThrowingTaskGroup(of: (url: URL, html: String).self) { group in
+            for url in urls {
+                group.addTask {
+                    try await downloadHTML(url)
+                }
+            }
+            var documents: [Document] = []
+            for try await result in group {
+                do {
+                    let parsed = try SwiftSoup.parse(result.html, result.url.path)
+
+                    let body = try parsed.body()?.text()
+                    let title = (try? parsed.title()) ?? "Untitled"
+                    
+                    if let body = body {
+                        let doc = Document(pageContent: body, metadata: [
+                            "title": title,
+                            "filename": result.url.lastPathComponent,
+                            "extension": result.url.pathExtension,
+                            "contentModificationDate": (try? result.url
+                                .resourceValues(forKeys: [.contentModificationDateKey])
+                                .contentModificationDate) ?? Date(),
+                        ])
+                        documents.append(doc)
+                    }
+                } catch let Exception.Error(_, message) {
+                    Logger.langchain.error(message)
+                } catch {
+                    Logger.langchain.error(error.localizedDescription)
+                }
+            }
+            return documents
+        }
+    }
+}
+