We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
1 parent addd0d5 commit 730669fCopy full SHA for 730669f
1 file changed
Tool/Sources/LangChain/DocumentTransformer/TextSplitter.swift
@@ -69,8 +69,17 @@ public extension TextSplitter {
69
endUTF16Offset: end
70
)
71
}.sorted(by: { $0.startUTF16Offset < $1.startUTF16Offset })
72
- let mergedChunks = mergeSplits(textChunks)
73
- let pageContent = mergedChunks.map(\.text).joined()
+ var sumChunk: TextChunk?
+ for chunk in textChunks {
74
+ if let current = sumChunk {
75
+ if let merged = current.merged(with: chunk, force: true) {
76
+ sumChunk = merged
77
+ }
78
+ } else {
79
+ sumChunk = chunk
80
81
82
+ let pageContent = sumChunk?.text ?? ""
83
var metadata = documents.first?.metadata ?? [String: JSONValue]()
84
metadata["startUTF16Offset"] = nil
85
metadata["endUTF16Offset"] = nil
0 commit comments