Skip to content

Commit 730669f

Browse files
committed
Fix document merging
1 parent addd0d5 commit 730669f

1 file changed

Lines changed: 11 additions & 2 deletions

File tree

Tool/Sources/LangChain/DocumentTransformer/TextSplitter.swift

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -69,8 +69,17 @@ public extension TextSplitter {
6969
endUTF16Offset: end
7070
)
7171
}.sorted(by: { $0.startUTF16Offset < $1.startUTF16Offset })
72-
let mergedChunks = mergeSplits(textChunks)
73-
let pageContent = mergedChunks.map(\.text).joined()
72+
var sumChunk: TextChunk?
73+
for chunk in textChunks {
74+
if let current = sumChunk {
75+
if let merged = current.merged(with: chunk, force: true) {
76+
sumChunk = merged
77+
}
78+
} else {
79+
sumChunk = chunk
80+
}
81+
}
82+
let pageContent = sumChunk?.text ?? ""
7483
var metadata = documents.first?.metadata ?? [String: JSONValue]()
7584
metadata["startUTF16Offset"] = nil
7685
metadata["endUTF16Offset"] = nil

0 commit comments

Comments
 (0)