Skip to content

Commit 0ecd3f8

Browse files
intitniintitni
committed
Update USearchIndex to work correctly
1 parent 250ea2f commit 0ecd3f8

15 files changed

Lines changed: 649 additions & 288 deletions

File tree

‎Copilot for Xcode.xcodeproj/project.pbxproj‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -451,6 +451,7 @@
451451
mainGroup = C8189B0D2938972F00C9DCDA;
452452
packageReferences = (
453453
C8216B792980373800AD38C7 /* XCRemoteSwiftPackageReference "swift-argument-parser" */,
454+
C80C91742A588DD800B5EADA /* XCRemoteSwiftPackageReference "usearch" */,
454455
);
455456
productRefGroup = C8189B172938972F00C9DCDA /* Products */;
456457
projectDirPath = "";
@@ -976,6 +977,14 @@
976977
/* End XCConfigurationList section */
977978

978979
/* Begin XCRemoteSwiftPackageReference section */
980+
C80C91742A588DD800B5EADA /* XCRemoteSwiftPackageReference "usearch" */ = {
981+
isa = XCRemoteSwiftPackageReference;
982+
repositoryURL = "https://github.com/unum-cloud/usearch";
983+
requirement = {
984+
kind = upToNextMajorVersion;
985+
minimumVersion = 0.19.1;
986+
};
987+
};
979988
C8216B792980373800AD38C7 /* XCRemoteSwiftPackageReference "swift-argument-parser" */ = {
980989
isa = XCRemoteSwiftPackageReference;
981990
repositoryURL = "https://github.com/apple/swift-argument-parser.git";

‎Copilot for Xcode.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved‎

Lines changed: 9 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Copilot for Xcode.xcodeproj/xcshareddata/xcschemes/ExtensionService.xcscheme‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,20 @@
2020
ReferencedContainer = "container:Copilot for Xcode.xcodeproj">
2121
</BuildableReference>
2222
</BuildActionEntry>
23+
<BuildActionEntry
24+
buildForTesting = "YES"
25+
buildForRunning = "YES"
26+
buildForProfiling = "YES"
27+
buildForArchiving = "YES"
28+
buildForAnalyzing = "YES">
29+
<BuildableReference
30+
BuildableIdentifier = "primary"
31+
BlueprintIdentifier = "Service"
32+
BuildableName = "Service"
33+
BlueprintName = "Service"
34+
ReferencedContainer = "container:Core">
35+
</BuildableReference>
36+
</BuildActionEntry>
2337
</BuildActionEntries>
2438
</BuildAction>
2539
<TestAction

‎Core/.swiftpm/xcode/xcshareddata/xcschemes/Core-Package.xcscheme‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<Scheme
3+
LastUpgradeVersion = "1430"
4+
version = "1.7">
5+
<BuildAction
6+
parallelizeBuildables = "YES"
7+
buildImplicitDependencies = "YES">
8+
<BuildActionEntries>
9+
<BuildActionEntry
10+
buildForTesting = "YES"
11+
buildForRunning = "YES"
12+
buildForProfiling = "YES"
13+
buildForArchiving = "YES"
14+
buildForAnalyzing = "YES">
15+
<BuildableReference
16+
BuildableIdentifier = "primary"
17+
BlueprintIdentifier = "Core"
18+
BuildableName = "Core"
19+
BlueprintName = "Core"
20+
ReferencedContainer = "container:">
21+
</BuildableReference>
22+
</BuildActionEntry>
23+
</BuildActionEntries>
24+
</BuildAction>
25+
<TestAction
26+
buildConfiguration = "Debug"
27+
selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
28+
selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
29+
shouldUseLaunchSchemeArgsEnv = "YES"
30+
shouldAutocreateTestPlan = "YES">
31+
</TestAction>
32+
<LaunchAction
33+
buildConfiguration = "Debug"
34+
selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
35+
selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
36+
launchStyle = "0"
37+
useCustomWorkingDirectory = "NO"
38+
ignoresPersistentStateOnLaunch = "NO"
39+
debugDocumentVersioning = "YES"
40+
debugServiceExtension = "internal"
41+
allowLocationSimulation = "YES">
42+
</LaunchAction>
43+
<ProfileAction
44+
buildConfiguration = "Release"
45+
shouldUseLaunchSchemeArgsEnv = "YES"
46+
savedToolIdentifier = ""
47+
useCustomWorkingDirectory = "NO"
48+
debugDocumentVersioning = "YES">
49+
<MacroExpansion>
50+
<BuildableReference
51+
BuildableIdentifier = "primary"
52+
BlueprintIdentifier = "Core"
53+
BuildableName = "Core"
54+
BlueprintName = "Core"
55+
ReferencedContainer = "container:">
56+
</BuildableReference>
57+
</MacroExpansion>
58+
</ProfileAction>
59+
<AnalyzeAction
60+
buildConfiguration = "Debug">
61+
</AnalyzeAction>
62+
<ArchiveAction
63+
buildConfiguration = "Release"
64+
revealArchiveInOrganizer = "YES">
65+
</ArchiveAction>
66+
</Scheme>

‎Core/Package.swift‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -89,12 +89,10 @@ let package = Package(
8989
"UserDefaultsObserver",
9090
"ChatTab",
9191
.product(name: "Logger", package: "Tool"),
92-
// .product(name: "PythonHelper", package: "Tool"),
9392
.product(name: "OpenAIService", package: "Tool"),
9493
.product(name: "Preferences", package: "Tool"),
9594
.product(name: "AsyncAlgorithms", package: "swift-async-algorithms"),
9695
.product(name: "ComposableArchitecture", package: "swift-composable-architecture"),
97-
// .product(name: "PythonKit", package: "PythonKit"),
9896
]
9997
),
10098
.testTarget(
@@ -210,7 +208,6 @@ let package = Package(
210208
"Environment",
211209
.product(name: "OpenAIService", package: "Tool"),
212210
.product(name: "Terminal", package: "Tool"),
213-
// .product(name: "PythonKit", package: "PythonKit"),
214211
]
215212
),
216213
.target(
@@ -341,7 +338,6 @@ let package = Package(
341338
"ChatPlugin",
342339
.product(name: "OpenAIService", package: "Tool"),
343340
.product(name: "LangChain", package: "Tool"),
344-
// .product(name: "PythonKit", package: "PythonKit"),
345341
],
346342
path: "Sources/ChatPlugins/MathChatPlugin"
347343
),

‎Core/Sources/ChatContextCollectors/WebChatContextCollector/QueryWebsiteFunction.swift‎

Lines changed: 65 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -56,73 +56,71 @@ struct QueryWebsiteFunction: ChatGPTFunction {
5656
}
5757

5858
func call(arguments: Arguments) async throws -> Result {
59-
throw CancellationError()
60-
// do {
61-
// throw CancellationError()
62-
// let embedding = OpenAIEmbedding(
63-
// configuration: UserPreferenceEmbeddingConfiguration()
64-
// )
65-
//
66-
// let queryEmbeddings = try await embedding.embed(query: arguments.query)
67-
// let searchCount = UserDefaults.shared.value(for: \.chatGPTMaxToken) > 5000 ? 3 : 2
68-
//
69-
// let result = try await withThrowingTaskGroup(
70-
// of: [(document: Document, distance: Float)].self
71-
// ) { group in
72-
// for urlString in arguments.urls {
73-
// guard let url = URL(string: urlString) else { continue }
74-
// group.addTask {
75-
// if let database = await TemporaryUSearch.view(identifier: urlString) {
76-
// return try await database.searchWithDistance(
77-
// embeddings: queryEmbeddings,
78-
// count: searchCount
79-
// )
80-
// }
81-
// // 1. grab the website content
82-
// await reportProgress("Loading \(url)..")
83-
// print("== load \(url)")
84-
// let loader = WebLoader(urls: [url])
85-
// let documents = try await loader.load()
86-
// await reportProgress("Processing \(url)..")
87-
// print("== loaded \(url), documents: \(documents.count)")
88-
// // 2. split the content
89-
// let splitter = RecursiveCharacterTextSplitter(
90-
// chunkSize: 1000,
91-
// chunkOverlap: 100
92-
// )
93-
// let splitDocuments = try await splitter.transformDocuments(documents)
94-
// print("== split \(url), documents: \(splitDocuments.count)")
95-
// // 3. embedding and store in db
96-
// await reportProgress("Embedding \(url)..")
97-
// let embeddedDocuments = try await embedding.embed(documents: splitDocuments)
98-
// print("== embedded \(url)")
99-
// let database = TemporaryUSearch(identifier: urlString)
100-
// try await database.set(embeddedDocuments)
101-
// print("== save to database \(url)")
102-
// let result = try await database.searchWithDistance(
103-
// embeddings: queryEmbeddings,
104-
// count: searchCount
105-
// )
106-
// print("== result of \(url): \(result)")
107-
// return result
108-
// }
109-
// }
110-
//
111-
// var all = [(document: Document, distance: Float)]()
112-
// for try await result in group {
113-
// all.append(contentsOf: result)
114-
// }
115-
// await reportProgress("Finish reading websites.")
116-
// return all
117-
// .sorted { $0.distance < $1.distance }
118-
// .prefix(searchCount)
119-
// }
120-
//
121-
// return .init(relevantDocuments: result.map(\.document))
122-
// } catch {
123-
// await reportProgress("Failed reading websites.")
124-
// throw error
125-
// }
59+
do {
60+
let embedding = OpenAIEmbedding(
61+
configuration: UserPreferenceEmbeddingConfiguration()
62+
)
63+
64+
let queryEmbeddings = try await embedding.embed(query: arguments.query)
65+
let searchCount = UserDefaults.shared.value(for: \.chatGPTMaxToken) > 5000 ? 3 : 20
66+
67+
let result = try await withThrowingTaskGroup(
68+
of: [(document: Document, distance: Float)].self
69+
) { group in
70+
for urlString in arguments.urls {
71+
guard let url = URL(string: urlString) else { continue }
72+
group.addTask {
73+
if let database = await TemporaryUSearch.view(identifier: urlString) {
74+
return try await database.searchWithDistance(
75+
embeddings: queryEmbeddings,
76+
count: searchCount
77+
)
78+
}
79+
// 1. grab the website content
80+
await reportProgress("Loading \(url)..")
81+
print("== load \(url)")
82+
let loader = WebLoader(urls: [url])
83+
let documents = try await loader.load()
84+
await reportProgress("Processing \(url)..")
85+
print("== loaded \(url), documents: \(documents.count)")
86+
// 2. split the content
87+
let splitter = RecursiveCharacterTextSplitter(
88+
chunkSize: 1000,
89+
chunkOverlap: 100
90+
)
91+
let splitDocuments = try await splitter.transformDocuments(documents)
92+
print("== split \(url), documents: \(splitDocuments.count)")
93+
// 3. embedding and store in db
94+
await reportProgress("Embedding \(url)..")
95+
let embeddedDocuments = try await embedding.embed(documents: splitDocuments)
96+
print("== embedded \(url)")
97+
let database = TemporaryUSearch(identifier: urlString)
98+
try await database.set(embeddedDocuments)
99+
print("== save to database \(url)")
100+
let result = try await database.searchWithDistance(
101+
embeddings: queryEmbeddings,
102+
count: searchCount
103+
)
104+
print("== result of \(url): \(result)")
105+
return result
106+
}
107+
}
108+
109+
var all = [(document: Document, distance: Float)]()
110+
for try await result in group {
111+
all.append(contentsOf: result)
112+
}
113+
await reportProgress("Finish reading websites.")
114+
return all
115+
.sorted { $0.distance < $1.distance }
116+
.prefix(searchCount)
117+
}
118+
119+
return .init(relevantDocuments: result.map(\.document))
120+
} catch {
121+
await reportProgress("Failed reading websites.")
122+
throw error
123+
}
126124
}
127125
}
128126

‎Core/Sources/XcodeInspector/XcodeInspector.swift‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ import AXNotificationStream
55
import Combine
66
import Foundation
77

8+
#warning("MUSTDO: - store(in:) thread safe crash")
9+
810
public final class XcodeInspector: ObservableObject {
911
public static let shared = XcodeInspector()
1012

‎ExtensionService/AppDelegate.swift‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
21
import Environment
32
import FileChangeChecker
43
import LaunchAgentManager

‎Tool/.swiftpm/xcode/xcshareddata/xcschemes/Tool-Package.xcscheme‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<Scheme
3+
LastUpgradeVersion = "1430"
4+
version = "1.7">
5+
<BuildAction
6+
parallelizeBuildables = "YES"
7+
buildImplicitDependencies = "YES">
8+
<BuildActionEntries>
9+
<BuildActionEntry
10+
buildForTesting = "YES"
11+
buildForRunning = "YES"
12+
buildForProfiling = "YES"
13+
buildForArchiving = "YES"
14+
buildForAnalyzing = "YES">
15+
<BuildableReference
16+
BuildableIdentifier = "primary"
17+
BlueprintIdentifier = "Tool"
18+
BuildableName = "Tool"
19+
BlueprintName = "Tool"
20+
ReferencedContainer = "container:">
21+
</BuildableReference>
22+
</BuildActionEntry>
23+
</BuildActionEntries>
24+
</BuildAction>
25+
<TestAction
26+
buildConfiguration = "Debug"
27+
selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
28+
selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
29+
shouldUseLaunchSchemeArgsEnv = "YES"
30+
shouldAutocreateTestPlan = "YES">
31+
</TestAction>
32+
<LaunchAction
33+
buildConfiguration = "Debug"
34+
selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
35+
selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
36+
launchStyle = "0"
37+
useCustomWorkingDirectory = "NO"
38+
ignoresPersistentStateOnLaunch = "NO"
39+
debugDocumentVersioning = "YES"
40+
debugServiceExtension = "internal"
41+
allowLocationSimulation = "YES">
42+
</LaunchAction>
43+
<ProfileAction
44+
buildConfiguration = "Release"
45+
shouldUseLaunchSchemeArgsEnv = "YES"
46+
savedToolIdentifier = ""
47+
useCustomWorkingDirectory = "NO"
48+
debugDocumentVersioning = "YES">
49+
<MacroExpansion>
50+
<BuildableReference
51+
BuildableIdentifier = "primary"
52+
BlueprintIdentifier = "Tool"
53+
BuildableName = "Tool"
54+
BlueprintName = "Tool"
55+
ReferencedContainer = "container:">
56+
</BuildableReference>
57+
</MacroExpansion>
58+
</ProfileAction>
59+
<AnalyzeAction
60+
buildConfiguration = "Debug">
61+
</AnalyzeAction>
62+
<ArchiveAction
63+
buildConfiguration = "Release"
64+
revealArchiveInOrganizer = "YES">
65+
</ArchiveAction>
66+
</Scheme>

0 commit comments

Comments
 (0)