Skip to content

Commit d8afbec

Browse files
intitniintitni
committed
Add images support to chat model
1 parent 888a436 commit d8afbec

4 files changed

Lines changed: 54 additions & 17 deletions

File tree

‎Core/Sources/HostApp/AccountSettings/ChatModelManagement/ChatModelEdit.swift‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ struct ChatModelEdit {
3535
var openAICompatibleSupportsMultipartMessageContent = true
3636
var requiresBeginWithUserMessage = false
3737
var customBody: String = ""
38+
var supportsImages: Bool = true
3839
}
3940

4041
enum Action: Equatable, BindableAction {
@@ -290,7 +291,9 @@ extension ChatModel {
290291
return state.supportsFunctionCalling
291292
}
292293
}(),
293-
modelName: state.modelName.trimmingCharacters(in: .whitespacesAndNewlines),
294+
supportsImage: state.supportsImages,
295+
modelName: state.modelName
296+
.trimmingCharacters(in: .whitespacesAndNewlines),
294297
openAIInfo: .init(
295298
organizationID: state.openAIOrganizationID,
296299
projectID: state.openAIProjectID
@@ -331,7 +334,8 @@ extension ChatModel {
331334
openAICompatibleSupportsMultipartMessageContent: info.openAICompatibleInfo
332335
.supportsMultipartMessageContent,
333336
requiresBeginWithUserMessage: info.openAICompatibleInfo.requiresBeginWithUserMessage,
334-
customBody: info.customBodyInfo.jsonBody
337+
customBody: info.customBodyInfo.jsonBody,
338+
supportsImages: info.supportsImage
335339
)
336340
}
337341
}

‎Core/Sources/HostApp/AccountSettings/ChatModelManagement/ChatModelEditView.swift‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -358,6 +358,10 @@ struct ChatModelEditView: View {
358358
TextField(text: $store.openAIProjectID, prompt: Text("Optional")) {
359359
Text("Project ID")
360360
}
361+
362+
Toggle(isOn: $store.supportsImages) {
363+
Text("Supports Images")
364+
}
361365

362366
VStack(alignment: .leading, spacing: 8) {
363367
Text(Image(systemName: "exclamationmark.triangle.fill")) + Text(
@@ -386,6 +390,10 @@ struct ChatModelEditView: View {
386390

387391
MaxTokensTextField(store: store)
388392
SupportsFunctionCallingToggle(store: store)
393+
394+
Toggle(isOn: $store.supportsImages) {
395+
Text("Supports Images")
396+
}
389397
}
390398
}
391399
}
@@ -435,6 +443,10 @@ struct ChatModelEditView: View {
435443
Toggle(isOn: $store.requiresBeginWithUserMessage) {
436444
Text("Requires the first message to be from the user")
437445
}
446+
447+
Toggle(isOn: $store.supportsImages) {
448+
Text("Supports Images")
449+
}
438450
}
439451
}
440452
}
@@ -473,6 +485,10 @@ struct ChatModelEditView: View {
473485
MaxTokensTextField(store: store)
474486

475487
TextField("API Version", text: $store.apiVersion, prompt: Text("v1"))
488+
489+
Toggle(isOn: $store.supportsImages) {
490+
Text("Supports Images")
491+
}
476492
}
477493
}
478494
}
@@ -496,6 +512,10 @@ struct ChatModelEditView: View {
496512
Text("Keep Alive")
497513
}
498514

515+
Toggle(isOn: $store.supportsImages) {
516+
Text("Supports Images")
517+
}
518+
499519
VStack(alignment: .leading, spacing: 8) {
500520
Text(Image(systemName: "exclamationmark.triangle.fill")) + Text(
501521
" For more details, please visit [https://ollama.com](https://ollama.com)."
@@ -539,6 +559,10 @@ struct ChatModelEditView: View {
539559
}
540560

541561
MaxTokensTextField(store: store)
562+
563+
Toggle(isOn: $store.supportsImages) {
564+
Text("Supports Images")
565+
}
542566

543567
VStack(alignment: .leading, spacing: 8) {
544568
Text(Image(systemName: "exclamationmark.triangle.fill")) + Text(
@@ -572,6 +596,10 @@ struct ChatModelEditView: View {
572596
Toggle(isOn: $store.openAICompatibleSupportsMultipartMessageContent) {
573597
Text("Support multi-part message content")
574598
}
599+
600+
Toggle(isOn: $store.supportsImages) {
601+
Text("Supports Images")
602+
}
575603

576604
VStack(alignment: .leading, spacing: 8) {
577605
Text(Image(systemName: "exclamationmark.triangle.fill")) + Text(

‎Tool/Sources/OpenAIService/APIs/OpenAIChatCompletionsService.swift‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1028,6 +1028,11 @@ extension OpenAIChatCompletionsService.RequestBody {
10281028
supportsTemperature: Bool,
10291029
supportsSystemPrompt: Bool
10301030
) {
1031+
let supportsMultipartMessageContent = if supportsAudio || supportsImage {
1032+
true
1033+
} else {
1034+
supportsMultipartMessageContent
1035+
}
10311036
temperature = body.temperature
10321037
stream = body.stream
10331038
stop = body.stop

‎Tool/Sources/OpenAIService/Memory/TemplateChatGPTMemory.swift‎

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,7 @@ public struct MemoryTemplate {
125125
return formatter(list.map { $0.0 })
126126
}
127127
}
128-
128+
129129
let composedContent = contents.joined(separator: "\n\n")
130130
if composedContent.isEmpty { return nil }
131131

@@ -162,7 +162,7 @@ public struct MemoryTemplate {
162162
_ messages: inout [Message],
163163
_ followUpMessages: inout [ChatMessage]
164164
) async throws -> Void
165-
165+
166166
let truncateRule: TruncateRule?
167167

168168
public init(
@@ -187,15 +187,15 @@ public struct MemoryTemplate {
187187

188188
mutating func truncate() async throws {
189189
if Task.isCancelled { return }
190-
190+
191191
if let truncateRule = truncateRule {
192192
try await truncateRule(&messages, &followUpMessages)
193193
return
194194
}
195195

196196
try await Self.defaultTruncateRule()(&messages, &followUpMessages)
197197
}
198-
198+
199199
public struct DefaultTruncateRuleOptions {
200200
public var numberOfContentListItemToKeep: (Int) -> Int = { $0 * 2 / 3 }
201201
}
@@ -206,14 +206,14 @@ public struct MemoryTemplate {
206206
var options = DefaultTruncateRuleOptions()
207207
updateOptions(&options)
208208
return { messages, followUpMessages in
209-
209+
210210
// Remove the oldest followup messages when available.
211-
211+
212212
if followUpMessages.count > 20 {
213213
followUpMessages.removeFirst(followUpMessages.count / 2)
214214
return
215215
}
216-
216+
217217
if followUpMessages.count > 2 {
218218
if followUpMessages.count.isMultiple(of: 2) {
219219
followUpMessages.removeFirst(2)
@@ -222,9 +222,9 @@ public struct MemoryTemplate {
222222
}
223223
return
224224
}
225-
225+
226226
// Remove according to the priority.
227-
227+
228228
var truncatingMessageIndex: Int?
229229
for (index, message) in messages.enumerated() {
230230
if message.priority == .max { continue }
@@ -234,20 +234,20 @@ public struct MemoryTemplate {
234234
truncatingMessageIndex = index
235235
}
236236
}
237-
237+
238238
guard let truncatingMessageIndex else { throw CancellationError() }
239239
var truncatingMessage: Message {
240240
get { messages[truncatingMessageIndex] }
241241
set { messages[truncatingMessageIndex] = newValue }
242242
}
243-
243+
244244
if truncatingMessage.isEmpty {
245245
messages.remove(at: truncatingMessageIndex)
246246
return
247247
}
248-
248+
249249
truncatingMessage.dynamicContent.removeAll(where: { $0.isEmpty })
250-
250+
251251
var truncatingContentIndex: Int?
252252
for (index, content) in truncatingMessage.dynamicContent.enumerated() {
253253
if content.isEmpty { continue }
@@ -257,13 +257,13 @@ public struct MemoryTemplate {
257257
truncatingContentIndex = index
258258
}
259259
}
260-
260+
261261
guard let truncatingContentIndex else { throw CancellationError() }
262262
var truncatingContent: Message.DynamicContent {
263263
get { truncatingMessage.dynamicContent[truncatingContentIndex] }
264264
set { truncatingMessage.dynamicContent[truncatingContentIndex] = newValue }
265265
}
266-
266+
267267
switch truncatingContent.content {
268268
case .text:
269269
truncatingMessage.dynamicContent.remove(at: truncatingContentIndex)

0 commit comments

Comments
 (0)