Skip to content

Commit

Permalink
Add voice-based interaction (#35)
Browse files Browse the repository at this point in the history
  • Loading branch information
vishnuravi committed Oct 31, 2023
1 parent 49aa7a3 commit 0d5fee8
Show file tree
Hide file tree
Showing 11 changed files with 155 additions and 33 deletions.
30 changes: 25 additions & 5 deletions HealthGPT.xcodeproj/project.pbxproj
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@
275DEFDD29EEC6DC0079D453 /* Welcome.swift in Sources */ = {isa = PBXBuildFile; fileRef = 275DEFDC29EEC6DC0079D453 /* Welcome.swift */; };
275DEFF229EECA030079D453 /* Binding+Negate.swift in Sources */ = {isa = PBXBuildFile; fileRef = 275DEFF129EECA030079D453 /* Binding+Negate.swift */; };
275DEFF429EECA180079D453 /* CodableArray+RawRepresentable.swift in Sources */ = {isa = PBXBuildFile; fileRef = 275DEFF329EECA170079D453 /* CodableArray+RawRepresentable.swift */; };
2784C0872AED5BF800F997D1 /* SpeziSpeechRecognizer in Frameworks */ = {isa = PBXBuildFile; productRef = 2784C0862AED5BF800F997D1 /* SpeziSpeechRecognizer */; };
2784C0892AED5BF800F997D1 /* SpeziSpeechSynthesizer in Frameworks */ = {isa = PBXBuildFile; productRef = 2784C0882AED5BF800F997D1 /* SpeziSpeechSynthesizer */; };
27859BFF2A34F15E00397C85 /* Spezi in Frameworks */ = {isa = PBXBuildFile; productRef = 27859BFE2A34F15E00397C85 /* Spezi */; };
27859C012A34F15E00397C85 /* XCTSpezi in Frameworks */ = {isa = PBXBuildFile; productRef = 27859C002A34F15E00397C85 /* XCTSpezi */; };
27859C042A34F16B00397C85 /* SpeziHealthKit in Frameworks */ = {isa = PBXBuildFile; productRef = 27859C032A34F16B00397C85 /* SpeziHealthKit */; };
Expand All @@ -35,10 +37,11 @@
27859C282A34F2DE00397C85 /* XCTRuntimeAssertions in Frameworks */ = {isa = PBXBuildFile; productRef = 27859C272A34F2DE00397C85 /* XCTRuntimeAssertions */; };
27B249672A065D360091E52C /* PromptGeneratorTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 27B249662A065D360091E52C /* PromptGeneratorTests.swift */; };
27BA6BE429EF9A910079DC17 /* OpenAI-Info.plist in Resources */ = {isa = PBXBuildFile; fileRef = 27BA6BE329EF9A910079DC17 /* OpenAI-Info.plist */; };
27E2CD172AEF564000998FCA /* HealthGPTViewUITests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 27E2CD162AEF564000998FCA /* HealthGPTViewUITests.swift */; };
2F4242952A8B0393006E2B01 /* OpenAIAPIKey.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2F4242942A8B0393006E2B01 /* OpenAIAPIKey.swift */; };
2F4242972A8B03A5006E2B01 /* OpenAIModelSelection.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2F4242962A8B03A5006E2B01 /* OpenAIModelSelection.swift */; };
2F4242992A8B0432006E2B01 /* HealthGPTStandard.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2F4242982A8B0432006E2B01 /* HealthGPTStandard.swift */; };
2F4E21D829F0518B0067EE98 /* HealthGPTUITests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2F4E21D729F0518B0067EE98 /* HealthGPTUITests.swift */; };
2F4E21D829F0518B0067EE98 /* OnboardingUITests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2F4E21D729F0518B0067EE98 /* OnboardingUITests.swift */; };
2F4E23832989D51F0013F3D9 /* HealthGPTAppTestingSetup.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2F4E23822989D51F0013F3D9 /* HealthGPTAppTestingSetup.swift */; };
2F5E32BD297E05EA003432F8 /* HealthGPTAppDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2F5E32BC297E05EA003432F8 /* HealthGPTAppDelegate.swift */; };
2FC9759F2978E39600BA99FE /* Localizable.strings in Resources */ = {isa = PBXBuildFile; fileRef = 2FC9759E2978E39600BA99FE /* Localizable.strings */; };
Expand Down Expand Up @@ -89,11 +92,12 @@
27B2495A2A06590E0091E52C /* HealthGPTTests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = HealthGPTTests.xctest; sourceTree = BUILT_PRODUCTS_DIR; };
27B249662A065D360091E52C /* PromptGeneratorTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PromptGeneratorTests.swift; sourceTree = "<group>"; };
27BA6BE329EF9A910079DC17 /* OpenAI-Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = "OpenAI-Info.plist"; sourceTree = "<group>"; };
27E2CD162AEF564000998FCA /* HealthGPTViewUITests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = HealthGPTViewUITests.swift; sourceTree = "<group>"; };
2F4242942A8B0393006E2B01 /* OpenAIAPIKey.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = OpenAIAPIKey.swift; sourceTree = "<group>"; };
2F4242962A8B03A5006E2B01 /* OpenAIModelSelection.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = OpenAIModelSelection.swift; sourceTree = "<group>"; };
2F4242982A8B0432006E2B01 /* HealthGPTStandard.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = HealthGPTStandard.swift; sourceTree = "<group>"; };
2F4E21D529F0518B0067EE98 /* HealthGPTUITests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = HealthGPTUITests.xctest; sourceTree = BUILT_PRODUCTS_DIR; };
2F4E21D729F0518B0067EE98 /* HealthGPTUITests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = HealthGPTUITests.swift; sourceTree = "<group>"; };
2F4E21D729F0518B0067EE98 /* OnboardingUITests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = OnboardingUITests.swift; sourceTree = "<group>"; };
2F4E23822989D51F0013F3D9 /* HealthGPTAppTestingSetup.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = HealthGPTAppTestingSetup.swift; sourceTree = "<group>"; };
2F5E32BC297E05EA003432F8 /* HealthGPTAppDelegate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = HealthGPTAppDelegate.swift; sourceTree = "<group>"; };
2FAEC07F297F583900C11C42 /* HealthGPT.entitlements */ = {isa = PBXFileReference; lastKnownFileType = text.plist.entitlements; path = HealthGPT.entitlements; sourceTree = "<group>"; };
Expand Down Expand Up @@ -137,9 +141,11 @@
27859C042A34F16B00397C85 /* SpeziHealthKit in Frameworks */,
27859BFF2A34F15E00397C85 /* Spezi in Frameworks */,
4A0549192A48088600F316A2 /* SpeziOpenAI in Frameworks */,
2784C0872AED5BF800F997D1 /* SpeziSpeechRecognizer in Frameworks */,
27859C102A34F1A900397C85 /* SpeziLocalStorage in Frameworks */,
2FF8DBF429F041C500239E1A /* OpenAI in Frameworks */,
27859C122A34F1A900397C85 /* SpeziSecureStorage in Frameworks */,
2784C0892AED5BF800F997D1 /* SpeziSpeechSynthesizer in Frameworks */,
);
runOnlyForDeploymentPostprocessing = 0;
};
Expand Down Expand Up @@ -190,7 +196,8 @@
2F4E21D629F0518B0067EE98 /* HealthGPTUITests */ = {
isa = PBXGroup;
children = (
2F4E21D729F0518B0067EE98 /* HealthGPTUITests.swift */,
2F4E21D729F0518B0067EE98 /* OnboardingUITests.swift */,
27E2CD162AEF564000998FCA /* HealthGPTViewUITests.swift */,
);
path = HealthGPTUITests;
sourceTree = "<group>";
Expand Down Expand Up @@ -337,6 +344,8 @@
27859C0F2A34F1A900397C85 /* SpeziLocalStorage */,
27859C112A34F1A900397C85 /* SpeziSecureStorage */,
4A0549182A48088600F316A2 /* SpeziOpenAI */,
2784C0862AED5BF800F997D1 /* SpeziSpeechRecognizer */,
2784C0882AED5BF800F997D1 /* SpeziSpeechSynthesizer */,
);
productName = TemplateApplication;
productReference = 653A254D283387FE005D4D48 /* HealthGPT.app */;
Expand Down Expand Up @@ -457,7 +466,8 @@
isa = PBXSourcesBuildPhase;
buildActionMask = 2147483647;
files = (
2F4E21D829F0518B0067EE98 /* HealthGPTUITests.swift in Sources */,
27E2CD172AEF564000998FCA /* HealthGPTViewUITests.swift in Sources */,
2F4E21D829F0518B0067EE98 /* OnboardingUITests.swift in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;
};
Expand Down Expand Up @@ -1051,12 +1061,22 @@
repositoryURL = "https://github.com/StanfordSpezi/SpeziML.git";
requirement = {
kind = upToNextMinorVersion;
minimumVersion = 0.2.2;
minimumVersion = 0.2.6;
};
};
/* End XCRemoteSwiftPackageReference section */

/* Begin XCSwiftPackageProductDependency section */
2784C0862AED5BF800F997D1 /* SpeziSpeechRecognizer */ = {
isa = XCSwiftPackageProductDependency;
package = 4A0549172A48088600F316A2 /* XCRemoteSwiftPackageReference "SpeziML" */;
productName = SpeziSpeechRecognizer;
};
2784C0882AED5BF800F997D1 /* SpeziSpeechSynthesizer */ = {
isa = XCSwiftPackageProductDependency;
package = 4A0549172A48088600F316A2 /* XCRemoteSwiftPackageReference "SpeziML" */;
productName = SpeziSpeechSynthesizer;
};
27859BFE2A34F15E00397C85 /* Spezi */ = {
isa = XCSwiftPackageProductDependency;
package = 27859BFD2A34F15E00397C85 /* XCRemoteSwiftPackageReference "Spezi" */;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@
"kind" : "remoteSourceControl",
"location" : "https://github.com/MacPaw/OpenAI.git",
"state" : {
"revision" : "a51a7fde78173e57b9166d38d1f665d17a3c4383",
"version" : "0.2.3"
"revision" : "c45f3320ffa760f043c0239f724850c0e4f8bde5",
"version" : "0.2.4"
}
},
{
Expand All @@ -32,17 +32,17 @@
"kind" : "remoteSourceControl",
"location" : "https://github.com/StanfordSpezi/SpeziML.git",
"state" : {
"revision" : "700ab5e524ec4f12f3012b514ea24822bdc6066b",
"version" : "0.2.2"
"revision" : "63cb6659876c58529407d7fa3556228345f9faa1",
"version" : "0.2.6"
}
},
{
"identity" : "spezionboarding",
"kind" : "remoteSourceControl",
"location" : "https://github.com/StanfordSpezi/SpeziOnboarding.git",
"state" : {
"revision" : "8e84137a08510f051d93cca546fdce3acb67e012",
"version" : "0.4.2"
"revision" : "0ea46a66c17615e1a933481a07434bfd41717c54",
"version" : "0.6.0"
}
},
{
Expand All @@ -59,8 +59,8 @@
"kind" : "remoteSourceControl",
"location" : "https://github.com/StanfordSpezi/SpeziViews",
"state" : {
"revision" : "3131708f262064231751a8963eb263f8648b6879",
"version" : "0.4.1"
"revision" : "4b7cc423fd823123d354ec1d541ca7d2e0a9d6e3",
"version" : "0.5.1"
}
},
{
Expand Down
3 changes: 2 additions & 1 deletion HealthGPT/HealthGPT/HealthDataInterpreter.swift
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,12 @@
import Foundation
import Spezi
import SpeziOpenAI
import SpeziSpeechSynthesizer


class HealthDataInterpreter: DefaultInitializable, Component, ObservableObject, ObservableObjectProvider {
@Dependency var openAIComponent = OpenAIComponent()


var querying = false {
willSet {
Expand Down
67 changes: 52 additions & 15 deletions HealthGPT/HealthGPT/HealthGPTView.swift
Original file line number Diff line number Diff line change
Expand Up @@ -7,51 +7,88 @@
//

import SpeziOpenAI
import SpeziSpeechSynthesizer
import SwiftUI


struct HealthGPTView: View {
@AppStorage(StorageKeys.onboardingFlowComplete) var completedOnboardingFlow = false
@AppStorage(StorageKeys.enableTextToSpeech) private var textToSpeech = StorageKeys.Defaults.enableTextToSpeech
@EnvironmentObject private var openAPIComponent: OpenAIComponent
@EnvironmentObject private var healthDataInterpreter: HealthDataInterpreter
@State private var showSettings = false

@StateObject private var speechSynthesizer = SpeechSynthesizer()


var body: some View {
NavigationView {
VStack {
ChatView($healthDataInterpreter.runningPrompt, disableInput: $healthDataInterpreter.querying)
.navigationBarTitle("WELCOME_TITLE")
.navigationTitle("WELCOME_TITLE")
.gesture(
TapGesture().onEnded {
UIApplication.shared.hideKeyboard()
}
)
.toolbar {
ToolbarItem(placement: .primaryAction) {
settingsButton
}
ToolbarItem(placement: .primaryAction) {
textToSpeechButton
}
}
}
.onAppear {
generatePrompt()
}
.onChange(of: completedOnboardingFlow) { _ in
generatePrompt()
}
.onChange(of: healthDataInterpreter.querying) { _ in
if textToSpeech,
healthDataInterpreter.runningPrompt.last?.role == .assistant,
let lastMessageContent = healthDataInterpreter.runningPrompt.last?.content {
speechSynthesizer.speak(lastMessageContent)
}
}
.sheet(isPresented: $showSettings) {
SettingsView(chat: $healthDataInterpreter.runningPrompt)
}
.navigationBarItems(
trailing:
Button(
action: {
showSettings = true
},
label: {
Image(systemName: "gearshape")
}
)
)
}
}



private var settingsButton: some View {
Button(
action: {
showSettings = true
},
label: {
Image(systemName: "gearshape")
.accessibilityLabel(Text("OPEN_SETTINGS"))
}
)
.accessibilityIdentifier("settingsButton")
}

private var textToSpeechButton: some View {
Button(
action: {
textToSpeech.toggle()
},
label: {
if textToSpeech {
Image(systemName: "speaker")
.accessibilityLabel(Text("SPEAKER_ENABLED"))
} else {
Image(systemName: "speaker.slash")
.accessibilityLabel(Text("SPEAKER_DISABLED"))
}
}
)
.accessibilityIdentifier("textToSpeechButton")
}

private func generatePrompt() {
_Concurrency.Task {
guard completedOnboardingFlow else {
Expand Down
1 change: 1 addition & 0 deletions HealthGPT/HealthGPT/SettingsView.swift
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ struct SettingsView: View {
RoundedRectangle(cornerRadius: 20)
.stroke(.red, lineWidth: 1)
)
.accessibilityIdentifier("clearThreadButton")

Text(disclaimer)
.foregroundColor(.gray)
Expand Down
1 change: 1 addition & 0 deletions HealthGPT/Onboarding/HealthKitPermissions.swift
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ struct HealthKitPermissions: View {
)
Spacer()
Image(systemName: "heart.text.square.fill")
.accessibilityHidden(true)
.font(.system(size: 150))
.foregroundColor(.accentColor)
Text("HEALTHKIT_PERMISSIONS_DESCRIPTION")
Expand Down
6 changes: 3 additions & 3 deletions HealthGPT/Onboarding/Welcome.swift
Original file line number Diff line number Diff line change
Expand Up @@ -20,17 +20,17 @@ struct Welcome: View {
subtitle: "WELCOME_SUBTITLE".moduleLocalized,
areas: [
.init(
icon: Image(systemName: "shippingbox.fill"),
icon: Image(systemName: "shippingbox.fill"), // swiftlint:disable:this accessibility_label_for_image
title: "WELCOME_AREA1_TITLE".moduleLocalized,
description: "WELCOME_AREA1_DESCRIPTION".moduleLocalized
),
.init(
icon: Image(systemName: "applewatch.side.right"),
icon: Image(systemName: "applewatch.side.right"), // swiftlint:disable:this accessibility_label_for_image
title: "WELCOME_AREA2_TITLE".moduleLocalized,
description: "WELCOME_AREA2_DESCRIPTION".moduleLocalized
),
.init(
icon: Image(systemName: "list.bullet.clipboard.fill"),
icon: Image(systemName: "list.bullet.clipboard.fill"), // swiftlint:disable:this accessibility_label_for_image
title: "WELCOME_AREA3_TITLE".moduleLocalized,
description: "WELCOME_AREA3_DESCRIPTION".moduleLocalized
)
Expand Down
6 changes: 6 additions & 0 deletions HealthGPT/SharedContext/StorageKeys.swift
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,17 @@
/// Constants shared across the HealthGPT Application to access
/// storage information including the `AppStorage` and `SceneStorage`
enum StorageKeys {
enum Defaults {
static let enableTextToSpeech = false
}

// MARK: - Onboarding
/// A `Bool` flag indicating of the onboarding was completed.
static let onboardingFlowComplete = "onboardingFlow.complete"
/// A `Step` flag indicating the current step in the onboarding process.
static let onboardingFlowStep = "onboardingFlow.step"
/// An `AIModel` flag indicating the OpenAI model to use
static let openAIModel = "openAI.model"
/// A `Bool` flag indicating if messages should be spoken.
static let enableTextToSpeech = "settings.enableTextToSpeech"
}
5 changes: 5 additions & 0 deletions HealthGPT/Supporting Files/Localizable.strings
Original file line number Diff line number Diff line change
Expand Up @@ -55,3 +55,8 @@
// MARK: Model Selection
"MODEL_SELECTION_TITLE" = "Select an OpenAI Model";
"MODEL_SELECTION_SUBTITLE" = "If you have access to GPT4, you may select it below. Otherwise select GPT 3.5 Turbo.";

// MARK: HealthGPT View
"SPEAKER_DISABLED" = "Text to speech is disabled, press to enable text to speech.";
"SPEAKER_ENABLED" = "Text to speech is enabled, press to disable text to speech.";
"OPEN_SETTINGS" = "Press to open the settings editor.";
51 changes: 51 additions & 0 deletions HealthGPTUITests/HealthGPTViewUITests.swift
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
//
// This source file is part of the Stanford HealthGPT project
//
// SPDX-FileCopyrightText: 2023 Stanford University & Project Contributors (see CONTRIBUTORS.md)
//
// SPDX-License-Identifier: MIT
//

import XCTest
import XCTestExtensions


final class HealthGPTViewUITests: XCTestCase {
override func setUpWithError() throws {
try super.setUpWithError()
continueAfterFailure = false

let app = XCUIApplication()
app.launchArguments = ["--showOnboarding", "--resetKeychain"]
app.deleteAndLaunch(withSpringboardAppName: "HealthGPT")
}

override func tearDownWithError() throws {
try super.tearDownWithError()
}

func testTextToSpeechToggle() throws {
let app = XCUIApplication()
try app.conductOnboardingIfNeeded()

let ttsButton = app.buttons["textToSpeechButton"]
XCTAssertTrue(ttsButton.waitForExistence(timeout: 5))

XCTAssertEqual(ttsButton.label, "Text to speech is disabled, press to enable text to speech.")
ttsButton.tap()
XCTAssertEqual(ttsButton.label, "Text to speech is enabled, press to disable text to speech.")
}

func testSettingsView() throws {
let app = XCUIApplication()
try app.conductOnboardingIfNeeded()

let settingsButton = app.buttons["settingsButton"]
XCTAssertTrue(settingsButton.waitForExistence(timeout: 5))
settingsButton.tap()

let clearThreadButton = app.buttons["clearThreadButton"]
XCTAssertTrue(clearThreadButton.waitForExistence(timeout: 5))
clearThreadButton.tap()
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import XCTestExtensions
import XCTHealthKit


final class HealthGPTUITests: XCTestCase {
final class OnboardingUITests: XCTestCase {
override func setUpWithError() throws {
try super.setUpWithError()

Expand Down

0 comments on commit 0d5fee8

Please sign in to comment.