- Revision
- 271935
- Author
- [email protected]
- Date
- 2021-01-26 22:31:06 -0800 (Tue, 26 Jan 2021)
Log Message
Mute audio capture for speech recognition based on shouldInterruptAudioOnPageVisibilityChange when page is invisible
https://bugs.webkit.org/show_bug.cgi?id=220960
Patch by Sihui Liu <[email protected]> on 2021-01-26
Reviewed by Youenn Fablet.
Source/WebCore:
Add a boolean to indicate recognition state so we don't send end event twice.
* Modules/speech/SpeechRecognizer.cpp:
(WebCore::SpeechRecognizer::startRecognition):
(WebCore::SpeechRecognizer::abortRecognition):
(WebCore::SpeechRecognizer::stopRecognition):
(WebCore::SpeechRecognizer::resetRecognition):
* Modules/speech/SpeechRecognizer.h:
Source/WebKit:
We used to abort ongoing recognition when page becomes invisible. To match media capture's behavior, now we only
abort recognition when shouldInterruptAudioOnPageVisibilityChange is true.
Updated API test: WebKit2.SpeechRecognitionPageBecomesInvisible
* UIProcess/SpeechRecognitionServer.cpp:
(WebKit::SpeechRecognitionServer::abortForPageIsBecomingInvisible): Deleted.
* UIProcess/SpeechRecognitionServer.h:
* UIProcess/WebProcessProxy.cpp:
(WebKit::WebProcessProxy::pageIsBecomingInvisible):
Tools:
* TestWebKitAPI/Tests/WebKitCocoa/SpeechRecognition.mm:
(TestWebKitAPI::TEST):
* TestWebKitAPI/Tests/WebKitCocoa/speechrecognition-basic.html:
Modified Paths
Diff
Modified: trunk/Source/WebCore/ChangeLog (271934 => 271935)
--- trunk/Source/WebCore/ChangeLog 2021-01-27 05:22:47 UTC (rev 271934)
+++ trunk/Source/WebCore/ChangeLog 2021-01-27 06:31:06 UTC (rev 271935)
@@ -1,3 +1,19 @@
+2021-01-26 Sihui Liu <[email protected]>
+
+ Mute audio capture for speech recognition based on shouldInterruptAudioOnPageVisibilityChange when page is invisible
+ https://bugs.webkit.org/show_bug.cgi?id=220960
+
+ Reviewed by Youenn Fablet.
+
+ Add a boolean to indicate recognition state so we don't send end event twice.
+
+ * Modules/speech/SpeechRecognizer.cpp:
+ (WebCore::SpeechRecognizer::startRecognition):
+ (WebCore::SpeechRecognizer::abortRecognition):
+ (WebCore::SpeechRecognizer::stopRecognition):
+ (WebCore::SpeechRecognizer::resetRecognition):
+ * Modules/speech/SpeechRecognizer.h:
+
2021-01-26 Rob Buis <[email protected]>
Refactor computePreferredLogicalWidths
Modified: trunk/Source/WebCore/Modules/speech/SpeechRecognizer.cpp (271934 => 271935)
--- trunk/Source/WebCore/Modules/speech/SpeechRecognizer.cpp 2021-01-27 05:22:47 UTC (rev 271934)
+++ trunk/Source/WebCore/Modules/speech/SpeechRecognizer.cpp 2021-01-27 06:31:06 UTC (rev 271935)
@@ -118,21 +118,26 @@
bool SpeechRecognizer::startRecognition(bool, SpeechRecognitionConnectionClientIdentifier, const String&, bool, bool, uint64_t)
{
+ m_isRecognizing = true;
return true;
}
void SpeechRecognizer::abortRecognition()
{
+ m_isRecognizing = false;
m_delegateCallback(SpeechRecognitionUpdate::create(*m_clientIdentifier, SpeechRecognitionUpdateType::End));
}
void SpeechRecognizer::stopRecognition()
{
+ m_isRecognizing = false;
m_delegateCallback(SpeechRecognitionUpdate::create(*m_clientIdentifier, SpeechRecognitionUpdateType::End));
}
void SpeechRecognizer::resetRecognition()
{
+ if (!m_isRecognizing)
+ return;
abortRecognition();
}
Modified: trunk/Source/WebCore/Modules/speech/SpeechRecognizer.h (271934 => 271935)
--- trunk/Source/WebCore/Modules/speech/SpeechRecognizer.h 2021-01-27 05:22:47 UTC (rev 271934)
+++ trunk/Source/WebCore/Modules/speech/SpeechRecognizer.h 2021-01-27 06:31:06 UTC (rev 271935)
@@ -73,6 +73,9 @@
#if HAVE(SPEECHRECOGNIZER)
RetainPtr<WebSpeechRecognizerTask> m_task;
+#else
+ // For testing.
+ bool m_isRecognizing { false };
#endif
};
Modified: trunk/Source/WebKit/ChangeLog (271934 => 271935)
--- trunk/Source/WebKit/ChangeLog 2021-01-27 05:22:47 UTC (rev 271934)
+++ trunk/Source/WebKit/ChangeLog 2021-01-27 06:31:06 UTC (rev 271935)
@@ -1,3 +1,21 @@
+2021-01-26 Sihui Liu <[email protected]>
+
+ Mute audio capture for speech recognition based on shouldInterruptAudioOnPageVisibilityChange when page is invisible
+ https://bugs.webkit.org/show_bug.cgi?id=220960
+
+ Reviewed by Youenn Fablet.
+
+ We used to abort ongoing recognition when page becomes invisible. To match media capture's behavior, now we only
+ abort recognition when shouldInterruptAudioOnPageVisibilityChange is true.
+
+ Updated API test: WebKit2.SpeechRecognitionPageBecomesInvisible
+
+ * UIProcess/SpeechRecognitionServer.cpp:
+ (WebKit::SpeechRecognitionServer::abortForPageIsBecomingInvisible): Deleted.
+ * UIProcess/SpeechRecognitionServer.h:
+ * UIProcess/WebProcessProxy.cpp:
+ (WebKit::WebProcessProxy::pageIsBecomingInvisible):
+
2021-01-26 Devin Rousso <[email protected]>
Expose the value of `<meta name="theme-color" content="...">` as SPI
Modified: trunk/Source/WebKit/UIProcess/SpeechRecognitionServer.cpp (271934 => 271935)
--- trunk/Source/WebKit/UIProcess/SpeechRecognitionServer.cpp 2021-01-27 05:22:47 UTC (rev 271934)
+++ trunk/Source/WebKit/UIProcess/SpeechRecognitionServer.cpp 2021-01-27 06:31:06 UTC (rev 271935)
@@ -167,20 +167,6 @@
}
}
-void SpeechRecognitionServer::abortForPageIsBecomingInvisible()
-{
- if (!m_recognizer)
- return;
-
- auto currentClientIdentifier = m_recognizer->currentClientIdentifier();
- if (!currentClientIdentifier)
- return;
-
- auto error = WebCore::SpeechRecognitionError { WebCore::SpeechRecognitionErrorType::Aborted, "Page is no longer visible"_s };
- sendUpdate(*currentClientIdentifier, WebCore::SpeechRecognitionUpdateType::Error, error);
- m_recognizer->reset();
-}
-
void SpeechRecognitionServer::sendUpdate(WebCore::SpeechRecognitionConnectionClientIdentifier clientIdentifier, WebCore::SpeechRecognitionUpdateType type, Optional<WebCore::SpeechRecognitionError> error, Optional<Vector<WebCore::SpeechRecognitionResultData>> result)
{
auto update = WebCore::SpeechRecognitionUpdate::create(clientIdentifier, type);
Modified: trunk/Source/WebKit/UIProcess/SpeechRecognitionServer.h (271934 => 271935)
--- trunk/Source/WebKit/UIProcess/SpeechRecognitionServer.h 2021-01-27 05:22:47 UTC (rev 271934)
+++ trunk/Source/WebKit/UIProcess/SpeechRecognitionServer.h 2021-01-27 06:31:06 UTC (rev 271935)
@@ -63,7 +63,6 @@
void abort(WebCore::SpeechRecognitionConnectionClientIdentifier);
void invalidate(WebCore::SpeechRecognitionConnectionClientIdentifier);
void mute();
- void abortForPageIsBecomingInvisible();
private:
void requestPermissionForRequest(WebCore::SpeechRecognitionRequest&);
Modified: trunk/Source/WebKit/UIProcess/WebProcessProxy.cpp (271934 => 271935)
--- trunk/Source/WebKit/UIProcess/WebProcessProxy.cpp 2021-01-27 05:22:47 UTC (rev 271934)
+++ trunk/Source/WebKit/UIProcess/WebProcessProxy.cpp 2021-01-27 06:31:06 UTC (rev 271935)
@@ -1790,8 +1790,13 @@
void WebProcessProxy::pageIsBecomingInvisible(WebCore::PageIdentifier identifier)
{
- if (auto server = m_speechRecognitionServerMap.get(identifier))
- server->abortForPageIsBecomingInvisible();
+#if ENABLE(MEDIA_STREAM)
+ if (!RealtimeMediaSourceCenter::shouldInterruptAudioOnPageVisibilityChange())
+ return;
+#endif
+
+ if (auto speechRecognitionServer = m_speechRecognitionServerMap.get(identifier))
+ speechRecognitionServer->mute();
}
#if PLATFORM(WATCHOS)
Modified: trunk/Tools/ChangeLog (271934 => 271935)
--- trunk/Tools/ChangeLog 2021-01-27 05:22:47 UTC (rev 271934)
+++ trunk/Tools/ChangeLog 2021-01-27 06:31:06 UTC (rev 271935)
@@ -1,3 +1,14 @@
+2021-01-26 Sihui Liu <[email protected]>
+
+ Mute audio capture for speech recognition based on shouldInterruptAudioOnPageVisibilityChange when page is invisible
+ https://bugs.webkit.org/show_bug.cgi?id=220960
+
+ Reviewed by Youenn Fablet.
+
+ * TestWebKitAPI/Tests/WebKitCocoa/SpeechRecognition.mm:
+ (TestWebKitAPI::TEST):
+ * TestWebKitAPI/Tests/WebKitCocoa/speechrecognition-basic.html:
+
2021-01-26 Devin Rousso <[email protected]>
Expose the value of `<meta name="theme-color" content="...">` as SPI
Modified: trunk/Tools/TestWebKitAPI/Tests/WebKitCocoa/SpeechRecognition.mm (271934 => 271935)
--- trunk/Tools/TestWebKitAPI/Tests/WebKitCocoa/SpeechRecognition.mm 2021-01-27 05:22:47 UTC (rev 271934)
+++ trunk/Tools/TestWebKitAPI/Tests/WebKitCocoa/SpeechRecognition.mm 2021-01-27 06:31:06 UTC (rev 271935)
@@ -203,7 +203,7 @@
EXPECT_WK_STREQ(@"Audio Mute", [lastScriptMessage body]);
}
-// FIXME: enable this test on iOS when https://webkit.org/b/175204 is fixed.
+// FIXME: test this on iOS when https://webkit.org/b/175204 is fixed.
#if PLATFORM(MAC)
TEST(WebKit2, SpeechRecognitionPageBecomesInvisible)
@@ -222,19 +222,19 @@
shouldGrantPermissionRequest = true;
receivedScriptMessage = false;
[webView synchronouslyLoadTestPageNamed:@"speechrecognition-basic"];
- [webView stringByEvaluatingJavaScript:@"start()"];
+ [webView evaluateJavaScript:@"setShouldHandleEndEvent(true); start();" completionHandler:nil];
TestWebKitAPI::Util::run(&receivedScriptMessage);
EXPECT_WK_STREQ(@"Start", [lastScriptMessage body]);
// Hide page.
receivedScriptMessage = false;
-#if PLATFORM(MAC)
[webView.get().window setIsVisible:NO];
-#else
- webView.get().window.hidden = YES;
-#endif
+ Util::sleep(0.1);
+ // Ongoing recognition does not stop automatically.
+ EXPECT_FALSE(receivedScriptMessage);
+ [webView stringByEvaluatingJavaScript:@"stop()"];
TestWebKitAPI::Util::run(&receivedScriptMessage);
- EXPECT_WK_STREQ(@"Error: aborted - Page is no longer visible", [lastScriptMessage body]);
+ EXPECT_WK_STREQ(@"End", [lastScriptMessage body]);
// Page is invisible.
receivedScriptMessage = false;
Modified: trunk/Tools/TestWebKitAPI/Tests/WebKitCocoa/speechrecognition-basic.html (271934 => 271935)
--- trunk/Tools/TestWebKitAPI/Tests/WebKitCocoa/speechrecognition-basic.html 2021-01-27 05:22:47 UTC (rev 271934)
+++ trunk/Tools/TestWebKitAPI/Tests/WebKitCocoa/speechrecognition-basic.html 2021-01-27 06:31:06 UTC (rev 271935)
@@ -6,7 +6,13 @@
var speechRecognition = null;
var recorder = null;
+var shouldHandleEndEvent = false;
+function setShouldHandleEndEvent(shouldHandleEvent)
+{
+ shouldHandleEndEvent = shouldHandleEvent;
+}
+
function start()
{
if (!speechRecognition)
@@ -14,6 +20,8 @@
speechRecognition.continuous = true;
speechRecognition._onstart_ = () => window.webkit.messageHandlers.testHandler.postMessage("Start");
speechRecognition._onerror_ = (event) => window.webkit.messageHandlers.testHandler.postMessage("Error: " + event.error + " - " + event.message);
+ if (shouldHandleEndEvent)
+ speechRecognition._onend_ = () => window.webkit.messageHandlers.testHandler.postMessage("End");
speechRecognition.start();
}
