Diff
Modified: trunk/LayoutTests/ChangeLog (270771 => 270772)
--- trunk/LayoutTests/ChangeLog 2020-12-14 16:54:48 UTC (rev 270771)
+++ trunk/LayoutTests/ChangeLog 2020-12-14 17:45:54 UTC (rev 270772)
@@ -1,3 +1,16 @@
+2020-12-14 Sihui Liu <[email protected]>
+
+ Implement recognizer for SpeechRecognition
+ https://bugs.webkit.org/show_bug.cgi?id=219459
+ <rdar://problem/71914465>
+
+ Reviewed by Youenn Fablet.
+
+ * fast/speechrecognition/ios/restart-recognition-after-stop-expected.txt: Added.
+ * fast/speechrecognition/ios/restart-recognition-after-stop.html: Added.
+ * fast/speechrecognition/ios/start-recognition-then-stop-expected.txt: Added.
+ * fast/speechrecognition/ios/start-recognition-then-stop.html: Added.
+
2020-12-14 Andy Estes <[email protected]>
Unreviewed test gardening.
Added: trunk/LayoutTests/fast/speechrecognition/ios/restart-recognition-after-stop-expected.txt (0 => 270772)
--- trunk/LayoutTests/fast/speechrecognition/ios/restart-recognition-after-stop-expected.txt (rev 0)
+++ trunk/LayoutTests/fast/speechrecognition/ios/restart-recognition-after-stop-expected.txt 2020-12-14 17:45:54 UTC (rev 270772)
@@ -0,0 +1,23 @@
+Verify that recognition can be restarted after being stopped.
+
+On success, you will see a series of "PASS" messages, followed by "TEST COMPLETE".
+
+
+PASS recognition = new SpeechRecognition() did not throw exception.
+PASS recognition.start() did not throw exception.
+Received start event
+Received audiostart event
+Received speechstart event
+PASS recognition.stop() did not throw exception.
+Received end event
+(Will restart recognition)
+PASS recognition.start() did not throw exception.
+Received start event
+Received audiostart event
+Received speechstart event
+PASS recognition.stop() did not throw exception.
+Received end event
+PASS successfullyParsed is true
+
+TEST COMPLETE
+
Added: trunk/LayoutTests/fast/speechrecognition/ios/restart-recognition-after-stop.html (0 => 270772)
--- trunk/LayoutTests/fast/speechrecognition/ios/restart-recognition-after-stop.html (rev 0)
+++ trunk/LayoutTests/fast/speechrecognition/ios/restart-recognition-after-stop.html 2020-12-14 17:45:54 UTC (rev 270772)
@@ -0,0 +1,46 @@
+<!DOCTYPE html>
+<html>
+<body>
+<script src=""
+<script>
+description("Verify that recognition can be restarted after being stopped.");
+
+if (window.testRunner) {
+ jsTestIsAsync = true;
+}
+
+var hasStopped = false;
+shouldNotThrow("recognition = new SpeechRecognition()");
+recognition.continuous = true;
+
+recognition._onstart_ = (event) => {
+ debug("Received start event");
+}
+
+recognition._onaudiostart_ = (event) => {
+ debug("Received audiostart event");
+}
+
+recognition._onspeechstart_ = (event) => {
+ debug("Received speechstart event");
+
+ shouldNotThrow("recognition.stop()");
+}
+
+recognition._onend_ = (event) => {
+ debug("Received end event");
+
+ if (!hasStopped) {
+ hasStopped = true;
+ debug("(Will restart recognition)");
+ shouldNotThrow("recognition.start()");
+ } else {
+ finishJSTest();
+ }
+}
+
+shouldNotThrow("recognition.start()");
+
+</script>
+</body>
+</html>
\ No newline at end of file
Added: trunk/LayoutTests/fast/speechrecognition/ios/start-recognition-then-stop-expected.txt (0 => 270772)
--- trunk/LayoutTests/fast/speechrecognition/ios/start-recognition-then-stop-expected.txt (rev 0)
+++ trunk/LayoutTests/fast/speechrecognition/ios/start-recognition-then-stop-expected.txt 2020-12-14 17:45:54 UTC (rev 270772)
@@ -0,0 +1,24 @@
+Verify that events are received corretly when start and stop recognition normally.
+
+On success, you will see a series of "PASS" messages, followed by "TEST COMPLETE".
+
+
+PASS recognition = new SpeechRecognition() did not throw exception.
+PASS recognition.start() did not throw exception.
+Received start event
+Received audiostart event
+Received speechstart event
+Received result event
+PASS event.results.length is 1
+PASS event.results.item(0).isFinal is true
+PASS event.results.item(0).length is 1
+PASS event.results.item(0).item(0).transcript is "Test"
+PASS event.results.item(0).item(0).confidence is 1
+PASS recognition.stop() did not throw exception.
+Received audioend event
+Received speechend event
+Received end event
+PASS successfullyParsed is true
+
+TEST COMPLETE
+
Added: trunk/LayoutTests/fast/speechrecognition/ios/start-recognition-then-stop.html (0 => 270772)
--- trunk/LayoutTests/fast/speechrecognition/ios/start-recognition-then-stop.html (rev 0)
+++ trunk/LayoutTests/fast/speechrecognition/ios/start-recognition-then-stop.html 2020-12-14 17:45:54 UTC (rev 270772)
@@ -0,0 +1,61 @@
+<!DOCTYPE html>
+<html>
+<body>
+<script src=""
+<script>
+description("Verify that events are received corretly when start and stop recognition normally.");
+
+if (window.testRunner) {
+ jsTestIsAsync = true;
+}
+
+var hasReceivedResult = false;
+shouldNotThrow("recognition = new SpeechRecognition()");
+recognition.continuous = true;
+
+recognition._onstart_ = (event) => {
+ debug("Received start event");
+}
+
+recognition._onaudiostart_ = (event) => {
+ debug("Received audiostart event");
+}
+
+recognition._onspeechstart_ = (event) => {
+ debug("Received speechstart event");
+}
+
+recognition._onresult_ = (event) => {
+ if (hasReceivedResult)
+ return;
+ hasReceivedResult = true;
+
+ debug("Received result event");
+ shouldBe("event.results.length", "1");
+ shouldBeTrue("event.results.item(0).isFinal");
+ shouldBe("event.results.item(0).length", "1");
+ shouldBeEqualToString("event.results.item(0).item(0).transcript", "Test");
+ shouldBe("event.results.item(0).item(0).confidence", "1");
+
+ shouldNotThrow("recognition.stop()");
+}
+
+recognition._onaudioend_ = (event) => {
+ debug("Received audioend event");
+}
+
+recognition._onspeechend_ = (event) => {
+ debug("Received speechend event");
+}
+
+recognition._onend_ = (event) => {
+ debug("Received end event");
+
+ finishJSTest();
+}
+
+shouldNotThrow("recognition.start()");
+
+</script>
+</body>
+</html>
\ No newline at end of file
Modified: trunk/Source/WebCore/ChangeLog (270771 => 270772)
--- trunk/Source/WebCore/ChangeLog 2020-12-14 16:54:48 UTC (rev 270771)
+++ trunk/Source/WebCore/ChangeLog 2020-12-14 17:45:54 UTC (rev 270772)
@@ -1,3 +1,65 @@
+2020-12-14 Sihui Liu <[email protected]>
+
+ Implement recognizer for SpeechRecognition
+ https://bugs.webkit.org/show_bug.cgi?id=219459
+ <rdar://problem/71914465>
+
+ Reviewed by Youenn Fablet.
+
+ Add WebSpeechRecognizerTask, which connects to speech recognition service using Speech framework APIs.
+
+ Tests: fast/speechrecognition/ios/restart-recognition-after-stop.html
+ fast/speechrecognition/ios/start-recognition-then-stop.html
+
+ * Modules/speech/SpeechRecognizer.cpp:
+ (WebCore::SpeechRecognizer::reset):
+ (WebCore::SpeechRecognizer::abort):
+ (WebCore::SpeechRecognizer::stop):
+ (WebCore::SpeechRecognizer::start):
+ (WebCore::SpeechRecognizer::startCapture):
+ (WebCore::SpeechRecognizer::stopCapture):
+ (WebCore::SpeechRecognizer::dataCaptured):
+ (WebCore::SpeechRecognizer::startRecognition):
+ (WebCore::SpeechRecognizer::abortRecognition):
+ (WebCore::SpeechRecognizer::stopRecognition):
+ (WebCore::SpeechRecognizer::resetRecognition):
+ (WebCore::SpeechRecognizer::setSource): Deleted.
+ (WebCore::SpeechRecognizer::stopInternal): Deleted.
+ * Modules/speech/SpeechRecognizer.h:
+ * Modules/speech/cocoa/SpeechRecognizerCocoa.mm: Added.
+ (WebCore::SpeechRecognizer::dataCaptured):
+ (WebCore::SpeechRecognizer::startRecognition):
+ (WebCore::SpeechRecognizer::stopRecognition):
+ (WebCore::SpeechRecognizer::abortRecognition):
+ (WebCore::SpeechRecognizer::resetRecognition):
+ * Modules/speech/cocoa/WebSpeechRecognizerTask.h: Added.
+ * Modules/speech/cocoa/WebSpeechRecognizerTask.mm: Added.
+ (-[WebSpeechRecognizerTaskImpl initWithIdentifier:locale:doMultipleRecognitions:reportInterimResults:maxAlternatives:delegateCallback:]):
+ (-[WebSpeechRecognizerTaskImpl callbackWithResult:isFinal:]):
+ (-[WebSpeechRecognizerTaskImpl audioSamplesAvailable:]):
+ (-[WebSpeechRecognizerTaskImpl abort]):
+ (-[WebSpeechRecognizerTaskImpl stop]):
+ (-[WebSpeechRecognizerTaskImpl sendSpeechStartIfNeeded]):
+ (-[WebSpeechRecognizerTaskImpl sendSpeechEndIfNeeded]):
+ (-[WebSpeechRecognizerTaskImpl sendEndIfNeeded]):
+ (-[WebSpeechRecognizerTaskImpl speechRecognizer:availabilityDidChange:]):
+ (-[WebSpeechRecognizerTaskImpl speechRecognitionTask:didHypothesizeTranscription:]):
+ (-[WebSpeechRecognizerTaskImpl speechRecognitionTask:didFinishRecognition:]):
+ (-[WebSpeechRecognizerTaskImpl speechRecognitionTaskWasCancelled:]):
+ (-[WebSpeechRecognizerTaskImpl speechRecognitionTask:didFinishSuccessfully:]):
+ (-[WebSpeechRecognizerTask initWithIdentifier:locale:doMultipleRecognitions:reportInterimResults:maxAlternatives:delegateCallback:]):
+ (-[WebSpeechRecognizerTask audioSamplesAvailable:]):
+ (-[WebSpeechRecognizerTask abort]):
+ (-[WebSpeechRecognizerTask stop]):
+ * Modules/speech/cocoa/WebSpeechRecognizerTaskMock.h: Added.
+ * Modules/speech/cocoa/WebSpeechRecognizerTaskMock.mm: Added.
+ (-[WebSpeechRecognizerTaskMock initWithIdentifier:locale:doMultipleRecognitions:reportInterimResults:maxAlternatives:delegateCallback:]):
+ (-[WebSpeechRecognizerTaskMock audioSamplesAvailable:]):
+ (-[WebSpeechRecognizerTaskMock abort]):
+ (-[WebSpeechRecognizerTaskMock stop]):
+ * SourcesCocoa.txt:
+ * WebCore.xcodeproj/project.pbxproj:
+
2020-12-14 Chris Dumez <[email protected]>
[GPUProcess] Crash under AudioDestinationCocoa::setIsPlaying(bool)
Modified: trunk/Source/WebCore/Modules/speech/SpeechRecognizer.cpp (270771 => 270772)
--- trunk/Source/WebCore/Modules/speech/SpeechRecognizer.cpp 2020-12-14 16:54:48 UTC (rev 270771)
+++ trunk/Source/WebCore/Modules/speech/SpeechRecognizer.cpp 2020-12-14 17:45:54 UTC (rev 270772)
@@ -27,10 +27,10 @@
#include "SpeechRecognizer.h"
#include "SpeechRecognitionUpdate.h"
+#include <wtf/MediaTime.h>
#if PLATFORM(COCOA)
#include "MediaUtilities.h"
-#include <pal/avfoundation/MediaTimeAVFoundation.h>
#endif
namespace WebCore {
@@ -45,42 +45,47 @@
if (!m_clientIdentifier)
return;
- if (m_source)
- m_source = nullptr;
+ stopCapture();
+ resetRecognition();
+ m_clientIdentifier = WTF::nullopt;
+}
- auto error = SpeechRecognitionError { SpeechRecognitionErrorType::Aborted, "Another request is started" };
- m_delegateCallback(SpeechRecognitionUpdate::createError(*m_clientIdentifier, error));
+void SpeechRecognizer::abort()
+{
+ ASSERT(m_clientIdentifier);
+ stopCapture();
+ abortRecognition();
+}
- m_clientIdentifier = WTF::nullopt;
+void SpeechRecognizer::stop()
+{
+ ASSERT(m_clientIdentifier);
+ stopCapture();
+ stopRecognition();
}
#if ENABLE(MEDIA_STREAM)
-void SpeechRecognizer::start(SpeechRecognitionConnectionClientIdentifier identifier, Ref<RealtimeMediaSource>&& source)
+void SpeechRecognizer::start(SpeechRecognitionConnectionClientIdentifier clientIdentifier, Ref<RealtimeMediaSource>&& source, bool mockSpeechRecognitionEnabled, const String& localeIdentifier, bool continuous, bool interimResults, uint64_t maxAlternatives)
{
- ASSERT(!m_source);
-
- m_clientIdentifier = identifier;
+ ASSERT(!m_clientIdentifier);
+ m_clientIdentifier = clientIdentifier;
m_delegateCallback(SpeechRecognitionUpdate::create(*m_clientIdentifier, SpeechRecognitionUpdateType::Start));
- setSource(WTFMove(source));
+ if (!startRecognition(mockSpeechRecognitionEnabled, clientIdentifier, localeIdentifier, continuous, interimResults, maxAlternatives)) {
+ auto error = WebCore::SpeechRecognitionError { WebCore::SpeechRecognitionErrorType::ServiceNotAllowed, "Failed to start recognition"_s };
+ m_delegateCallback(WebCore::SpeechRecognitionUpdate::createError(clientIdentifier, WTFMove(error)));
+ return;
+ }
+
+ startCapture(WTFMove(source));
}
-void SpeechRecognizer::setSource(Ref<RealtimeMediaSource>&& source)
+void SpeechRecognizer::startCapture(Ref<RealtimeMediaSource>&& source)
{
auto dataCallback = [weakThis = makeWeakPtr(this)](const auto& time, const auto& data, const auto& description, auto sampleCount) {
- if (!weakThis)
- return;
-
-#if PLATFORM(COCOA)
- auto buffer = createAudioSampleBuffer(data, description, PAL::toCMTime(time), sampleCount);
- UNUSED_PARAM(buffer);
-#else
- UNUSED_PARAM(time);
- UNUSED_PARAM(data);
- UNUSED_PARAM(description);
- UNUSED_PARAM(sampleCount);
-#endif
+ if (weakThis)
+ weakThis->dataCaptured(time, data, description, sampleCount);
};
auto stateUpdateCallback = [this, weakThis = makeWeakPtr(this)](const auto& update) {
@@ -89,9 +94,6 @@
ASSERT(m_clientIdentifier && m_clientIdentifier.value() == update.clientIdentifier());
m_delegateCallback(update);
-
- if (update.type() == SpeechRecognitionUpdateType::Error)
- m_source = nullptr;
};
m_source = makeUnique<SpeechRecognitionCaptureSource>(*m_clientIdentifier, WTFMove(dataCallback), WTFMove(stateUpdateCallback), WTFMove(source));
@@ -99,30 +101,40 @@
#endif
-void SpeechRecognizer::stop(ShouldGenerateFinalResult shouldGenerateFinalResult)
+void SpeechRecognizer::stopCapture()
{
- if (!m_clientIdentifier)
+ if (!m_source)
return;
- stopInternal();
+ m_source = nullptr;
+ m_delegateCallback(SpeechRecognitionUpdate::create(*m_clientIdentifier, SpeechRecognitionUpdateType::AudioEnd));
+}
- if (shouldGenerateFinalResult == ShouldGenerateFinalResult::Yes) {
- // TODO: generate real result when speech recognition backend is implemented.
- Vector<SpeechRecognitionResultData> resultDatas;
- m_delegateCallback(SpeechRecognitionUpdate::createResult(*m_clientIdentifier, resultDatas));
- }
+#if !HAVE(SPEECHRECOGNIZER)
+void SpeechRecognizer::dataCaptured(const MediaTime&, const PlatformAudioData&, const AudioStreamDescription&, size_t)
+{
+}
+
+bool SpeechRecognizer::startRecognition(bool, SpeechRecognitionConnectionClientIdentifier, const String&, bool, bool, uint64_t)
+{
+ return true;
+}
+
+void SpeechRecognizer::abortRecognition()
+{
m_delegateCallback(SpeechRecognitionUpdate::create(*m_clientIdentifier, SpeechRecognitionUpdateType::End));
- m_clientIdentifier = WTF::nullopt;
}
-void SpeechRecognizer::stopInternal()
+void SpeechRecognizer::stopRecognition()
{
- if (!m_source)
- return;
+ m_delegateCallback(SpeechRecognitionUpdate::create(*m_clientIdentifier, SpeechRecognitionUpdateType::End));
+}
- m_source = nullptr;
- m_delegateCallback(SpeechRecognitionUpdate::create(*m_clientIdentifier, SpeechRecognitionUpdateType::AudioEnd));
+void SpeechRecognizer::resetRecognition()
+{
}
+#endif
+
} // namespace WebCore
Modified: trunk/Source/WebCore/Modules/speech/SpeechRecognizer.h (270771 => 270772)
--- trunk/Source/WebCore/Modules/speech/SpeechRecognizer.h 2020-12-14 16:54:48 UTC (rev 270771)
+++ trunk/Source/WebCore/Modules/speech/SpeechRecognizer.h 2020-12-14 17:45:54 UTC (rev 270772)
@@ -28,6 +28,11 @@
#include "SpeechRecognitionCaptureSource.h"
#include "SpeechRecognitionConnectionClientIdentifier.h"
+#if HAVE(SPEECHRECOGNIZER)
+#include <wtf/RetainPtr.h>
+OBJC_CLASS WebSpeechRecognizerTask;
+#endif
+
namespace WebCore {
class SpeechRecognitionUpdate;
@@ -40,11 +45,11 @@
WEBCORE_EXPORT ~SpeechRecognizer() = default;
#if ENABLE(MEDIA_STREAM)
- WEBCORE_EXPORT void start(SpeechRecognitionConnectionClientIdentifier, Ref<RealtimeMediaSource>&&);
+ WEBCORE_EXPORT void start(SpeechRecognitionConnectionClientIdentifier, Ref<RealtimeMediaSource>&&, bool mockSpeechRecognitionEnabled, const String& localeIdentifier, bool continuous, bool interimResults, uint64_t maxAlternatives);
#endif
WEBCORE_EXPORT void reset();
- enum class ShouldGenerateFinalResult { No, Yes };
- WEBCORE_EXPORT void stop(ShouldGenerateFinalResult = ShouldGenerateFinalResult::Yes);
+ WEBCORE_EXPORT void abort();
+ WEBCORE_EXPORT void stop();
Optional<SpeechRecognitionConnectionClientIdentifier> currentClientIdentifier() const { return m_clientIdentifier; }
@@ -52,12 +57,22 @@
void stopInternal();
#if ENABLE(MEDIA_STREAM)
- void setSource(Ref<RealtimeMediaSource>&&);
+ void startCapture(Ref<RealtimeMediaSource>&&);
#endif
+ void stopCapture();
+ void dataCaptured(const WTF::MediaTime&, const PlatformAudioData&, const AudioStreamDescription&, size_t sampleCount);
+ bool startRecognition(bool mockSpeechRecognitionEnabled, SpeechRecognitionConnectionClientIdentifier, const String& localeIdentifier, bool continuous, bool interimResults, uint64_t alternatives);
+ void abortRecognition();
+ void stopRecognition();
+ void resetRecognition();
Optional<SpeechRecognitionConnectionClientIdentifier> m_clientIdentifier;
DelegateCallback m_delegateCallback;
std::unique_ptr<SpeechRecognitionCaptureSource> m_source;
+
+#if HAVE(SPEECHRECOGNIZER)
+ RetainPtr<WebSpeechRecognizerTask> m_task;
+#endif
};
} // namespace WebCore
Added: trunk/Source/WebCore/Modules/speech/cocoa/SpeechRecognizerCocoa.mm (0 => 270772)
--- trunk/Source/WebCore/Modules/speech/cocoa/SpeechRecognizerCocoa.mm (rev 0)
+++ trunk/Source/WebCore/Modules/speech/cocoa/SpeechRecognizerCocoa.mm 2020-12-14 17:45:54 UTC (rev 270772)
@@ -0,0 +1,81 @@
+/*
+ * Copyright (C) 2020 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#import "config.h"
+#import "SpeechRecognizer.h"
+
+#if HAVE(SPEECHRECOGNIZER)
+
+#import "MediaUtilities.h"
+#import "SpeechRecognitionUpdate.h"
+#import "WebSpeechRecognizerTaskMock.h"
+#import <Speech/Speech.h>
+#import <pal/avfoundation/MediaTimeAVFoundation.h>
+
+namespace WebCore {
+
+void SpeechRecognizer::dataCaptured(const MediaTime& time, const PlatformAudioData& data, const AudioStreamDescription& description, size_t sampleCount)
+{
+ auto buffer = createAudioSampleBuffer(data, description, PAL::toCMTime(time), sampleCount);
+ [m_task audioSamplesAvailable:buffer.get()];
+}
+
+bool SpeechRecognizer::startRecognition(bool mockSpeechRecognitionEnabled, SpeechRecognitionConnectionClientIdentifier identifier, const String& localeIdentifier, bool continuous, bool interimResults, uint64_t alternatives)
+{
+ auto taskClass = mockSpeechRecognitionEnabled ? [WebSpeechRecognizerTaskMock class] : [WebSpeechRecognizerTask class];
+ m_task = adoptNS([[taskClass alloc] initWithIdentifier:identifier locale:localeIdentifier doMultipleRecognitions:continuous reportInterimResults:interimResults maxAlternatives:alternatives delegateCallback:[this, weakThis = makeWeakPtr(this)](const WebCore::SpeechRecognitionUpdate& update) {
+ if (!weakThis)
+ return;
+
+ m_delegateCallback(update);
+ }]);
+
+ return !!m_task;
+}
+
+void SpeechRecognizer::stopRecognition()
+{
+ ASSERT(m_task);
+ [m_task stop];
+}
+
+void SpeechRecognizer::abortRecognition()
+{
+ ASSERT(m_task);
+ [m_task abort];
+}
+
+void SpeechRecognizer::resetRecognition()
+{
+ if (!m_task)
+ return;
+
+ auto task = std::exchange(m_task, nullptr);
+ [task abort];
+}
+
+} // namespace WebCore
+
+#endif // HAVE(SPEECHRECOGNIZER)
Copied: trunk/Source/WebCore/Modules/speech/cocoa/WebSpeechRecognizerTask.h (from rev 270771, trunk/Source/WebCore/Modules/speech/SpeechRecognizer.h) (0 => 270772)
--- trunk/Source/WebCore/Modules/speech/cocoa/WebSpeechRecognizerTask.h (rev 0)
+++ trunk/Source/WebCore/Modules/speech/cocoa/WebSpeechRecognizerTask.h 2020-12-14 17:45:54 UTC (rev 270772)
@@ -0,0 +1,51 @@
+/*
+ * Copyright (C) 2020 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#if HAVE(SPEECHRECOGNIZER)
+
+#import "SpeechRecognitionConnectionClientIdentifier.h"
+#import "SpeechRecognitionUpdate.h"
+
+NS_ASSUME_NONNULL_BEGIN
+
+typedef struct opaqueCMSampleBuffer *CMSampleBufferRef;
+
+@class WebSpeechRecognizerTaskImpl;
+
+@interface WebSpeechRecognizerTask : NSObject {
+@private
+ RetainPtr<WebSpeechRecognizerTaskImpl> _impl;
+}
+
+- (instancetype)initWithIdentifier:(WebCore::SpeechRecognitionConnectionClientIdentifier)identifier locale:(NSString*)localeIdentifier doMultipleRecognitions:(BOOL)continuous reportInterimResults:(BOOL)interimResults maxAlternatives:(unsigned long)alternatives delegateCallback:(void(^)(const WebCore::SpeechRecognitionUpdate&))callback;
+- (void)audioSamplesAvailable:(CMSampleBufferRef)sampleBuffer;
+- (void)abort;
+- (void)stop;
+
+@end
+
+NS_ASSUME_NONNULL_END
+
+#endif
Added: trunk/Source/WebCore/Modules/speech/cocoa/WebSpeechRecognizerTask.mm (0 => 270772)
--- trunk/Source/WebCore/Modules/speech/cocoa/WebSpeechRecognizerTask.mm (rev 0)
+++ trunk/Source/WebCore/Modules/speech/cocoa/WebSpeechRecognizerTask.mm 2020-12-14 17:45:54 UTC (rev 270772)
@@ -0,0 +1,295 @@
+/*
+ * Copyright (C) 2020 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#import "config.h"
+#import "WebSpeechRecognizerTask.h"
+
+#if HAVE(SPEECHRECOGNIZER)
+
+#import <pal/spi/cocoa/SpeechSPI.h>
+#import <wtf/BlockPtr.h>
+#import <wtf/WeakObjCPtr.h>
+
+#import <pal/cocoa/SpeechSoftLink.h>
+
+// Set the maximum duration to be an hour; we can adjust this if needed.
+static constexpr size_t maximumRecognitionDuration = 60 * 60;
+
+using namespace WebCore;
+
+NS_ASSUME_NONNULL_BEGIN
+
+@interface WebSpeechRecognizerTaskImpl : NSObject<SFSpeechRecognitionTaskDelegate, SFSpeechRecognizerDelegate> {
+@private
+ SpeechRecognitionConnectionClientIdentifier _identifier;
+ BlockPtr<void(const SpeechRecognitionUpdate&)> _delegateCallback;
+ bool _doMultipleRecognitions;
+ uint64_t _maxAlternatives;
+ RetainPtr<SFSpeechRecognizer> _recognizer;
+ RetainPtr<SFSpeechAudioBufferRecognitionRequest> _request;
+ WeakObjCPtr<SFSpeechRecognitionTask> _task;
+ bool _hasSentSpeechStart;
+ bool _hasSentSpeechEnd;
+ bool _hasSentEnd;
+}
+
+- (instancetype)initWithIdentifier:(SpeechRecognitionConnectionClientIdentifier)identifier locale:(NSString*)localeIdentifier doMultipleRecognitions:(BOOL)continuous reportInterimResults:(BOOL)interimResults maxAlternatives:(unsigned long)alternatives delegateCallback:(void(^)(const SpeechRecognitionUpdate&))callback;
+- (void)callbackWithResult:(SFTranscription *)transcription isFinal:(BOOL)isFinal;
+- (void)audioSamplesAvailable:(CMSampleBufferRef)sampleBuffer;
+- (void)abort;
+- (void)stop;
+- (void)sendSpeechStartIfNeeded;
+- (void)sendSpeechEndIfNeeded;
+- (void)sendEndIfNeeded;
+
+@end
+
+@implementation WebSpeechRecognizerTaskImpl
+
+- (instancetype)initWithIdentifier:(SpeechRecognitionConnectionClientIdentifier)identifier locale:(NSString*)localeIdentifier doMultipleRecognitions:(BOOL)continuous reportInterimResults:(BOOL)interimResults maxAlternatives:(unsigned long)alternatives delegateCallback:(void(^)(const SpeechRecognitionUpdate&))callback
+{
+ if (!(self = [super init]))
+ return nil;
+
+ _identifier = identifier;
+ _doMultipleRecognitions = continuous;
+ _delegateCallback = callback;
+ _hasSentSpeechStart = false;
+ _hasSentSpeechEnd = false;
+ _hasSentEnd = false;
+
+ _maxAlternatives = alternatives ? alternatives : 1;
+
+ if (![localeIdentifier length])
+ _recognizer = adoptNS([PAL::allocSFSpeechRecognizerInstance() init]);
+ else
+ _recognizer = adoptNS([PAL::allocSFSpeechRecognizerInstance() initWithLocale:[NSLocale localeWithLocaleIdentifier:localeIdentifier]]);
+ if (!_recognizer) {
+ [self release];
+ return nil;
+ }
+
+ if (![_recognizer isAvailable]) {
+ [self release];
+ return nil;
+ }
+
+ [_recognizer setDelegate:self];
+
+ _request = adoptNS([PAL::allocSFSpeechAudioBufferRecognitionRequestInstance() init]);
+ if ([_recognizer supportsOnDeviceRecognition])
+ [_request setRequiresOnDeviceRecognition:YES];
+ [_request setShouldReportPartialResults:interimResults];
+ [_request setTaskHint:SFSpeechRecognitionTaskHintDictation];
+
+#if USE(APPLE_INTERNAL_SDK)
+ [_request setDetectMultipleUtterances:YES];
+ [_request _setMaximumRecognitionDuration:maximumRecognitionDuration];
+#endif
+
+ _task = [_recognizer recognitionTaskWithRequest:_request.get() delegate:self];
+ return self;
+}
+
+- (void)callbackWithResult:(SFTranscription *)transcription isFinal:(BOOL)isFinal
+{
+ auto segments = [transcription segments];
+ Vector<SpeechRecognitionResultData> datas;
+ datas.reserveInitialCapacity(segments.count);
+ for (SFTranscriptionSegment* segment in segments) {
+ // Note segment confidence is reported 0 when result is not final.
+ Vector<SpeechRecognitionAlternativeData> alternatives;
+ alternatives.reserveInitialCapacity(_maxAlternatives);
+ alternatives.uncheckedAppend(SpeechRecognitionAlternativeData { [segment substring], [segment confidence] });
+ for (NSString* segmentAlternative : [segment alternativeSubstrings]) {
+ if (alternatives.size() == _maxAlternatives)
+ break;
+ // FIXME: calculate or get alternative confidence if possible.
+ alternatives.uncheckedAppend(SpeechRecognitionAlternativeData { segmentAlternative, 0.0 });
+ }
+ datas.uncheckedAppend(SpeechRecognitionResultData { WTFMove(alternatives), bool(isFinal) });
+ }
+ _delegateCallback(SpeechRecognitionUpdate::createResult(_identifier, WTFMove(datas)));
+}
+
+- (void)audioSamplesAvailable:(CMSampleBufferRef)sampleBuffer
+{
+ ASSERT(isMainThread());
+ [_request appendAudioSampleBuffer:sampleBuffer];
+}
+
+- (void)abort
+{
+ if (!_task || [_task state] == SFSpeechRecognitionTaskStateCanceling)
+ return;
+
+ if ([_task state] == SFSpeechRecognitionTaskStateCompleted) {
+ [self sendSpeechEndIfNeeded];
+ [self sendEndIfNeeded];
+ return;
+ }
+
+ [self sendSpeechEndIfNeeded];
+ [_request endAudio];
+ [_task cancel];
+}
+
+- (void)stop
+{
+ if (!_task || [_task state] == SFSpeechRecognitionTaskStateCanceling)
+ return;
+
+ if ([_task state] == SFSpeechRecognitionTaskStateCompleted) {
+ [self sendSpeechEndIfNeeded];
+ [self sendEndIfNeeded];
+ return;
+ }
+
+ [self sendSpeechEndIfNeeded];
+ [_request endAudio];
+ [_task finish];
+}
+
+- (void)sendSpeechStartIfNeeded
+{
+ if (_hasSentSpeechStart)
+ return;
+
+ _hasSentSpeechStart = true;
+ _delegateCallback(SpeechRecognitionUpdate::create(_identifier, SpeechRecognitionUpdateType::SpeechStart));
+}
+
+- (void)sendSpeechEndIfNeeded
+{
+ if (!_hasSentSpeechStart || _hasSentSpeechEnd)
+ return;
+
+ _hasSentSpeechEnd = true;
+ _delegateCallback(SpeechRecognitionUpdate::create(_identifier, SpeechRecognitionUpdateType::SpeechEnd));
+}
+
+- (void)sendEndIfNeeded
+{
+ if (_hasSentEnd)
+ return;
+
+ _hasSentEnd = true;
+ _delegateCallback(SpeechRecognitionUpdate::create(_identifier, SpeechRecognitionUpdateType::End));
+}
+
+#pragma mark SFSpeechRecognizerDelegate
+
+- (void)speechRecognizer:(SFSpeechRecognizer *)speechRecognizer availabilityDidChange:(BOOL)available
+{
+ ASSERT(isMainThread());
+
+ if (available || !_task)
+ return;
+
+ auto error = SpeechRecognitionError { SpeechRecognitionErrorType::ServiceNotAllowed, "Speech recognition service becomes unavailable"_s };
+ _delegateCallback(SpeechRecognitionUpdate::createError(_identifier, WTFMove(error)));
+}
+
+#pragma mark SFSpeechRecognitionTaskDelegate
+
+- (void)speechRecognitionTask:(SFSpeechRecognitionTask *)task didHypothesizeTranscription:(SFTranscription *)transcription
+{
+ ASSERT(isMainThread());
+
+ [self sendSpeechStartIfNeeded];
+ [self callbackWithResult:transcription isFinal:NO];
+}
+
+- (void)speechRecognitionTask:(SFSpeechRecognitionTask *)task didFinishRecognition:(SFSpeechRecognitionResult *)recognitionResult
+{
+ ASSERT(isMainThread());
+
+ [self callbackWithResult:recognitionResult.bestTranscription isFinal:YES];
+
+ if (!_doMultipleRecognitions) {
+ [self sendSpeechEndIfNeeded];
+ [self sendEndIfNeeded];
+ }
+}
+
+- (void)speechRecognitionTaskWasCancelled:(SFSpeechRecognitionTask *)task
+{
+ ASSERT(isMainThread());
+
+ [self sendSpeechEndIfNeeded];
+ [self sendEndIfNeeded];
+}
+
+- (void)speechRecognitionTask:(SFSpeechRecognitionTask *)task didFinishSuccessfully:(BOOL)successfully
+{
+ ASSERT(isMainThread());
+
+ if (!successfully) {
+ auto error = SpeechRecognitionError { SpeechRecognitionErrorType::Aborted, task.error.localizedDescription };
+ _delegateCallback(SpeechRecognitionUpdate::createError(_identifier, WTFMove(error)));
+ }
+
+ [self sendEndIfNeeded];
+}
+
+@end
+
+@implementation WebSpeechRecognizerTask
+
+- (instancetype)initWithIdentifier:(SpeechRecognitionConnectionClientIdentifier)identifier locale:(NSString*)localeIdentifier doMultipleRecognitions:(BOOL)continuous reportInterimResults:(BOOL)interimResults maxAlternatives:(unsigned long)alternatives delegateCallback:(void(^)(const SpeechRecognitionUpdate&))callback
+{
+ if (!(self = [super init]))
+ return nil;
+
+ _impl = adoptNS([[WebSpeechRecognizerTaskImpl alloc] initWithIdentifier:identifier locale:localeIdentifier doMultipleRecognitions:continuous reportInterimResults:interimResults maxAlternatives:alternatives delegateCallback:callback]);
+
+ if (!_impl) {
+ [self release];
+ return nil;
+ }
+
+ return self;
+}
+
+- (void)audioSamplesAvailable:(CMSampleBufferRef)sampleBuffer
+{
+ [_impl audioSamplesAvailable:sampleBuffer];
+}
+
+- (void)abort
+{
+ [_impl abort];
+}
+
+- (void)stop
+{
+ [_impl stop];
+}
+
+@end
+
+NS_ASSUME_NONNULL_END
+
+#endif
Copied: trunk/Source/WebCore/Modules/speech/cocoa/WebSpeechRecognizerTaskMock.h (from rev 270771, trunk/Source/WebCore/Modules/speech/SpeechRecognizer.h) (0 => 270772)
--- trunk/Source/WebCore/Modules/speech/cocoa/WebSpeechRecognizerTaskMock.h (rev 0)
+++ trunk/Source/WebCore/Modules/speech/cocoa/WebSpeechRecognizerTaskMock.h 2020-12-14 17:45:54 UTC (rev 270772)
@@ -0,0 +1,54 @@
+/*
+ * Copyright (C) 2020 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#if HAVE(SPEECHRECOGNIZER)
+
+#import "WebSpeechRecognizerTask.h"
+#import <wtf/BlockPtr.h>
+
+NS_ASSUME_NONNULL_BEGIN
+
+@class WebSpeechRecognizerTaskMock;
+
+@interface WebSpeechRecognizerTaskMock : WebSpeechRecognizerTask {
+@private
+ WebCore::SpeechRecognitionConnectionClientIdentifier _identifier;
+ BlockPtr<void(const WebCore::SpeechRecognitionUpdate&)> _delegateCallback;
+ bool _doMultipleRecognitions;
+ bool _hasSentSpeechStart;
+ bool _hasSentSpeechEnd;
+ bool _completed;
+}
+
+- (instancetype)initWithIdentifier:(WebCore::SpeechRecognitionConnectionClientIdentifier)identifier locale:(NSString*)localeIdentifier doMultipleRecognitions:(BOOL)continuous reportInterimResults:(BOOL)interimResults maxAlternatives:(unsigned long)alternatives delegateCallback:(void(^)(const WebCore::SpeechRecognitionUpdate&))callback;
+- (void)audioSamplesAvailable:(CMSampleBufferRef)sampleBuffer;
+- (void)abort;
+- (void)stop;
+
+@end
+
+NS_ASSUME_NONNULL_END
+
+#endif
Added: trunk/Source/WebCore/Modules/speech/cocoa/WebSpeechRecognizerTaskMock.mm (0 => 270772)
--- trunk/Source/WebCore/Modules/speech/cocoa/WebSpeechRecognizerTaskMock.mm (rev 0)
+++ trunk/Source/WebCore/Modules/speech/cocoa/WebSpeechRecognizerTaskMock.mm 2020-12-14 17:45:54 UTC (rev 270772)
@@ -0,0 +1,93 @@
+/*
+ * Copyright (C) 2020 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#import "config.h"
+#import "WebSpeechRecognizerTaskMock.h"
+
+#if HAVE(SPEECHRECOGNIZER)
+
+using namespace WebCore;
+
+NS_ASSUME_NONNULL_BEGIN
+
+@implementation WebSpeechRecognizerTaskMock
+
+- (instancetype)initWithIdentifier:(SpeechRecognitionConnectionClientIdentifier)identifier locale:(NSString*)localeIdentifier doMultipleRecognitions:(BOOL)continuous reportInterimResults:(BOOL)interimResults maxAlternatives:(unsigned long)alternatives delegateCallback:(void(^)(const SpeechRecognitionUpdate&))callback
+{
+ if (!(self = [super init]))
+ return nil;
+
+ _doMultipleRecognitions = continuous;
+ _identifier = identifier;
+ _delegateCallback = callback;
+ _completed = false;
+ _hasSentSpeechStart = false;
+ _hasSentSpeechEnd = false;
+
+ return self;
+}
+
+- (void)audioSamplesAvailable:(CMSampleBufferRef)sampleBuffer
+{
+ if (!_hasSentSpeechStart) {
+ _hasSentSpeechStart = true;
+ _delegateCallback(SpeechRecognitionUpdate::create(_identifier, SpeechRecognitionUpdateType::SpeechStart));
+ }
+
+ // Fake some recognition results.
+ Vector<SpeechRecognitionAlternativeData> alternatives;
+ alternatives.append(SpeechRecognitionAlternativeData { "Test", 1.0 });
+ Vector<SpeechRecognitionResultData> datas;
+ datas.append(SpeechRecognitionResultData { WTFMove(alternatives), true });
+ _delegateCallback(SpeechRecognitionUpdate::createResult(_identifier, WTFMove(datas)));
+
+ if (!_doMultipleRecognitions)
+ [self abort];
+}
+
+- (void)abort
+{
+ if (_completed)
+ return;
+ _completed = true;
+
+ if (!_hasSentSpeechEnd && _hasSentSpeechStart) {
+ _hasSentSpeechEnd = true;
+ _delegateCallback(SpeechRecognitionUpdate::create(_identifier, SpeechRecognitionUpdateType::SpeechEnd));
+ }
+
+ _delegateCallback(SpeechRecognitionUpdate::create(_identifier, SpeechRecognitionUpdateType::End));
+}
+
+- (void)stop
+{
+ [self abort];
+}
+
+@end
+
+NS_ASSUME_NONNULL_END
+
+#endif
Modified: trunk/Source/WebCore/PAL/ChangeLog (270771 => 270772)
--- trunk/Source/WebCore/PAL/ChangeLog 2020-12-14 16:54:48 UTC (rev 270771)
+++ trunk/Source/WebCore/PAL/ChangeLog 2020-12-14 17:45:54 UTC (rev 270772)
@@ -1,3 +1,18 @@
+2020-12-14 Sihui Liu <[email protected]>
+
+ Implement recognizer for SpeechRecognition
+ https://bugs.webkit.org/show_bug.cgi?id=219459
+ <rdar://problem/71914465>
+
+ Reviewed by Youenn Fablet.
+
+ Add soft linking to Speech framework and SPI.
+
+ * PAL.xcodeproj/project.pbxproj:
+ * pal/cocoa/SpeechSoftLink.h: Added.
+ * pal/cocoa/SpeechSoftLink.mm: Added.
+ * pal/spi/cocoa/SpeechSPI.h: Added.
+
2020-12-13 Andy Estes <[email protected]>
[Mac] Create a MediaToolbox format reader plug-in for WebM
Modified: trunk/Source/WebCore/PAL/PAL.xcodeproj/project.pbxproj (270771 => 270772)
--- trunk/Source/WebCore/PAL/PAL.xcodeproj/project.pbxproj 2020-12-14 16:54:48 UTC (rev 270771)
+++ trunk/Source/WebCore/PAL/PAL.xcodeproj/project.pbxproj 2020-12-14 17:45:54 UTC (rev 270772)
@@ -144,6 +144,9 @@
63C7EDC721AFAE04006A7B99 /* NSProgressSPI.h in Headers */ = {isa = PBXBuildFile; fileRef = 63E369F921AFA83F001C14BC /* NSProgressSPI.h */; };
7A36D0F9223AD9AB00B0522E /* CommonCryptoSPI.h in Headers */ = {isa = PBXBuildFile; fileRef = 7A36D0F8223AD9AB00B0522E /* CommonCryptoSPI.h */; };
7A3A6A8020CADB4700317AAE /* NSImageSPI.h in Headers */ = {isa = PBXBuildFile; fileRef = 7A3A6A7F20CADB4600317AAE /* NSImageSPI.h */; };
+ 93B38EBE25821CB600198E63 /* SpeechSoftLink.h in Headers */ = {isa = PBXBuildFile; fileRef = 93B38EBD25821CB600198E63 /* SpeechSoftLink.h */; };
+ 93B38EC025821CD800198E63 /* SpeechSoftLink.mm in Sources */ = {isa = PBXBuildFile; fileRef = 93B38EBF25821CD700198E63 /* SpeechSoftLink.mm */; };
+ 93B38EC225821D2200198E63 /* SpeechSPI.h in Headers */ = {isa = PBXBuildFile; fileRef = 93B38EC125821D2200198E63 /* SpeechSPI.h */; };
93DB7D3724626BCD004BD8A3 /* NSTextInputContextSPI.h in Headers */ = {isa = PBXBuildFile; fileRef = 93DB7D3624626BCC004BD8A3 /* NSTextInputContextSPI.h */; };
93DB7D3A24626F86004BD8A3 /* NSUndoManagerSPI.h in Headers */ = {isa = PBXBuildFile; fileRef = 93DB7D3924626F86004BD8A3 /* NSUndoManagerSPI.h */; };
A10265891F56747A00B4C844 /* HIToolboxSPI.h in Headers */ = {isa = PBXBuildFile; fileRef = A10265881F56747A00B4C844 /* HIToolboxSPI.h */; };
@@ -340,6 +343,9 @@
63E369F921AFA83F001C14BC /* NSProgressSPI.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = NSProgressSPI.h; sourceTree = "<group>"; };
7A36D0F8223AD9AB00B0522E /* CommonCryptoSPI.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = CommonCryptoSPI.h; sourceTree = "<group>"; };
7A3A6A7F20CADB4600317AAE /* NSImageSPI.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = NSImageSPI.h; sourceTree = "<group>"; };
+ 93B38EBD25821CB600198E63 /* SpeechSoftLink.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = SpeechSoftLink.h; sourceTree = "<group>"; };
+ 93B38EBF25821CD700198E63 /* SpeechSoftLink.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; path = SpeechSoftLink.mm; sourceTree = "<group>"; };
+ 93B38EC125821D2200198E63 /* SpeechSPI.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = SpeechSPI.h; sourceTree = "<group>"; };
93DB7D3624626BCC004BD8A3 /* NSTextInputContextSPI.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = NSTextInputContextSPI.h; sourceTree = "<group>"; };
93DB7D3924626F86004BD8A3 /* NSUndoManagerSPI.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = NSUndoManagerSPI.h; sourceTree = "<group>"; };
93E5909C1F93BF1E0067F8CF /* UnencodableHandling.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = UnencodableHandling.h; sourceTree = "<group>"; };
@@ -511,6 +517,7 @@
442956CC218A72DE0080DB54 /* RevealSPI.h */,
570AB8F020AE2E8D00B8BE87 /* SecKeyProxySPI.h */,
0C2DA13C1F3BEB4900DBC317 /* ServersSPI.h */,
+ 93B38EC125821D2200198E63 /* SpeechSPI.h */,
0C2DA12B1F3BEB4900DBC317 /* URLFormattingSPI.h */,
0C2DA13D1F3BEB4900DBC317 /* WebFilterEvaluatorSPI.h */,
);
@@ -682,6 +689,8 @@
31647FAE251759DB0010F8FB /* OpenGLSoftLinkCocoa.mm */,
A1F63C9D21A4DBF7006FB43B /* PassKitSoftLink.h */,
A1F63C9E21A4DBF7006FB43B /* PassKitSoftLink.mm */,
+ 93B38EBD25821CB600198E63 /* SpeechSoftLink.h */,
+ 93B38EBF25821CD700198E63 /* SpeechSoftLink.mm */,
07611DB4243FA5BE00D80704 /* UsageTrackingSoftLink.h */,
07611DB5243FA5BF00D80704 /* UsageTrackingSoftLink.mm */,
);
@@ -907,6 +916,8 @@
A3AB6E561F3D1DDB009C14B1 /* SleepDisabler.h in Headers */,
A3AB6E611F3D1E39009C14B1 /* SleepDisablerCocoa.h in Headers */,
A3788E981F05B6CE00679425 /* Sound.h in Headers */,
+ 93B38EBE25821CB600198E63 /* SpeechSoftLink.h in Headers */,
+ 93B38EC225821D2200198E63 /* SpeechSPI.h in Headers */,
A1175B491F6AFF8E00C4B9F0 /* SpeechSynthesisSPI.h in Headers */,
0C5AF9211F43A4C7002EAC02 /* SQLite3SPI.h in Headers */,
31308B1420A21705003FB929 /* SystemPreviewSPI.h in Headers */,
@@ -1037,6 +1048,7 @@
A3AB6E601F3D1E39009C14B1 /* SleepDisablerCocoa.cpp in Sources */,
A3788E9C1F05B78200679425 /* Sound.cpp in Sources */,
A3788E9E1F05B78E00679425 /* SoundMac.mm in Sources */,
+ 93B38EC025821CD800198E63 /* SpeechSoftLink.mm in Sources */,
A3AB6E571F3D1DDB009C14B1 /* SystemSleepListener.cpp in Sources */,
A3AB6E651F3D217F009C14B1 /* SystemSleepListenerMac.mm in Sources */,
2E1342CD215AA10A007199D2 /* UIKitSoftLink.mm in Sources */,
Copied: trunk/Source/WebCore/PAL/pal/cocoa/SpeechSoftLink.h (from rev 270771, trunk/Source/WebCore/Modules/speech/SpeechRecognizer.h) (0 => 270772)
--- trunk/Source/WebCore/PAL/pal/cocoa/SpeechSoftLink.h (rev 0)
+++ trunk/Source/WebCore/PAL/pal/cocoa/SpeechSoftLink.h 2020-12-14 17:45:54 UTC (rev 270772)
@@ -0,0 +1,42 @@
+/*
+ * Copyright (C) 2020 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#pragma once
+
+#if HAVE(SPEECHRECOGNIZER)
+
+#import <Speech/Speech.h>
+#import <wtf/SoftLinking.h>
+
+SOFT_LINK_FRAMEWORK_FOR_HEADER(PAL, Speech);
+SOFT_LINK_CLASS_FOR_HEADER(PAL, SFSpeechRecognitionResult);
+SOFT_LINK_CLASS_FOR_HEADER(PAL, SFSpeechRecognitionRequest);
+SOFT_LINK_CLASS_FOR_HEADER(PAL, SFSpeechAudioBufferRecognitionRequest);
+SOFT_LINK_CLASS_FOR_HEADER(PAL, SFSpeechRecognitionTask);
+SOFT_LINK_CLASS_FOR_HEADER(PAL, SFSpeechRecognizer);
+SOFT_LINK_CLASS_FOR_HEADER(PAL, SFTranscriptionSegment);
+SOFT_LINK_CLASS_FOR_HEADER(PAL, SFTranscription);
+
+#endif
Copied: trunk/Source/WebCore/PAL/pal/cocoa/SpeechSoftLink.mm (from rev 270771, trunk/Source/WebCore/Modules/speech/SpeechRecognizer.h) (0 => 270772)
--- trunk/Source/WebCore/PAL/pal/cocoa/SpeechSoftLink.mm (rev 0)
+++ trunk/Source/WebCore/PAL/pal/cocoa/SpeechSoftLink.mm 2020-12-14 17:45:54 UTC (rev 270772)
@@ -0,0 +1,43 @@
+/*
+ * Copyright (C) 2020 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#import "config.h"
+
+#if HAVE(SPEECHRECOGNIZER)
+
+#import <Speech/Speech.h>
+#import <wtf/SoftLinking.h>
+
+SOFT_LINK_FRAMEWORK_FOR_SOURCE(PAL, Speech)
+
+SOFT_LINK_CLASS_FOR_SOURCE_WITH_EXPORT(PAL, Speech, SFSpeechRecognitionResult, PAL_EXPORT);
+SOFT_LINK_CLASS_FOR_SOURCE_WITH_EXPORT(PAL, Speech, SFSpeechRecognitionRequest, PAL_EXPORT);
+SOFT_LINK_CLASS_FOR_SOURCE_WITH_EXPORT(PAL, Speech, SFSpeechAudioBufferRecognitionRequest, PAL_EXPORT);
+SOFT_LINK_CLASS_FOR_SOURCE_WITH_EXPORT(PAL, Speech, SFSpeechRecognitionTask, PAL_EXPORT);
+SOFT_LINK_CLASS_FOR_SOURCE_WITH_EXPORT(PAL, Speech, SFSpeechRecognizer, PAL_EXPORT);
+SOFT_LINK_CLASS_FOR_SOURCE_WITH_EXPORT(PAL, Speech, SFTranscriptionSegment, PAL_EXPORT);
+SOFT_LINK_CLASS_FOR_SOURCE_WITH_EXPORT(PAL, Speech, SFTranscription, PAL_EXPORT);
+
+#endif
Copied: trunk/Source/WebCore/PAL/pal/spi/cocoa/SpeechSPI.h (from rev 270771, trunk/Source/WebCore/Modules/speech/SpeechRecognizer.h) (0 => 270772)
--- trunk/Source/WebCore/PAL/pal/spi/cocoa/SpeechSPI.h (rev 0)
+++ trunk/Source/WebCore/PAL/pal/spi/cocoa/SpeechSPI.h 2020-12-14 17:45:54 UTC (rev 270772)
@@ -0,0 +1,46 @@
+/*
+ * Copyright (C) 2020 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#if HAVE(SPEECHRECOGNIZER)
+
+#if USE(APPLE_INTERNAL_SDK)
+
+#import <Speech/SFSpeechRecognitionRequest_Private.h>
+
+#else
+
+NS_ASSUME_NONNULL_BEGIN
+
+@interface SFSpeechRecognitionRequest ()
+@property (nonatomic) BOOL detectMultipleUtterances;
+@property (nonatomic, getter=_maximumRecognitionDuration, setter=_setMaximumRecognitionDuration:) NSTimeInterval _maximumRecognitionDuration;
+
+@end
+
+NS_ASSUME_NONNULL_END
+
+#endif // USE(APPLE_INTERNAL_SDK)
+
+#endif // HAVE(SPEECHRECOGNIZER)
Modified: trunk/Source/WebCore/SourcesCocoa.txt (270771 => 270772)
--- trunk/Source/WebCore/SourcesCocoa.txt 2020-12-14 16:54:48 UTC (rev 270771)
+++ trunk/Source/WebCore/SourcesCocoa.txt 2020-12-14 17:45:54 UTC (rev 270772)
@@ -92,6 +92,9 @@
Modules/mediastream/RTCRtpSFrameTransformerCocoa.cpp
Modules/plugins/QuickTimePluginReplacement.mm
Modules/plugins/YouTubePluginReplacement.cpp
+Modules/speech/cocoa/SpeechRecognizerCocoa.mm
+Modules/speech/cocoa/WebSpeechRecognizerTask.mm
+Modules/speech/cocoa/WebSpeechRecognizerTaskMock.mm
Modules/webdatabase/cocoa/DatabaseManagerCocoa.mm
accessibility/ios/AXObjectCacheIOS.mm
accessibility/ios/AccessibilityObjectIOS.mm
Modified: trunk/Source/WebCore/WebCore.xcodeproj/project.pbxproj (270771 => 270772)
--- trunk/Source/WebCore/WebCore.xcodeproj/project.pbxproj 2020-12-14 16:54:48 UTC (rev 270771)
+++ trunk/Source/WebCore/WebCore.xcodeproj/project.pbxproj 2020-12-14 17:45:54 UTC (rev 270772)
@@ -2819,6 +2819,8 @@
93A8061E1E03B585008A1F26 /* JSDoubleRange.h in Headers */ = {isa = PBXBuildFile; fileRef = 93A8061A1E03B585008A1F26 /* JSDoubleRange.h */; };
93A806201E03B585008A1F26 /* JSLongRange.h in Headers */ = {isa = PBXBuildFile; fileRef = 93A8061C1E03B585008A1F26 /* JSLongRange.h */; };
93B2D8160F9920D2006AE6B2 /* SuddenTermination.h in Headers */ = {isa = PBXBuildFile; fileRef = 93B2D8150F9920D2006AE6B2 /* SuddenTermination.h */; settings = {ATTRIBUTES = (Private, ); }; };
+ 93B38EC325821DB400198E63 /* WebSpeechRecognizerTask.h in Headers */ = {isa = PBXBuildFile; fileRef = 93B38EBA2582193B00198E63 /* WebSpeechRecognizerTask.h */; };
+ 93B38EC425821DB700198E63 /* WebSpeechRecognizerTaskMock.h in Headers */ = {isa = PBXBuildFile; fileRef = 93B38EBB2582193D00198E63 /* WebSpeechRecognizerTaskMock.h */; };
93B6A0E60B0BCA5C00F5027A /* ContextMenu.h in Headers */ = {isa = PBXBuildFile; fileRef = 93B6A0E50B0BCA5C00F5027A /* ContextMenu.h */; settings = {ATTRIBUTES = (Private, ); }; };
93B70D6409EB0C7C009D8468 /* JSDOMBinding.h in Headers */ = {isa = PBXBuildFile; fileRef = 93B70D4809EB0C7C009D8468 /* JSDOMBinding.h */; settings = {ATTRIBUTES = (Private, ); }; };
93B70D6A09EB0C7C009D8468 /* JSEventListener.h in Headers */ = {isa = PBXBuildFile; fileRef = 93B70D4E09EB0C7C009D8468 /* JSEventListener.h */; settings = {ATTRIBUTES = (Private, ); }; };
@@ -11541,6 +11543,11 @@
93A8061C1E03B585008A1F26 /* JSLongRange.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = JSLongRange.h; sourceTree = "<group>"; };
93B2D8150F9920D2006AE6B2 /* SuddenTermination.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = SuddenTermination.h; sourceTree = "<group>"; };
93B2D8170F9920EE006AE6B2 /* SuddenTermination.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; path = SuddenTermination.mm; sourceTree = "<group>"; };
+ 93B38EB82582189900198E63 /* SpeechRecognizerCocoa.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; path = SpeechRecognizerCocoa.mm; sourceTree = "<group>"; };
+ 93B38EB92582193A00198E63 /* WebSpeechRecognizerTaskMock.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; path = WebSpeechRecognizerTaskMock.mm; sourceTree = "<group>"; };
+ 93B38EBA2582193B00198E63 /* WebSpeechRecognizerTask.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = WebSpeechRecognizerTask.h; sourceTree = "<group>"; };
+ 93B38EBB2582193D00198E63 /* WebSpeechRecognizerTaskMock.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = WebSpeechRecognizerTaskMock.h; sourceTree = "<group>"; };
+ 93B38EBC2582193E00198E63 /* WebSpeechRecognizerTask.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; path = WebSpeechRecognizerTask.mm; sourceTree = "<group>"; };
93B6A0E50B0BCA5C00F5027A /* ContextMenu.h */ = {isa = PBXFileReference; fileEncoding = 30; lastKnownFileType = sourcecode.c.h; path = ContextMenu.h; sourceTree = "<group>"; };
93B70D4809EB0C7C009D8468 /* JSDOMBinding.h */ = {isa = PBXFileReference; fileEncoding = 30; lastKnownFileType = sourcecode.c.h; path = JSDOMBinding.h; sourceTree = "<group>"; };
93B70D4D09EB0C7C009D8468 /* JSEventListener.cpp */ = {isa = PBXFileReference; fileEncoding = 30; lastKnownFileType = sourcecode.cpp.cpp; path = JSEventListener.cpp; sourceTree = "<group>"; };
@@ -23191,6 +23198,18 @@
path = mac;
sourceTree = "<group>";
};
+ 93B38EB62582183300198E63 /* cocoa */ = {
+ isa = PBXGroup;
+ children = (
+ 93B38EB82582189900198E63 /* SpeechRecognizerCocoa.mm */,
+ 93B38EBA2582193B00198E63 /* WebSpeechRecognizerTask.h */,
+ 93B38EBC2582193E00198E63 /* WebSpeechRecognizerTask.mm */,
+ 93B38EBB2582193D00198E63 /* WebSpeechRecognizerTaskMock.h */,
+ 93B38EB92582193A00198E63 /* WebSpeechRecognizerTaskMock.mm */,
+ );
+ path = cocoa;
+ sourceTree = "<group>";
+ };
93C09A820B064F05005ABD4D /* mac */ = {
isa = PBXGroup;
children = (
@@ -25723,6 +25742,7 @@
AA2A5AB716A485A400975A25 /* speech */ = {
isa = PBXGroup;
children = (
+ 93B38EB62582183300198E63 /* cocoa */,
AA2A5ABA16A485D500975A25 /* DOMWindow+SpeechSynthesis.idl */,
AA2A5AB816A485D500975A25 /* DOMWindowSpeechSynthesis.cpp */,
AA2A5AB916A485D500975A25 /* DOMWindowSpeechSynthesis.h */,
@@ -35303,6 +35323,8 @@
97AABD2514FA09D5007457AE /* WebSocketFrame.h in Headers */,
97AABD2714FA09D5007457AE /* WebSocketHandshake.h in Headers */,
41E12E9F24FE74E20093FFB4 /* WebSocketIdentifier.h in Headers */,
+ 93B38EC325821DB400198E63 /* WebSpeechRecognizerTask.h in Headers */,
+ 93B38EC425821DB700198E63 /* WebSpeechRecognizerTaskMock.h in Headers */,
1F8756B21E22C3350042C40D /* WebSQLiteDatabaseTrackerClient.h in Headers */,
31DEA4561B39F4D900F77178 /* WebSystemBackdropLayer.h in Headers */,
0F580FA31496939100FB5BD8 /* WebTiledBackingLayer.h in Headers */,
Modified: trunk/Source/WebKit/ChangeLog (270771 => 270772)
--- trunk/Source/WebKit/ChangeLog 2020-12-14 16:54:48 UTC (rev 270771)
+++ trunk/Source/WebKit/ChangeLog 2020-12-14 17:45:54 UTC (rev 270772)
@@ -1,3 +1,21 @@
+2020-12-14 Sihui Liu <[email protected]>
+
+ Implement recognizer for SpeechRecognition
+ https://bugs.webkit.org/show_bug.cgi?id=219459
+ <rdar://problem/71914465>
+
+ Reviewed by Youenn Fablet.
+
+ * UIProcess/SpeechRecognitionServer.cpp:
+ (WebKit::SpeechRecognitionServer::SpeechRecognitionServer):
+ (WebKit::SpeechRecognitionServer::requestPermissionForRequest):
+ (WebKit::SpeechRecognitionServer::handleRequest):
+ (WebKit::SpeechRecognitionServer::abort):
+ (WebKit::SpeechRecognitionServer::invalidate):
+ * UIProcess/SpeechRecognitionServer.h:
+ * UIProcess/WebProcessProxy.cpp:
+ (WebKit::WebProcessProxy::createSpeechRecognitionServer):
+
2020-12-14 Chris Dumez <[email protected]>
[GPUProcess] Crash under AudioDestinationCocoa::setIsPlaying(bool)
Modified: trunk/Source/WebKit/UIProcess/SpeechRecognitionServer.cpp (270771 => 270772)
--- trunk/Source/WebKit/UIProcess/SpeechRecognitionServer.cpp 2020-12-14 16:54:48 UTC (rev 270771)
+++ trunk/Source/WebKit/UIProcess/SpeechRecognitionServer.cpp 2020-12-14 17:45:54 UTC (rev 270772)
@@ -36,22 +36,21 @@
namespace WebKit {
+
+SpeechRecognitionServer::SpeechRecognitionServer(Ref<IPC::Connection>&& connection, SpeechRecognitionServerIdentifier identifier, SpeechRecognitionPermissionChecker&& permissionChecker, SpeechRecognitionCheckIfmockSpeechRecognitionEnabled&& checkIfEnabled
#if ENABLE(MEDIA_STREAM)
-SpeechRecognitionServer::SpeechRecognitionServer(Ref<IPC::Connection>&& connection, SpeechRecognitionServerIdentifier identifier, SpeechRecognitionPermissionChecker&& permissionChecker, RealtimeMediaSourceCreateFunction&& function)
+ , RealtimeMediaSourceCreateFunction&& function
+#endif
+ )
: m_connection(WTFMove(connection))
, m_identifier(identifier)
, m_permissionChecker(WTFMove(permissionChecker))
+ , m_checkIfmockSpeechRecognitionEnabled(WTFMove(checkIfEnabled))
+#if ENABLE(MEDIA_STREAM)
, m_realtimeMediaSourceCreateFunction(WTFMove(function))
+#endif
{
}
-#else
-SpeechRecognitionServer::SpeechRecognitionServer(Ref<IPC::Connection>&& connection, SpeechRecognitionServerIdentifier identifier, SpeechRecognitionPermissionChecker&& permissionChecker)
- : m_connection(WTFMove(connection))
- , m_identifier(identifier)
- , m_permissionChecker(WTFMove(permissionChecker))
-{
-}
-#endif
void SpeechRecognitionServer::start(WebCore::SpeechRecognitionConnectionClientIdentifier clientIdentifier, String&& lang, bool continuous, bool interimResults, uint64_t maxAlternatives, WebCore::ClientOrigin&& origin)
{
@@ -80,11 +79,11 @@
return;
}
- handleRequest(identifier);
+ handleRequest(*weakRequest);
});
}
-void SpeechRecognitionServer::handleRequest(WebCore::SpeechRecognitionConnectionClientIdentifier clientIdentifier)
+void SpeechRecognitionServer::handleRequest(WebCore::SpeechRecognitionRequest& request)
{
if (!m_recognizer) {
m_recognizer = makeUnique<WebCore::SpeechRecognizer>([this, weakThis = makeWeakPtr(this)](auto& update) {
@@ -95,16 +94,29 @@
if (!m_requests.contains(clientIdentifier))
return;
+ sendUpdate(update);
+
auto type = update.type();
- if (type == WebCore::SpeechRecognitionUpdateType::Error || type == WebCore::SpeechRecognitionUpdateType::End)
- m_requests.remove(clientIdentifier);
+ if (type != WebCore::SpeechRecognitionUpdateType::Error && type != WebCore::SpeechRecognitionUpdateType::End)
+ return;
- sendUpdate(update);
+ if (m_isResetting)
+ return;
+ m_isResetting = true;
+
+ m_recognizer->reset();
+ m_requests.remove(clientIdentifier);
+ m_isResetting = false;
});
}
-
- m_recognizer->reset();
+ if (auto currentClientIdentifier = m_recognizer->currentClientIdentifier()) {
+ auto error = WebCore::SpeechRecognitionError { WebCore::SpeechRecognitionErrorType::Aborted, "Another request is started"_s };
+ sendUpdate(*currentClientIdentifier, WebCore::SpeechRecognitionUpdateType::Error, error);
+ m_recognizer->reset();
+ }
+
+ auto clientIdentifier = request.clientIdentifier();
#if ENABLE(MEDIA_STREAM)
auto sourceOrError = m_realtimeMediaSourceCreateFunction();
if (!sourceOrError) {
@@ -112,10 +124,12 @@
sendUpdate(WebCore::SpeechRecognitionUpdate::createError(clientIdentifier, WebCore::SpeechRecognitionError { WebCore::SpeechRecognitionErrorType::AudioCapture, sourceOrError.errorMessage }));
return;
}
- m_recognizer->start(clientIdentifier, sourceOrError.source());
+
+ bool mockDeviceCapturesEnabled = m_checkIfmockSpeechRecognitionEnabled();
+ m_recognizer->start(clientIdentifier, sourceOrError.source(), mockDeviceCapturesEnabled, request.lang(), request.continuous(), request.interimResults(), request.maxAlternatives());
#else
m_requests.remove(clientIdentifier);
- sendUpdate(clientIdentifier, WebCore::SpeechRecognitionUpdateType::Error, WebCore::SpeechRecognitionError { WebCore::SpeechRecognitionErrorType::AudioCapture, "Audio capture is not implemented"});
+ sendUpdate(clientIdentifier, WebCore::SpeechRecognitionUpdateType::Error, WebCore::SpeechRecognitionError { WebCore::SpeechRecognitionErrorType::AudioCapture, "Audio capture is not implemented"_s });
#endif
}
@@ -135,7 +149,7 @@
{
MESSAGE_CHECK(clientIdentifier);
if (m_recognizer && m_recognizer->currentClientIdentifier() == clientIdentifier) {
- m_recognizer->stop(WebCore::SpeechRecognizer::ShouldGenerateFinalResult::No);
+ m_recognizer->abort();
return;
}
@@ -148,7 +162,7 @@
MESSAGE_CHECK(clientIdentifier);
if (m_requests.remove(clientIdentifier)) {
if (m_recognizer && m_recognizer->currentClientIdentifier() == clientIdentifier)
- m_recognizer->stop();
+ m_recognizer->abort();
}
}
Modified: trunk/Source/WebKit/UIProcess/SpeechRecognitionServer.h (270771 => 270772)
--- trunk/Source/WebKit/UIProcess/SpeechRecognitionServer.h 2020-12-14 16:54:48 UTC (rev 270771)
+++ trunk/Source/WebKit/UIProcess/SpeechRecognitionServer.h 2020-12-14 17:45:54 UTC (rev 270772)
@@ -47,6 +47,7 @@
using SpeechRecognitionServerIdentifier = WebCore::PageIdentifier;
using SpeechRecognitionPermissionChecker = Function<void(const WebCore::ClientOrigin&, CompletionHandler<void(SpeechRecognitionPermissionDecision)>&&)>;
+using SpeechRecognitionCheckIfmockSpeechRecognitionEnabled = Function<bool()>;
class SpeechRecognitionServer : public CanMakeWeakPtr<SpeechRecognitionServer>, public IPC::MessageReceiver, private IPC::MessageSender {
WTF_MAKE_FAST_ALLOCATED;
@@ -53,9 +54,9 @@
public:
#if ENABLE(MEDIA_STREAM)
using RealtimeMediaSourceCreateFunction = Function<WebCore::CaptureSourceOrError()>;
- SpeechRecognitionServer(Ref<IPC::Connection>&&, SpeechRecognitionServerIdentifier, SpeechRecognitionPermissionChecker&&, RealtimeMediaSourceCreateFunction&&);
+ SpeechRecognitionServer(Ref<IPC::Connection>&&, SpeechRecognitionServerIdentifier, SpeechRecognitionPermissionChecker&&, SpeechRecognitionCheckIfmockSpeechRecognitionEnabled&&, RealtimeMediaSourceCreateFunction&&);
#else
- SpeechRecognitionServer(Ref<IPC::Connection>&&, SpeechRecognitionServerIdentifier, SpeechRecognitionPermissionChecker&&);
+ SpeechRecognitionServer(Ref<IPC::Connection>&&, SpeechRecognitionServerIdentifier, SpeechRecognitionPermissionChecker&&, SpeechRecognitionCheckIfmockSpeechRecognitionEnabled&&);
#endif
void start(WebCore::SpeechRecognitionConnectionClientIdentifier, String&& lang, bool continuous, bool interimResults, uint64_t maxAlternatives, WebCore::ClientOrigin&&);
@@ -65,7 +66,7 @@
private:
void requestPermissionForRequest(WebCore::SpeechRecognitionRequest&);
- void handleRequest(WebCore::SpeechRecognitionConnectionClientIdentifier);
+ void handleRequest(WebCore::SpeechRecognitionRequest&);
void sendUpdate(WebCore::SpeechRecognitionConnectionClientIdentifier, WebCore::SpeechRecognitionUpdateType, Optional<WebCore::SpeechRecognitionError> = WTF::nullopt, Optional<Vector<WebCore::SpeechRecognitionResultData>> = WTF::nullopt);
void sendUpdate(const WebCore::SpeechRecognitionUpdate&);
@@ -81,6 +82,8 @@
HashMap<WebCore::SpeechRecognitionConnectionClientIdentifier, std::unique_ptr<WebCore::SpeechRecognitionRequest>> m_requests;
SpeechRecognitionPermissionChecker m_permissionChecker;
std::unique_ptr<WebCore::SpeechRecognizer> m_recognizer;
+ SpeechRecognitionCheckIfmockSpeechRecognitionEnabled m_checkIfmockSpeechRecognitionEnabled;
+ bool m_isResetting { false };
#if ENABLE(MEDIA_STREAM)
RealtimeMediaSourceCreateFunction m_realtimeMediaSourceCreateFunction;
Modified: trunk/Source/WebKit/UIProcess/WebProcessProxy.cpp (270771 => 270772)
--- trunk/Source/WebKit/UIProcess/WebProcessProxy.cpp 2020-12-14 16:54:48 UTC (rev 270771)
+++ trunk/Source/WebKit/UIProcess/WebProcessProxy.cpp 2020-12-14 17:45:54 UTC (rev 270772)
@@ -1722,7 +1722,7 @@
ASSERT(!m_speechRecognitionServerMap.contains(identifier));
auto& speechRecognitionServer = m_speechRecognitionServerMap.add(identifier, nullptr).iterator->value;
- speechRecognitionServer = makeUnique<SpeechRecognitionServer>(makeRef(*connection()), identifier, [weakPage = makeWeakPtr(targetPage)](auto& origin, auto&& completionHandler) mutable {
+ auto permissionChecker = [weakPage = makeWeakPtr(targetPage)](auto& origin, auto&& completionHandler) mutable {
if (!weakPage) {
completionHandler(SpeechRecognitionPermissionDecision::Deny);
return;
@@ -1729,13 +1729,20 @@
}
weakPage->requestSpeechRecognitionPermission(origin, WTFMove(completionHandler));
- }
+ };
+ auto checkIfMockCaptureDevicesEnabled = [weakPage = makeWeakPtr(targetPage)]() {
+ return weakPage && weakPage->preferences().mockCaptureDevicesEnabled();
+ };
+
#if ENABLE(MEDIA_STREAM)
- , [weakPage = makeWeakPtr(targetPage)]() {
+ auto createRealtimeMediaSource = [weakPage = makeWeakPtr(targetPage)]() {
return weakPage ? weakPage->createRealtimeMediaSourceForSpeechRecognition() : CaptureSourceOrError { "Page is invalid" };
- }
+ };
+ speechRecognitionServer = makeUnique<SpeechRecognitionServer>(makeRef(*connection()), identifier, WTFMove(permissionChecker), WTFMove(checkIfMockCaptureDevicesEnabled), WTFMove(createRealtimeMediaSource));
+#else
+ speechRecognitionServer = makeUnique<SpeechRecognitionServer>(makeRef(*connection()), identifier, WTFMove(permissionChecker), WTFMove(checkIfMockCaptureDevicesEnabled));
#endif
- );
+
addMessageReceiver(Messages::SpeechRecognitionServer::messageReceiverName(), identifier, *speechRecognitionServer);
}
