| /** |
| * Copyright (C) 2014 The Android Open Source Project |
| * |
| * Licensed under the Apache License, Version 2.0 (the "License"); |
| * you may not use this file except in compliance with the License. |
| * You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| package android.service.voice; |
| |
| import android.content.Intent; |
| import android.hardware.soundtrigger.IRecognitionStatusCallback; |
| import android.hardware.soundtrigger.KeyphraseEnrollmentInfo; |
| import android.hardware.soundtrigger.KeyphraseMetadata; |
| import android.hardware.soundtrigger.SoundTrigger; |
| import android.hardware.soundtrigger.SoundTrigger.ConfidenceLevel; |
| import android.hardware.soundtrigger.SoundTrigger.Keyphrase; |
| import android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionExtra; |
| import android.hardware.soundtrigger.SoundTrigger.KeyphraseSoundModel; |
| import android.hardware.soundtrigger.SoundTrigger.ModuleProperties; |
| import android.hardware.soundtrigger.SoundTrigger.RecognitionConfig; |
| import android.os.RemoteException; |
| import android.util.Slog; |
| |
| import com.android.internal.app.IVoiceInteractionManagerService; |
| |
| import java.util.List; |
| |
| /** |
| * A class that lets a VoiceInteractionService implementation interact with |
| * always-on keyphrase detection APIs. |
| */ |
| public class AlwaysOnHotwordDetector { |
| //---- States of Keyphrase availability ----// |
| /** |
| * Indicates that the given keyphrase is not available on the system because of the |
| * hardware configuration. |
| */ |
| public static final int KEYPHRASE_HARDWARE_UNAVAILABLE = -2; |
| /** |
| * Indicates that the given keyphrase is not supported. |
| */ |
| public static final int KEYPHRASE_UNSUPPORTED = -1; |
| /** |
| * Indicates that the given keyphrase is not enrolled. |
| */ |
| public static final int KEYPHRASE_UNENROLLED = 1; |
| /** |
| * Indicates that the given keyphrase is currently enrolled but not being actively listened for. |
| */ |
| public static final int KEYPHRASE_ENROLLED = 2; |
| |
| // Keyphrase management actions ----// |
| /** Indicates that we need to enroll. */ |
| public static final int MANAGE_ACTION_ENROLL = 0; |
| /** Indicates that we need to re-enroll. */ |
| public static final int MANAGE_ACTION_RE_ENROLL = 1; |
| /** Indicates that we need to un-enroll. */ |
| public static final int MANAGE_ACTION_UN_ENROLL = 2; |
| |
| /** |
| * Return codes for {@link #startRecognition(int)}, {@link #stopRecognition()} |
| */ |
| public static final int STATUS_ERROR = SoundTrigger.STATUS_ERROR; |
| public static final int STATUS_OK = SoundTrigger.STATUS_OK; |
| |
| //---- Keyphrase recognition status ----// |
| /** Indicates that recognition is not available. */ |
| public static final int RECOGNITION_STATUS_NOT_AVAILABLE = 0x01; |
| /** Indicates that recognition has not been requested. */ |
| public static final int RECOGNITION_STATUS_NOT_REQUESTED = 0x02; |
| /** Indicates that recognition has been requested. */ |
| public static final int RECOGNITION_STATUS_REQUESTED = 0x04; |
| /** Indicates that recognition has been temporarily disabled. */ |
| public static final int RECOGNITION_STATUS_DISABLED_TEMPORARILY = 0x08; |
| /** Indicates that recognition is currently active . */ |
| public static final int RECOGNITION_STATUS_ACTIVE = 0x10; |
| |
| //-- Flags for startRecogntion ----// |
| /** Empty flag for {@link #startRecognition(int)}. */ |
| public static final int RECOGNITION_FLAG_NONE = 0; |
| /** |
| * Recognition flag for {@link #startRecognition(int)} that indicates |
| * whether the trigger audio for hotword needs to be captured. |
| */ |
| public static final int RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO = 0x1; |
| |
| //---- Recognition mode flags ----// |
| // Must be kept in sync with the related attribute defined as searchKeyphraseRecognitionFlags. |
| |
| /** Simple recognition of the key phrase. Returned by {@link #getRecognitionStatus()} */ |
| public static final int RECOGNITION_MODE_VOICE_TRIGGER |
| = SoundTrigger.RECOGNITION_MODE_VOICE_TRIGGER; |
| /** Trigger only if one user is identified. Returned by {@link #getRecognitionStatus()} */ |
| public static final int RECOGNITION_MODE_USER_IDENTIFICATION |
| = SoundTrigger.RECOGNITION_MODE_USER_IDENTIFICATION; |
| |
| static final String TAG = "AlwaysOnHotwordDetector"; |
| |
| private final String mText; |
| private final String mLocale; |
| /** |
| * The metadata of the Keyphrase, derived from the enrollment application. |
| * This may be null if this keyphrase isn't supported by the enrollment application. |
| */ |
| private final KeyphraseMetadata mKeyphraseMetadata; |
| /** |
| * The sound model for the keyphrase, derived from the model management service |
| * (IVoiceInteractionManagerService). May be null if the keyphrase isn't enrolled yet. |
| */ |
| private final KeyphraseSoundModel mEnrolledSoundModel; |
| private final KeyphraseEnrollmentInfo mKeyphraseEnrollmentInfo; |
| private final int mAvailability; |
| private final IVoiceInteractionService mVoiceInteractionService; |
| private final IVoiceInteractionManagerService mModelManagementService; |
| private final SoundTriggerListener mInternalCallback; |
| |
| private int mRecognitionState; |
| |
| /** |
| * Callbacks for always-on hotword detection. |
| */ |
| public interface Callback { |
| /** |
| * Called when the keyphrase is spoken. |
| * |
| * @param data Optional trigger audio data, if it was requested during |
| * {@link AlwaysOnHotwordDetector#startRecognition(int)}. |
| */ |
| void onDetected(byte[] data); |
| /** |
| * Called when the detection for the associated keyphrase starts. |
| */ |
| void onDetectionStarted(); |
| /** |
| * Called when the detection for the associated keyphrase stops. |
| */ |
| void onDetectionStopped(); |
| } |
| |
| /** |
| * @param text The keyphrase text to get the detector for. |
| * @param locale The java locale for the detector. |
| * @param callback A non-null Callback for receiving the recognition events. |
| * @param voiceInteractionService The current voice interaction service. |
| * @param modelManagementService A service that allows management of sound models. |
| * |
| * @hide |
| */ |
| public AlwaysOnHotwordDetector(String text, String locale, Callback callback, |
| KeyphraseEnrollmentInfo keyphraseEnrollmentInfo, |
| IVoiceInteractionService voiceInteractionService, |
| IVoiceInteractionManagerService modelManagementService) { |
| mText = text; |
| mLocale = locale; |
| mKeyphraseEnrollmentInfo = keyphraseEnrollmentInfo; |
| mKeyphraseMetadata = mKeyphraseEnrollmentInfo.getKeyphraseMetadata(text, locale); |
| mInternalCallback = new SoundTriggerListener(callback); |
| mVoiceInteractionService = voiceInteractionService; |
| mModelManagementService = modelManagementService; |
| if (mKeyphraseMetadata != null) { |
| mEnrolledSoundModel = internalGetKeyphraseSoundModel(mKeyphraseMetadata.id); |
| } else { |
| mEnrolledSoundModel = null; |
| } |
| mAvailability = internalGetAvailability(); |
| } |
| |
| /** |
| * Gets the state of always-on hotword detection for the given keyphrase and locale |
| * on this system. |
| * Availability implies that the hardware on this system is capable of listening for |
| * the given keyphrase or not. |
| * |
| * @return Indicates if always-on hotword detection is available for the given keyphrase. |
| * The return code is one of {@link #KEYPHRASE_HARDWARE_UNAVAILABLE}, |
| * {@link #KEYPHRASE_UNSUPPORTED}, {@link #KEYPHRASE_UNENROLLED} or |
| * {@link #KEYPHRASE_ENROLLED}. |
| */ |
| public int getAvailability() { |
| return mAvailability; |
| } |
| |
| /** |
| * Gets the recognition modes supported by the associated keyphrase. |
| * |
| * @throws UnsupportedOperationException if the keyphrase itself isn't supported. |
| * Callers should check the availability by calling {@link #getAvailability()} |
| * before calling this method to avoid this exception. |
| */ |
| public int getSupportedRecognitionModes() { |
| if (mAvailability == KEYPHRASE_HARDWARE_UNAVAILABLE |
| || mAvailability == KEYPHRASE_UNSUPPORTED) { |
| throw new UnsupportedOperationException( |
| "Getting supported recognition modes for the keyphrase is not supported"); |
| } |
| |
| return mKeyphraseMetadata.recognitionModeFlags; |
| } |
| |
| /** |
| * Gets the status of the recognition. |
| * @return A flag comprised of {@link #RECOGNITION_STATUS_NOT_AVAILABLE}, |
| * {@link #RECOGNITION_STATUS_NOT_REQUESTED}, {@link #RECOGNITION_STATUS_REQUESTED}, |
| * {@link #RECOGNITION_STATUS_DISABLED_TEMPORARILY} and |
| * {@link #RECOGNITION_STATUS_ACTIVE}. |
| */ |
| public int getRecognitionStatus() { |
| return mRecognitionState; |
| } |
| |
| /** |
| * Starts recognition for the associated keyphrase. |
| * |
| * @param recognitionFlags The flags to control the recognition properties. |
| * The allowed flags are {@link #RECOGNITION_FLAG_NONE} and |
| * {@link #RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO}. |
| * @return {@link #STATUS_OK} if the call succeeds, an error code otherwise. |
| * @throws UnsupportedOperationException if the recognition isn't supported. |
| * Callers should check the availability by calling {@link #getAvailability()} |
| * before calling this method to avoid this exception. |
| */ |
| public int startRecognition(int recognitionFlags) { |
| if (mAvailability != KEYPHRASE_ENROLLED |
| || (mRecognitionState&RECOGNITION_STATUS_NOT_AVAILABLE) != 0) { |
| throw new UnsupportedOperationException( |
| "Recognition for the given keyphrase is not supported"); |
| } |
| |
| mRecognitionState &= RECOGNITION_STATUS_REQUESTED; |
| KeyphraseRecognitionExtra[] recognitionExtra = new KeyphraseRecognitionExtra[1]; |
| // TODO: Do we need to do something about the confidence level here? |
| // TODO: Take in captureTriggerAudio as a method param here. |
| recognitionExtra[0] = new KeyphraseRecognitionExtra(mKeyphraseMetadata.id, |
| mKeyphraseMetadata.recognitionModeFlags, new ConfidenceLevel[0]); |
| boolean captureTriggerAudio = |
| (recognitionFlags & RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO) != 0; |
| int code = STATUS_ERROR; |
| try { |
| code = mModelManagementService.startRecognition(mVoiceInteractionService, |
| mKeyphraseMetadata.id, mEnrolledSoundModel, mInternalCallback, |
| new RecognitionConfig( |
| captureTriggerAudio, recognitionExtra, null /* additional data */)); |
| } catch (RemoteException e) { |
| Slog.w(TAG, "RemoteException in startRecognition!"); |
| } |
| if (code != STATUS_OK) { |
| Slog.w(TAG, "startRecognition() failed with error code " + code); |
| } |
| return code; |
| } |
| |
| /** |
| * Stops recognition for the associated keyphrase. |
| * |
| * @return {@link #STATUS_OK} if the call succeeds, an error code otherwise. |
| * @throws UnsupportedOperationException if the recognition isn't supported. |
| * Callers should check the availability by calling {@link #getAvailability()} |
| * before calling this method to avoid this exception. |
| */ |
| public int stopRecognition() { |
| if (mAvailability != KEYPHRASE_ENROLLED) { |
| throw new UnsupportedOperationException( |
| "Recognition for the given keyphrase is not supported"); |
| } |
| |
| mRecognitionState &= ~RECOGNITION_STATUS_NOT_REQUESTED; |
| int code = STATUS_ERROR; |
| try { |
| code = mModelManagementService.stopRecognition( |
| mVoiceInteractionService, mKeyphraseMetadata.id, mInternalCallback); |
| } catch (RemoteException e) { |
| Slog.w(TAG, "RemoteException in stopRecognition!"); |
| } |
| |
| if (code != STATUS_OK) { |
| Slog.w(TAG, "stopRecognition() failed with error code " + code); |
| } |
| return code; |
| } |
| |
| /** |
| * Gets an intent to manage the associated keyphrase. |
| * |
| * @param action The manage action that needs to be performed. |
| * One of {@link #MANAGE_ACTION_ENROLL}, {@link #MANAGE_ACTION_RE_ENROLL} or |
| * {@link #MANAGE_ACTION_UN_ENROLL}. |
| * @return An {@link Intent} to manage the given keyphrase. |
| * @throws UnsupportedOperationException if managing they keyphrase isn't supported. |
| * Callers should check the availability by calling {@link #getAvailability()} |
| * before calling this method to avoid this exception. |
| */ |
| public Intent getManageIntent(int action) { |
| if (mAvailability == KEYPHRASE_HARDWARE_UNAVAILABLE |
| || mAvailability == KEYPHRASE_UNSUPPORTED) { |
| throw new UnsupportedOperationException( |
| "Managing the given keyphrase is not supported"); |
| } |
| if (action != MANAGE_ACTION_ENROLL |
| && action != MANAGE_ACTION_RE_ENROLL |
| && action != MANAGE_ACTION_UN_ENROLL) { |
| throw new IllegalArgumentException("Invalid action specified " + action); |
| } |
| |
| return mKeyphraseEnrollmentInfo.getManageKeyphraseIntent(action, mText, mLocale); |
| } |
| |
| private int internalGetAvailability() { |
| ModuleProperties dspModuleProperties = null; |
| try { |
| dspModuleProperties = |
| mModelManagementService.getDspModuleProperties(mVoiceInteractionService); |
| } catch (RemoteException e) { |
| Slog.w(TAG, "RemoteException in getDspProperties!"); |
| } |
| // No DSP available |
| if (dspModuleProperties == null) { |
| mRecognitionState = RECOGNITION_STATUS_NOT_AVAILABLE; |
| return KEYPHRASE_HARDWARE_UNAVAILABLE; |
| } |
| // No enrollment application supports this keyphrase/locale |
| if (mKeyphraseMetadata == null) { |
| mRecognitionState = RECOGNITION_STATUS_NOT_AVAILABLE; |
| return KEYPHRASE_UNSUPPORTED; |
| } |
| // This keyphrase hasn't been enrolled. |
| if (mEnrolledSoundModel == null) { |
| mRecognitionState = RECOGNITION_STATUS_NOT_AVAILABLE; |
| return KEYPHRASE_UNENROLLED; |
| } |
| // Mark recognition as available |
| mRecognitionState &= ~RECOGNITION_STATUS_NOT_AVAILABLE; |
| return KEYPHRASE_ENROLLED; |
| } |
| |
| /** |
| * @return The corresponding {@link KeyphraseSoundModel} or null if none is found. |
| */ |
| private KeyphraseSoundModel internalGetKeyphraseSoundModel(int keyphraseId) { |
| List<KeyphraseSoundModel> soundModels; |
| try { |
| soundModels = mModelManagementService |
| .listRegisteredKeyphraseSoundModels(mVoiceInteractionService); |
| if (soundModels == null || soundModels.isEmpty()) { |
| Slog.i(TAG, "No available sound models for keyphrase ID: " + keyphraseId); |
| return null; |
| } |
| for (KeyphraseSoundModel soundModel : soundModels) { |
| if (soundModel.keyphrases == null) { |
| continue; |
| } |
| for (Keyphrase keyphrase : soundModel.keyphrases) { |
| // TODO: Check the user handle here to only load a model for the current user. |
| if (keyphrase.id == keyphraseId) { |
| return soundModel; |
| } |
| } |
| } |
| } catch (RemoteException e) { |
| Slog.w(TAG, "RemoteException in listRegisteredKeyphraseSoundModels!"); |
| } |
| return null; |
| } |
| |
| /** @hide */ |
| static final class SoundTriggerListener extends IRecognitionStatusCallback.Stub { |
| private final Callback mCallback; |
| |
| public SoundTriggerListener(Callback callback) { |
| this.mCallback = callback; |
| } |
| |
| @Override |
| public void onDetected(byte[] data) { |
| Slog.i(TAG, "onKeyphraseSpoken"); |
| mCallback.onDetected(data); |
| } |
| |
| @Override |
| public void onDetectionStarted() { |
| // TODO: Set the RECOGNITION_STATUS_ACTIVE flag here. |
| mCallback.onDetectionStarted(); |
| } |
| |
| @Override |
| public void onDetectionStopped() { |
| // TODO: Unset the RECOGNITION_STATUS_ACTIVE flag here. |
| mCallback.onDetectionStopped(); |
| } |
| } |
| } |