Chris Thornton | dfa7c3b | 2016-06-30 22:05:51 -0700 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (C) 2014 The Android Open Source Project |
| 3 | * |
| 4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | * you may not use this file except in compliance with the License. |
| 6 | * You may obtain a copy of the License at |
| 7 | * |
| 8 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | * |
| 10 | * Unless required by applicable law or agreed to in writing, software |
| 11 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | * See the License for the specific language governing permissions and |
| 14 | * limitations under the License. |
| 15 | */ |
| 16 | |
| 17 | package com.android.test.soundtrigger; |
| 18 | |
| 19 | import android.Manifest; |
| 20 | import android.app.Service; |
| 21 | import android.content.BroadcastReceiver; |
| 22 | import android.content.Context; |
| 23 | import android.content.Intent; |
| 24 | import android.content.IntentFilter; |
| 25 | import android.content.pm.PackageManager; |
| 26 | import android.hardware.soundtrigger.SoundTrigger.GenericSoundModel; |
| 27 | import android.media.AudioAttributes; |
| 28 | import android.media.AudioFormat; |
| 29 | import android.media.AudioManager; |
| 30 | import android.media.AudioRecord; |
| 31 | import android.media.AudioTrack; |
| 32 | import android.media.MediaPlayer; |
| 33 | import android.media.soundtrigger.SoundTriggerDetector; |
| 34 | import android.net.Uri; |
| 35 | import android.os.Binder; |
| 36 | import android.os.IBinder; |
| 37 | import android.util.Log; |
| 38 | |
| 39 | import java.io.File; |
| 40 | import java.io.FileInputStream; |
| 41 | import java.io.FileOutputStream; |
| 42 | import java.io.IOException; |
| 43 | import java.util.HashMap; |
| 44 | import java.util.Map; |
| 45 | import java.util.Properties; |
| 46 | import java.util.Random; |
| 47 | import java.util.UUID; |
| 48 | |
| 49 | public class SoundTriggerTestService extends Service { |
| 50 | private static final String TAG = "SoundTriggerTestSrv"; |
| 51 | private static final String INTENT_ACTION = "com.android.intent.action.MANAGE_SOUND_TRIGGER"; |
| 52 | |
| 53 | // Binder given to clients. |
| 54 | private final IBinder mBinder; |
| 55 | private final Map<UUID, ModelInfo> mModelInfoMap; |
| 56 | private SoundTriggerUtil mSoundTriggerUtil; |
| 57 | private Random mRandom; |
| 58 | private UserActivity mUserActivity; |
| 59 | |
| 60 | public interface UserActivity { |
| 61 | void addModel(UUID modelUuid, String state); |
| 62 | void setModelState(UUID modelUuid, String state); |
| 63 | void showMessage(String msg, boolean showToast); |
| 64 | void handleDetection(UUID modelUuid); |
| 65 | } |
| 66 | |
| 67 | public SoundTriggerTestService() { |
| 68 | super(); |
| 69 | mRandom = new Random(); |
| 70 | mModelInfoMap = new HashMap(); |
| 71 | mBinder = new SoundTriggerTestBinder(); |
| 72 | } |
| 73 | |
| 74 | @Override |
| 75 | public synchronized int onStartCommand(Intent intent, int flags, int startId) { |
| 76 | if (mModelInfoMap.isEmpty()) { |
| 77 | mSoundTriggerUtil = new SoundTriggerUtil(this); |
| 78 | loadModelsInDataDir(); |
| 79 | } |
| 80 | |
| 81 | // If we get killed, after returning from here, restart |
| 82 | return START_STICKY; |
| 83 | } |
| 84 | |
| 85 | @Override |
| 86 | public void onCreate() { |
| 87 | super.onCreate(); |
| 88 | IntentFilter filter = new IntentFilter(); |
| 89 | filter.addAction(INTENT_ACTION); |
| 90 | registerReceiver(mBroadcastReceiver, filter); |
| 91 | |
| 92 | // Make sure the data directory exists, and we're the owner of it. |
| 93 | try { |
| 94 | getFilesDir().mkdir(); |
| 95 | } catch (Exception e) { |
| 96 | // Don't care - we either made it, or it already exists. |
| 97 | } |
| 98 | } |
| 99 | |
| 100 | @Override |
| 101 | public void onDestroy() { |
| 102 | super.onDestroy(); |
Chris Thornton | e492e49 | 2017-03-22 13:50:04 -0700 | [diff] [blame] | 103 | stopAllRecognitionsAndUnload(); |
Chris Thornton | dfa7c3b | 2016-06-30 22:05:51 -0700 | [diff] [blame] | 104 | unregisterReceiver(mBroadcastReceiver); |
| 105 | } |
| 106 | |
| 107 | private final BroadcastReceiver mBroadcastReceiver = new BroadcastReceiver() { |
| 108 | @Override |
| 109 | public void onReceive(Context context, Intent intent) { |
| 110 | if (intent != null && INTENT_ACTION.equals(intent.getAction())) { |
| 111 | String command = intent.getStringExtra("command"); |
| 112 | if (command == null) { |
| 113 | Log.e(TAG, "No 'command' specified in " + INTENT_ACTION); |
| 114 | } else { |
| 115 | try { |
| 116 | if (command.equals("load")) { |
| 117 | loadModel(getModelUuidFromIntent(intent)); |
| 118 | } else if (command.equals("unload")) { |
| 119 | unloadModel(getModelUuidFromIntent(intent)); |
| 120 | } else if (command.equals("start")) { |
| 121 | startRecognition(getModelUuidFromIntent(intent)); |
| 122 | } else if (command.equals("stop")) { |
| 123 | stopRecognition(getModelUuidFromIntent(intent)); |
| 124 | } else if (command.equals("play_trigger")) { |
| 125 | playTriggerAudio(getModelUuidFromIntent(intent)); |
| 126 | } else if (command.equals("play_captured")) { |
| 127 | playCapturedAudio(getModelUuidFromIntent(intent)); |
| 128 | } else if (command.equals("set_capture")) { |
| 129 | setCaptureAudio(getModelUuidFromIntent(intent), |
| 130 | intent.getBooleanExtra("enabled", true)); |
| 131 | } else if (command.equals("set_capture_timeout")) { |
| 132 | setCaptureAudioTimeout(getModelUuidFromIntent(intent), |
| 133 | intent.getIntExtra("timeout", 5000)); |
| 134 | } else { |
| 135 | Log.e(TAG, "Unknown command '" + command + "'"); |
| 136 | } |
| 137 | } catch (Exception e) { |
| 138 | Log.e(TAG, "Failed to process " + command, e); |
| 139 | } |
| 140 | } |
| 141 | } |
| 142 | } |
| 143 | }; |
| 144 | |
| 145 | private UUID getModelUuidFromIntent(Intent intent) { |
| 146 | // First, see if the specified the UUID straight up. |
| 147 | String value = intent.getStringExtra("modelUuid"); |
| 148 | if (value != null) { |
| 149 | return UUID.fromString(value); |
| 150 | } |
| 151 | |
| 152 | // If they specified a name, use that to iterate through the map of models and find it. |
| 153 | value = intent.getStringExtra("name"); |
| 154 | if (value != null) { |
| 155 | for (ModelInfo modelInfo : mModelInfoMap.values()) { |
| 156 | if (value.equals(modelInfo.name)) { |
| 157 | return modelInfo.modelUuid; |
| 158 | } |
| 159 | } |
| 160 | Log.e(TAG, "Failed to find a matching model with name '" + value + "'"); |
| 161 | } |
| 162 | |
| 163 | // We couldn't figure out what they were asking for. |
| 164 | throw new RuntimeException("Failed to get model from intent - specify either " + |
| 165 | "'modelUuid' or 'name'"); |
| 166 | } |
| 167 | |
| 168 | /** |
| 169 | * Will be called when the service is killed (through swipe aways, not if we're force killed). |
| 170 | */ |
| 171 | @Override |
| 172 | public void onTaskRemoved(Intent rootIntent) { |
| 173 | super.onTaskRemoved(rootIntent); |
Chris Thornton | e492e49 | 2017-03-22 13:50:04 -0700 | [diff] [blame] | 174 | stopAllRecognitionsAndUnload(); |
Chris Thornton | dfa7c3b | 2016-06-30 22:05:51 -0700 | [diff] [blame] | 175 | stopSelf(); |
| 176 | } |
| 177 | |
| 178 | @Override |
| 179 | public synchronized IBinder onBind(Intent intent) { |
| 180 | return mBinder; |
| 181 | } |
| 182 | |
| 183 | public class SoundTriggerTestBinder extends Binder { |
| 184 | SoundTriggerTestService getService() { |
| 185 | // Return instance of our parent so clients can call public methods. |
| 186 | return SoundTriggerTestService.this; |
| 187 | } |
| 188 | } |
| 189 | |
| 190 | public synchronized void setUserActivity(UserActivity activity) { |
| 191 | mUserActivity = activity; |
| 192 | if (mUserActivity != null) { |
| 193 | for (Map.Entry<UUID, ModelInfo> entry : mModelInfoMap.entrySet()) { |
| 194 | mUserActivity.addModel(entry.getKey(), entry.getValue().name); |
| 195 | mUserActivity.setModelState(entry.getKey(), entry.getValue().state); |
| 196 | } |
| 197 | } |
| 198 | } |
| 199 | |
Chris Thornton | e492e49 | 2017-03-22 13:50:04 -0700 | [diff] [blame] | 200 | private synchronized void stopAllRecognitionsAndUnload() { |
| 201 | Log.e(TAG, "Stop all recognitions"); |
Chris Thornton | dfa7c3b | 2016-06-30 22:05:51 -0700 | [diff] [blame] | 202 | for (ModelInfo modelInfo : mModelInfoMap.values()) { |
Chris Thornton | e492e49 | 2017-03-22 13:50:04 -0700 | [diff] [blame] | 203 | Log.e(TAG, "Loop " + modelInfo.modelUuid); |
Chris Thornton | dfa7c3b | 2016-06-30 22:05:51 -0700 | [diff] [blame] | 204 | if (modelInfo.detector != null) { |
| 205 | Log.i(TAG, "Stopping recognition for " + modelInfo.name); |
| 206 | try { |
| 207 | modelInfo.detector.stopRecognition(); |
| 208 | } catch (Exception e) { |
| 209 | Log.e(TAG, "Failed to stop recognition", e); |
| 210 | } |
Chris Thornton | e492e49 | 2017-03-22 13:50:04 -0700 | [diff] [blame] | 211 | try { |
| 212 | mSoundTriggerUtil.deleteSoundModel(modelInfo.modelUuid); |
| 213 | modelInfo.detector = null; |
| 214 | } catch (Exception e) { |
| 215 | Log.e(TAG, "Failed to unload sound model", e); |
| 216 | } |
Chris Thornton | dfa7c3b | 2016-06-30 22:05:51 -0700 | [diff] [blame] | 217 | } |
| 218 | } |
| 219 | } |
| 220 | |
| 221 | // Helper struct for holding information about a model. |
| 222 | public static class ModelInfo { |
| 223 | public String name; |
| 224 | public String state; |
| 225 | public UUID modelUuid; |
| 226 | public UUID vendorUuid; |
| 227 | public MediaPlayer triggerAudioPlayer; |
| 228 | public SoundTriggerDetector detector; |
| 229 | public byte modelData[]; |
| 230 | public boolean captureAudio; |
| 231 | public int captureAudioMs; |
| 232 | public AudioTrack captureAudioTrack; |
| 233 | } |
| 234 | |
| 235 | private GenericSoundModel createNewSoundModel(ModelInfo modelInfo) { |
| 236 | return new GenericSoundModel(modelInfo.modelUuid, modelInfo.vendorUuid, |
| 237 | modelInfo.modelData); |
| 238 | } |
| 239 | |
| 240 | public synchronized void loadModel(UUID modelUuid) { |
| 241 | ModelInfo modelInfo = mModelInfoMap.get(modelUuid); |
| 242 | if (modelInfo == null) { |
| 243 | postError("Could not find model for: " + modelUuid.toString()); |
| 244 | return; |
| 245 | } |
| 246 | |
| 247 | postMessage("Loading model: " + modelInfo.name); |
| 248 | |
| 249 | GenericSoundModel soundModel = createNewSoundModel(modelInfo); |
| 250 | |
| 251 | boolean status = mSoundTriggerUtil.addOrUpdateSoundModel(soundModel); |
| 252 | if (status) { |
Nicholas Ambur | 1e43767 | 2020-03-24 17:09:58 -0700 | [diff] [blame] | 253 | postToast("Successfully loaded " + modelInfo.name + ", UUID=" |
| 254 | + soundModel.getUuid()); |
Chris Thornton | dfa7c3b | 2016-06-30 22:05:51 -0700 | [diff] [blame] | 255 | setModelState(modelInfo, "Loaded"); |
| 256 | } else { |
Nicholas Ambur | 1e43767 | 2020-03-24 17:09:58 -0700 | [diff] [blame] | 257 | postErrorToast("Failed to load " + modelInfo.name + ", UUID=" |
| 258 | + soundModel.getUuid() + "!"); |
Chris Thornton | dfa7c3b | 2016-06-30 22:05:51 -0700 | [diff] [blame] | 259 | setModelState(modelInfo, "Failed to load"); |
| 260 | } |
| 261 | } |
| 262 | |
| 263 | public synchronized void unloadModel(UUID modelUuid) { |
| 264 | ModelInfo modelInfo = mModelInfoMap.get(modelUuid); |
| 265 | if (modelInfo == null) { |
| 266 | postError("Could not find model for: " + modelUuid.toString()); |
| 267 | return; |
| 268 | } |
| 269 | |
| 270 | postMessage("Unloading model: " + modelInfo.name); |
| 271 | |
| 272 | GenericSoundModel soundModel = mSoundTriggerUtil.getSoundModel(modelUuid); |
| 273 | if (soundModel == null) { |
| 274 | postErrorToast("Sound model not found for " + modelInfo.name + "!"); |
| 275 | return; |
| 276 | } |
| 277 | modelInfo.detector = null; |
| 278 | boolean status = mSoundTriggerUtil.deleteSoundModel(modelUuid); |
| 279 | if (status) { |
Nicholas Ambur | 1e43767 | 2020-03-24 17:09:58 -0700 | [diff] [blame] | 280 | postToast("Successfully unloaded " + modelInfo.name + ", UUID=" |
| 281 | + soundModel.getUuid()); |
Chris Thornton | dfa7c3b | 2016-06-30 22:05:51 -0700 | [diff] [blame] | 282 | setModelState(modelInfo, "Unloaded"); |
| 283 | } else { |
| 284 | postErrorToast("Failed to unload " + |
Nicholas Ambur | 1e43767 | 2020-03-24 17:09:58 -0700 | [diff] [blame] | 285 | modelInfo.name + ", UUID=" + soundModel.getUuid() + "!"); |
Chris Thornton | dfa7c3b | 2016-06-30 22:05:51 -0700 | [diff] [blame] | 286 | setModelState(modelInfo, "Failed to unload"); |
| 287 | } |
| 288 | } |
| 289 | |
| 290 | public synchronized void reloadModel(UUID modelUuid) { |
| 291 | ModelInfo modelInfo = mModelInfoMap.get(modelUuid); |
| 292 | if (modelInfo == null) { |
| 293 | postError("Could not find model for: " + modelUuid.toString()); |
| 294 | return; |
| 295 | } |
| 296 | postMessage("Reloading model: " + modelInfo.name); |
| 297 | GenericSoundModel soundModel = mSoundTriggerUtil.getSoundModel(modelUuid); |
| 298 | if (soundModel == null) { |
| 299 | postErrorToast("Sound model not found for " + modelInfo.name + "!"); |
| 300 | return; |
| 301 | } |
| 302 | GenericSoundModel updated = createNewSoundModel(modelInfo); |
| 303 | boolean status = mSoundTriggerUtil.addOrUpdateSoundModel(updated); |
| 304 | if (status) { |
Nicholas Ambur | 1e43767 | 2020-03-24 17:09:58 -0700 | [diff] [blame] | 305 | postToast("Successfully reloaded " + modelInfo.name + ", UUID=" |
| 306 | + modelInfo.modelUuid); |
Chris Thornton | dfa7c3b | 2016-06-30 22:05:51 -0700 | [diff] [blame] | 307 | setModelState(modelInfo, "Reloaded"); |
| 308 | } else { |
| 309 | postErrorToast("Failed to reload " |
| 310 | + modelInfo.name + ", UUID=" + modelInfo.modelUuid + "!"); |
| 311 | setModelState(modelInfo, "Failed to reload"); |
| 312 | } |
| 313 | } |
| 314 | |
| 315 | public synchronized void startRecognition(UUID modelUuid) { |
| 316 | ModelInfo modelInfo = mModelInfoMap.get(modelUuid); |
| 317 | if (modelInfo == null) { |
| 318 | postError("Could not find model for: " + modelUuid.toString()); |
| 319 | return; |
| 320 | } |
| 321 | |
| 322 | if (modelInfo.detector == null) { |
| 323 | postMessage("Creating SoundTriggerDetector for " + modelInfo.name); |
| 324 | modelInfo.detector = mSoundTriggerUtil.createSoundTriggerDetector( |
| 325 | modelUuid, new DetectorCallback(modelInfo)); |
| 326 | } |
| 327 | |
Nicholas Ambur | 1e43767 | 2020-03-24 17:09:58 -0700 | [diff] [blame] | 328 | postMessage("Starting recognition for " + modelInfo.name + ", UUID=" |
| 329 | + modelInfo.modelUuid); |
Chris Thornton | dfa7c3b | 2016-06-30 22:05:51 -0700 | [diff] [blame] | 330 | if (modelInfo.detector.startRecognition(modelInfo.captureAudio ? |
| 331 | SoundTriggerDetector.RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO : |
| 332 | SoundTriggerDetector.RECOGNITION_FLAG_ALLOW_MULTIPLE_TRIGGERS)) { |
| 333 | setModelState(modelInfo, "Started"); |
| 334 | } else { |
| 335 | postErrorToast("Fast failure attempting to start recognition for " + |
| 336 | modelInfo.name + ", UUID=" + modelInfo.modelUuid); |
| 337 | setModelState(modelInfo, "Failed to start"); |
| 338 | } |
| 339 | } |
| 340 | |
| 341 | public synchronized void stopRecognition(UUID modelUuid) { |
| 342 | ModelInfo modelInfo = mModelInfoMap.get(modelUuid); |
| 343 | if (modelInfo == null) { |
| 344 | postError("Could not find model for: " + modelUuid.toString()); |
| 345 | return; |
| 346 | } |
| 347 | |
| 348 | if (modelInfo.detector == null) { |
| 349 | postErrorToast("Stop called on null detector for " + |
| 350 | modelInfo.name + ", UUID=" + modelInfo.modelUuid); |
| 351 | return; |
| 352 | } |
| 353 | postMessage("Triggering stop recognition for " + |
| 354 | modelInfo.name + ", UUID=" + modelInfo.modelUuid); |
| 355 | if (modelInfo.detector.stopRecognition()) { |
| 356 | setModelState(modelInfo, "Stopped"); |
| 357 | } else { |
| 358 | postErrorToast("Fast failure attempting to stop recognition for " + |
| 359 | modelInfo.name + ", UUID=" + modelInfo.modelUuid); |
| 360 | setModelState(modelInfo, "Failed to stop"); |
| 361 | } |
| 362 | } |
| 363 | |
| 364 | public synchronized void playTriggerAudio(UUID modelUuid) { |
| 365 | ModelInfo modelInfo = mModelInfoMap.get(modelUuid); |
| 366 | if (modelInfo == null) { |
| 367 | postError("Could not find model for: " + modelUuid.toString()); |
| 368 | return; |
| 369 | } |
| 370 | if (modelInfo.triggerAudioPlayer != null) { |
| 371 | postMessage("Playing trigger audio for " + modelInfo.name); |
| 372 | modelInfo.triggerAudioPlayer.start(); |
| 373 | } else { |
| 374 | postMessage("No trigger audio for " + modelInfo.name); |
| 375 | } |
| 376 | } |
| 377 | |
| 378 | public synchronized void playCapturedAudio(UUID modelUuid) { |
| 379 | ModelInfo modelInfo = mModelInfoMap.get(modelUuid); |
| 380 | if (modelInfo == null) { |
| 381 | postError("Could not find model for: " + modelUuid.toString()); |
| 382 | return; |
| 383 | } |
| 384 | if (modelInfo.captureAudioTrack != null) { |
| 385 | postMessage("Playing captured audio for " + modelInfo.name); |
| 386 | modelInfo.captureAudioTrack.stop(); |
| 387 | modelInfo.captureAudioTrack.reloadStaticData(); |
| 388 | modelInfo.captureAudioTrack.play(); |
| 389 | } else { |
| 390 | postMessage("No captured audio for " + modelInfo.name); |
| 391 | } |
| 392 | } |
| 393 | |
| 394 | public synchronized void setCaptureAudioTimeout(UUID modelUuid, int captureTimeoutMs) { |
| 395 | ModelInfo modelInfo = mModelInfoMap.get(modelUuid); |
| 396 | if (modelInfo == null) { |
| 397 | postError("Could not find model for: " + modelUuid.toString()); |
| 398 | return; |
| 399 | } |
| 400 | modelInfo.captureAudioMs = captureTimeoutMs; |
| 401 | Log.i(TAG, "Set " + modelInfo.name + " capture audio timeout to " + |
| 402 | captureTimeoutMs + "ms"); |
| 403 | } |
| 404 | |
| 405 | public synchronized void setCaptureAudio(UUID modelUuid, boolean captureAudio) { |
| 406 | ModelInfo modelInfo = mModelInfoMap.get(modelUuid); |
| 407 | if (modelInfo == null) { |
| 408 | postError("Could not find model for: " + modelUuid.toString()); |
| 409 | return; |
| 410 | } |
| 411 | modelInfo.captureAudio = captureAudio; |
| 412 | Log.i(TAG, "Set " + modelInfo.name + " capture audio to " + captureAudio); |
| 413 | } |
| 414 | |
| 415 | public synchronized boolean hasMicrophonePermission() { |
| 416 | return getBaseContext().checkSelfPermission(Manifest.permission.RECORD_AUDIO) |
| 417 | == PackageManager.PERMISSION_GRANTED; |
| 418 | } |
| 419 | |
| 420 | public synchronized boolean modelHasTriggerAudio(UUID modelUuid) { |
| 421 | ModelInfo modelInfo = mModelInfoMap.get(modelUuid); |
| 422 | return modelInfo != null && modelInfo.triggerAudioPlayer != null; |
| 423 | } |
| 424 | |
| 425 | public synchronized boolean modelWillCaptureTriggerAudio(UUID modelUuid) { |
| 426 | ModelInfo modelInfo = mModelInfoMap.get(modelUuid); |
| 427 | return modelInfo != null && modelInfo.captureAudio; |
| 428 | } |
| 429 | |
| 430 | public synchronized boolean modelHasCapturedAudio(UUID modelUuid) { |
| 431 | ModelInfo modelInfo = mModelInfoMap.get(modelUuid); |
| 432 | return modelInfo != null && modelInfo.captureAudioTrack != null; |
| 433 | } |
| 434 | |
| 435 | private void loadModelsInDataDir() { |
| 436 | // Load all the models in the data dir. |
| 437 | boolean loadedModel = false; |
| 438 | for (File file : getFilesDir().listFiles()) { |
| 439 | // Find meta-data in .properties files, ignore everything else. |
| 440 | if (!file.getName().endsWith(".properties")) { |
| 441 | continue; |
| 442 | } |
mike dooley | cabbb11 | 2018-04-17 17:19:04 +0200 | [diff] [blame] | 443 | |
| 444 | try (FileInputStream in = new FileInputStream(file)) { |
Chris Thornton | dfa7c3b | 2016-06-30 22:05:51 -0700 | [diff] [blame] | 445 | Properties properties = new Properties(); |
mike dooley | cabbb11 | 2018-04-17 17:19:04 +0200 | [diff] [blame] | 446 | properties.load(in); |
Chris Thornton | dfa7c3b | 2016-06-30 22:05:51 -0700 | [diff] [blame] | 447 | createModelInfo(properties); |
| 448 | loadedModel = true; |
| 449 | } catch (Exception e) { |
| 450 | Log.e(TAG, "Failed to load properties file " + file.getName()); |
| 451 | } |
| 452 | } |
| 453 | |
| 454 | // Create a few dummy models if we didn't load anything. |
| 455 | if (!loadedModel) { |
| 456 | Properties dummyModelProperties = new Properties(); |
| 457 | for (String name : new String[]{"1", "2", "3"}) { |
| 458 | dummyModelProperties.setProperty("name", "Model " + name); |
| 459 | createModelInfo(dummyModelProperties); |
| 460 | } |
| 461 | } |
| 462 | } |
| 463 | |
| 464 | /** Parses a Properties collection to generate a sound model. |
| 465 | * |
| 466 | * Missing keys are filled in with default/random values. |
| 467 | * @param properties Has the required 'name' property, but the remaining 'modelUuid', |
| 468 | * 'vendorUuid', 'triggerAudio', and 'dataFile' optional properties. |
| 469 | * |
| 470 | */ |
| 471 | private synchronized void createModelInfo(Properties properties) { |
| 472 | try { |
| 473 | ModelInfo modelInfo = new ModelInfo(); |
| 474 | |
| 475 | if (!properties.containsKey("name")) { |
| 476 | throw new RuntimeException("must have a 'name' property"); |
| 477 | } |
| 478 | modelInfo.name = properties.getProperty("name"); |
| 479 | |
| 480 | if (properties.containsKey("modelUuid")) { |
| 481 | modelInfo.modelUuid = UUID.fromString(properties.getProperty("modelUuid")); |
| 482 | } else { |
| 483 | modelInfo.modelUuid = UUID.randomUUID(); |
| 484 | } |
| 485 | |
| 486 | if (properties.containsKey("vendorUuid")) { |
| 487 | modelInfo.vendorUuid = UUID.fromString(properties.getProperty("vendorUuid")); |
| 488 | } else { |
| 489 | modelInfo.vendorUuid = UUID.randomUUID(); |
| 490 | } |
| 491 | |
| 492 | if (properties.containsKey("triggerAudio")) { |
| 493 | modelInfo.triggerAudioPlayer = MediaPlayer.create(this, Uri.parse( |
| 494 | getFilesDir().getPath() + "/" + properties.getProperty("triggerAudio"))); |
| 495 | if (modelInfo.triggerAudioPlayer.getDuration() == 0) { |
| 496 | modelInfo.triggerAudioPlayer.release(); |
| 497 | modelInfo.triggerAudioPlayer = null; |
| 498 | } |
| 499 | } |
| 500 | |
| 501 | if (properties.containsKey("dataFile")) { |
| 502 | File modelDataFile = new File( |
Nicholas Ambur | 1e43767 | 2020-03-24 17:09:58 -0700 | [diff] [blame] | 503 | getFilesDir().getPath() + "/" |
| 504 | + properties.getProperty("dataFile")); |
Chris Thornton | dfa7c3b | 2016-06-30 22:05:51 -0700 | [diff] [blame] | 505 | modelInfo.modelData = new byte[(int) modelDataFile.length()]; |
| 506 | FileInputStream input = new FileInputStream(modelDataFile); |
| 507 | input.read(modelInfo.modelData, 0, modelInfo.modelData.length); |
| 508 | } else { |
| 509 | modelInfo.modelData = new byte[1024]; |
| 510 | mRandom.nextBytes(modelInfo.modelData); |
| 511 | } |
| 512 | |
| 513 | modelInfo.captureAudioMs = Integer.parseInt((String) properties.getOrDefault( |
| 514 | "captureAudioDurationMs", "5000")); |
| 515 | |
| 516 | // TODO: Add property support for keyphrase models when they're exposed by the |
| 517 | // service. |
| 518 | |
| 519 | // Update our maps containing the button -> id and id -> modelInfo. |
| 520 | mModelInfoMap.put(modelInfo.modelUuid, modelInfo); |
| 521 | if (mUserActivity != null) { |
| 522 | mUserActivity.addModel(modelInfo.modelUuid, modelInfo.name); |
| 523 | mUserActivity.setModelState(modelInfo.modelUuid, modelInfo.state); |
| 524 | } |
| 525 | } catch (IOException e) { |
| 526 | Log.e(TAG, "Error parsing properties for " + properties.getProperty("name"), e); |
| 527 | } |
| 528 | } |
| 529 | |
| 530 | private class CaptureAudioRecorder implements Runnable { |
| 531 | private final ModelInfo mModelInfo; |
| 532 | private final SoundTriggerDetector.EventPayload mEvent; |
| 533 | |
| 534 | public CaptureAudioRecorder(ModelInfo modelInfo, SoundTriggerDetector.EventPayload event) { |
| 535 | mModelInfo = modelInfo; |
| 536 | mEvent = event; |
| 537 | } |
| 538 | |
| 539 | @Override |
| 540 | public void run() { |
| 541 | AudioFormat format = mEvent.getCaptureAudioFormat(); |
| 542 | if (format == null) { |
| 543 | postErrorToast("No audio format in recognition event."); |
| 544 | return; |
| 545 | } |
| 546 | |
| 547 | AudioRecord audioRecord = null; |
| 548 | AudioTrack playbackTrack = null; |
| 549 | try { |
| 550 | // Inform the audio flinger that we really do want the stream from the soundtrigger. |
| 551 | AudioAttributes.Builder attributesBuilder = new AudioAttributes.Builder(); |
| 552 | attributesBuilder.setInternalCapturePreset(1999); |
| 553 | AudioAttributes attributes = attributesBuilder.build(); |
| 554 | |
| 555 | // Make sure we understand this kind of playback so we know how many bytes to read. |
| 556 | String encoding; |
| 557 | int bytesPerSample; |
| 558 | switch (format.getEncoding()) { |
| 559 | case AudioFormat.ENCODING_PCM_8BIT: |
| 560 | encoding = "8bit"; |
| 561 | bytesPerSample = 1; |
| 562 | break; |
| 563 | case AudioFormat.ENCODING_PCM_16BIT: |
| 564 | encoding = "16bit"; |
| 565 | bytesPerSample = 2; |
| 566 | break; |
| 567 | case AudioFormat.ENCODING_PCM_FLOAT: |
| 568 | encoding = "float"; |
| 569 | bytesPerSample = 4; |
| 570 | break; |
| 571 | default: |
| 572 | throw new RuntimeException("Unhandled audio format in event"); |
| 573 | } |
| 574 | |
| 575 | int bytesRequired = format.getSampleRate() * format.getChannelCount() * |
| 576 | bytesPerSample * mModelInfo.captureAudioMs / 1000; |
| 577 | int minBufferSize = AudioRecord.getMinBufferSize( |
| 578 | format.getSampleRate(), format.getChannelMask(), format.getEncoding()); |
| 579 | if (minBufferSize > bytesRequired) { |
| 580 | bytesRequired = minBufferSize; |
| 581 | } |
| 582 | |
| 583 | // Make an AudioTrack so we can play the data back out after it's finished |
| 584 | // recording. |
| 585 | try { |
| 586 | int channelConfig = AudioFormat.CHANNEL_OUT_MONO; |
| 587 | if (format.getChannelCount() == 2) { |
| 588 | channelConfig = AudioFormat.CHANNEL_OUT_STEREO; |
| 589 | } else if (format.getChannelCount() >= 3) { |
| 590 | throw new RuntimeException( |
| 591 | "Too many channels in captured audio for playback"); |
| 592 | } |
| 593 | |
| 594 | playbackTrack = new AudioTrack(AudioManager.STREAM_MUSIC, |
| 595 | format.getSampleRate(), channelConfig, format.getEncoding(), |
| 596 | bytesRequired, AudioTrack.MODE_STATIC); |
| 597 | } catch (Exception e) { |
| 598 | Log.e(TAG, "Exception creating playback track", e); |
| 599 | postErrorToast("Failed to create playback track: " + e.getMessage()); |
| 600 | } |
| 601 | |
| 602 | audioRecord = new AudioRecord(attributes, format, bytesRequired, |
| 603 | mEvent.getCaptureSession()); |
| 604 | |
| 605 | byte[] buffer = new byte[bytesRequired]; |
| 606 | |
| 607 | // Create a file so we can save the output data there for analysis later. |
| 608 | FileOutputStream fos = null; |
| 609 | try { |
| 610 | fos = new FileOutputStream( new File( |
Nicholas Ambur | 1e43767 | 2020-03-24 17:09:58 -0700 | [diff] [blame] | 611 | getFilesDir() + File.separator |
| 612 | + mModelInfo.name.replace(' ', '_') |
| 613 | + "_capture_" + format.getChannelCount() + "ch_" |
| 614 | + format.getSampleRate() + "hz_" + encoding + ".pcm")); |
Chris Thornton | dfa7c3b | 2016-06-30 22:05:51 -0700 | [diff] [blame] | 615 | } catch (IOException e) { |
| 616 | Log.e(TAG, "Failed to open output for saving PCM data", e); |
Nicholas Ambur | 1e43767 | 2020-03-24 17:09:58 -0700 | [diff] [blame] | 617 | postErrorToast("Failed to open output for saving PCM data: " |
| 618 | + e.getMessage()); |
Chris Thornton | dfa7c3b | 2016-06-30 22:05:51 -0700 | [diff] [blame] | 619 | } |
| 620 | |
| 621 | // Inform the user we're recording. |
| 622 | setModelState(mModelInfo, "Recording"); |
| 623 | audioRecord.startRecording(); |
| 624 | while (bytesRequired > 0) { |
| 625 | int bytesRead = audioRecord.read(buffer, 0, buffer.length); |
| 626 | if (bytesRead == -1) { |
| 627 | break; |
| 628 | } |
| 629 | if (fos != null) { |
| 630 | fos.write(buffer, 0, bytesRead); |
| 631 | } |
| 632 | if (playbackTrack != null) { |
| 633 | playbackTrack.write(buffer, 0, bytesRead); |
| 634 | } |
| 635 | bytesRequired -= bytesRead; |
| 636 | } |
| 637 | audioRecord.stop(); |
| 638 | } catch (Exception e) { |
| 639 | Log.e(TAG, "Error recording trigger audio", e); |
| 640 | postErrorToast("Error recording trigger audio: " + e.getMessage()); |
| 641 | } finally { |
| 642 | if (audioRecord != null) { |
| 643 | audioRecord.release(); |
| 644 | } |
| 645 | synchronized (SoundTriggerTestService.this) { |
| 646 | if (mModelInfo.captureAudioTrack != null) { |
| 647 | mModelInfo.captureAudioTrack.release(); |
| 648 | } |
| 649 | mModelInfo.captureAudioTrack = playbackTrack; |
| 650 | } |
| 651 | setModelState(mModelInfo, "Recording finished"); |
| 652 | } |
| 653 | } |
| 654 | } |
| 655 | |
| 656 | // Implementation of SoundTriggerDetector.Callback. |
| 657 | private class DetectorCallback extends SoundTriggerDetector.Callback { |
| 658 | private final ModelInfo mModelInfo; |
| 659 | |
| 660 | public DetectorCallback(ModelInfo modelInfo) { |
| 661 | mModelInfo = modelInfo; |
| 662 | } |
| 663 | |
| 664 | public void onAvailabilityChanged(int status) { |
Chris Thornton | 2d2ba9d0 | 2016-07-12 15:05:18 -0700 | [diff] [blame] | 665 | postMessage(mModelInfo.name + " availability changed to: " + status); |
Chris Thornton | dfa7c3b | 2016-06-30 22:05:51 -0700 | [diff] [blame] | 666 | } |
| 667 | |
| 668 | public void onDetected(SoundTriggerDetector.EventPayload event) { |
Chris Thornton | 2d2ba9d0 | 2016-07-12 15:05:18 -0700 | [diff] [blame] | 669 | postMessage(mModelInfo.name + " onDetected(): " + eventPayloadToString(event)); |
Chris Thornton | dfa7c3b | 2016-06-30 22:05:51 -0700 | [diff] [blame] | 670 | synchronized (SoundTriggerTestService.this) { |
| 671 | if (mUserActivity != null) { |
| 672 | mUserActivity.handleDetection(mModelInfo.modelUuid); |
| 673 | } |
| 674 | if (mModelInfo.captureAudio) { |
| 675 | new Thread(new CaptureAudioRecorder(mModelInfo, event)).start(); |
| 676 | } |
| 677 | } |
| 678 | } |
| 679 | |
| 680 | public void onError() { |
Chris Thornton | 2d2ba9d0 | 2016-07-12 15:05:18 -0700 | [diff] [blame] | 681 | postMessage(mModelInfo.name + " onError()"); |
Chris Thornton | dfa7c3b | 2016-06-30 22:05:51 -0700 | [diff] [blame] | 682 | setModelState(mModelInfo, "Error"); |
| 683 | } |
| 684 | |
| 685 | public void onRecognitionPaused() { |
| 686 | postMessage(mModelInfo.name + " onRecognitionPaused()"); |
| 687 | setModelState(mModelInfo, "Paused"); |
| 688 | } |
| 689 | |
| 690 | public void onRecognitionResumed() { |
Chris Thornton | 2d2ba9d0 | 2016-07-12 15:05:18 -0700 | [diff] [blame] | 691 | postMessage(mModelInfo.name + " onRecognitionResumed()"); |
Chris Thornton | dfa7c3b | 2016-06-30 22:05:51 -0700 | [diff] [blame] | 692 | setModelState(mModelInfo, "Resumed"); |
| 693 | } |
| 694 | } |
| 695 | |
| 696 | private String eventPayloadToString(SoundTriggerDetector.EventPayload event) { |
| 697 | String result = "EventPayload("; |
| 698 | AudioFormat format = event.getCaptureAudioFormat(); |
| 699 | result = result + "AudioFormat: " + ((format == null) ? "null" : format.toString()); |
| 700 | byte[] triggerAudio = event.getTriggerAudio(); |
Nicholas Ambur | 1e43767 | 2020-03-24 17:09:58 -0700 | [diff] [blame] | 701 | result = result + ", TriggerAudio: " |
| 702 | + (triggerAudio == null ? "null" : triggerAudio.length); |
Chris Thornton | 7554ff0 | 2017-04-30 19:46:39 -0700 | [diff] [blame] | 703 | byte[] data = event.getData(); |
| 704 | result = result + ", Data: " + (data == null ? "null" : data.length); |
| 705 | if (data != null) { |
| 706 | try { |
| 707 | String decodedData = new String(data, "UTF-8"); |
| 708 | if (decodedData.chars().allMatch(c -> (c >= 32 && c < 128) || c == 0)) { |
| 709 | result = result + ", Decoded Data: '" + decodedData + "'"; |
| 710 | } |
| 711 | } catch (Exception e) { |
| 712 | Log.e(TAG, "Failed to decode data"); |
| 713 | } |
| 714 | } |
| 715 | result = result + ", CaptureSession: " + event.getCaptureSession(); |
Chris Thornton | dfa7c3b | 2016-06-30 22:05:51 -0700 | [diff] [blame] | 716 | result += " )"; |
| 717 | return result; |
| 718 | } |
| 719 | |
| 720 | private void postMessage(String msg) { |
| 721 | showMessage(msg, Log.INFO, false); |
| 722 | } |
| 723 | |
| 724 | private void postError(String msg) { |
| 725 | showMessage(msg, Log.ERROR, false); |
| 726 | } |
| 727 | |
| 728 | private void postToast(String msg) { |
| 729 | showMessage(msg, Log.INFO, true); |
| 730 | } |
| 731 | |
| 732 | private void postErrorToast(String msg) { |
| 733 | showMessage(msg, Log.ERROR, true); |
| 734 | } |
| 735 | |
| 736 | /** Logs the message at the specified level, then forwards it to the activity if present. */ |
| 737 | private synchronized void showMessage(String msg, int logLevel, boolean showToast) { |
| 738 | Log.println(logLevel, TAG, msg); |
| 739 | if (mUserActivity != null) { |
| 740 | mUserActivity.showMessage(msg, showToast); |
| 741 | } |
| 742 | } |
| 743 | |
| 744 | private synchronized void setModelState(ModelInfo modelInfo, String state) { |
| 745 | modelInfo.state = state; |
| 746 | if (mUserActivity != null) { |
| 747 | mUserActivity.setModelState(modelInfo.modelUuid, modelInfo.state); |
| 748 | } |
| 749 | } |
| 750 | } |