Chris Thornton | dfa7c3b | 2016-06-30 22:05:51 -0700 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (C) 2014 The Android Open Source Project |
| 3 | * |
| 4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | * you may not use this file except in compliance with the License. |
| 6 | * You may obtain a copy of the License at |
| 7 | * |
| 8 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | * |
| 10 | * Unless required by applicable law or agreed to in writing, software |
| 11 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | * See the License for the specific language governing permissions and |
| 14 | * limitations under the License. |
| 15 | */ |
| 16 | |
| 17 | package com.android.test.soundtrigger; |
| 18 | |
| 19 | import android.Manifest; |
| 20 | import android.app.Service; |
| 21 | import android.content.BroadcastReceiver; |
| 22 | import android.content.Context; |
| 23 | import android.content.Intent; |
| 24 | import android.content.IntentFilter; |
| 25 | import android.content.pm.PackageManager; |
| 26 | import android.hardware.soundtrigger.SoundTrigger.GenericSoundModel; |
| 27 | import android.media.AudioAttributes; |
| 28 | import android.media.AudioFormat; |
| 29 | import android.media.AudioManager; |
| 30 | import android.media.AudioRecord; |
| 31 | import android.media.AudioTrack; |
| 32 | import android.media.MediaPlayer; |
| 33 | import android.media.soundtrigger.SoundTriggerDetector; |
| 34 | import android.net.Uri; |
| 35 | import android.os.Binder; |
| 36 | import android.os.IBinder; |
| 37 | import android.util.Log; |
| 38 | |
| 39 | import java.io.File; |
| 40 | import java.io.FileInputStream; |
| 41 | import java.io.FileOutputStream; |
| 42 | import java.io.IOException; |
| 43 | import java.util.HashMap; |
| 44 | import java.util.Map; |
| 45 | import java.util.Properties; |
| 46 | import java.util.Random; |
| 47 | import java.util.UUID; |
| 48 | |
| 49 | public class SoundTriggerTestService extends Service { |
| 50 | private static final String TAG = "SoundTriggerTestSrv"; |
| 51 | private static final String INTENT_ACTION = "com.android.intent.action.MANAGE_SOUND_TRIGGER"; |
| 52 | |
| 53 | // Binder given to clients. |
| 54 | private final IBinder mBinder; |
| 55 | private final Map<UUID, ModelInfo> mModelInfoMap; |
| 56 | private SoundTriggerUtil mSoundTriggerUtil; |
| 57 | private Random mRandom; |
| 58 | private UserActivity mUserActivity; |
| 59 | |
| 60 | public interface UserActivity { |
| 61 | void addModel(UUID modelUuid, String state); |
| 62 | void setModelState(UUID modelUuid, String state); |
| 63 | void showMessage(String msg, boolean showToast); |
| 64 | void handleDetection(UUID modelUuid); |
| 65 | } |
| 66 | |
| 67 | public SoundTriggerTestService() { |
| 68 | super(); |
| 69 | mRandom = new Random(); |
| 70 | mModelInfoMap = new HashMap(); |
| 71 | mBinder = new SoundTriggerTestBinder(); |
| 72 | } |
| 73 | |
| 74 | @Override |
| 75 | public synchronized int onStartCommand(Intent intent, int flags, int startId) { |
| 76 | if (mModelInfoMap.isEmpty()) { |
| 77 | mSoundTriggerUtil = new SoundTriggerUtil(this); |
| 78 | loadModelsInDataDir(); |
| 79 | } |
| 80 | |
| 81 | // If we get killed, after returning from here, restart |
| 82 | return START_STICKY; |
| 83 | } |
| 84 | |
| 85 | @Override |
| 86 | public void onCreate() { |
| 87 | super.onCreate(); |
| 88 | IntentFilter filter = new IntentFilter(); |
| 89 | filter.addAction(INTENT_ACTION); |
| 90 | registerReceiver(mBroadcastReceiver, filter); |
| 91 | |
| 92 | // Make sure the data directory exists, and we're the owner of it. |
| 93 | try { |
| 94 | getFilesDir().mkdir(); |
| 95 | } catch (Exception e) { |
| 96 | // Don't care - we either made it, or it already exists. |
| 97 | } |
| 98 | } |
| 99 | |
| 100 | @Override |
| 101 | public void onDestroy() { |
| 102 | super.onDestroy(); |
Chris Thornton | e492e49 | 2017-03-22 13:50:04 -0700 | [diff] [blame^] | 103 | stopAllRecognitionsAndUnload(); |
Chris Thornton | dfa7c3b | 2016-06-30 22:05:51 -0700 | [diff] [blame] | 104 | unregisterReceiver(mBroadcastReceiver); |
| 105 | } |
| 106 | |
| 107 | private final BroadcastReceiver mBroadcastReceiver = new BroadcastReceiver() { |
| 108 | @Override |
| 109 | public void onReceive(Context context, Intent intent) { |
| 110 | if (intent != null && INTENT_ACTION.equals(intent.getAction())) { |
| 111 | String command = intent.getStringExtra("command"); |
| 112 | if (command == null) { |
| 113 | Log.e(TAG, "No 'command' specified in " + INTENT_ACTION); |
| 114 | } else { |
| 115 | try { |
| 116 | if (command.equals("load")) { |
| 117 | loadModel(getModelUuidFromIntent(intent)); |
| 118 | } else if (command.equals("unload")) { |
| 119 | unloadModel(getModelUuidFromIntent(intent)); |
| 120 | } else if (command.equals("start")) { |
| 121 | startRecognition(getModelUuidFromIntent(intent)); |
| 122 | } else if (command.equals("stop")) { |
| 123 | stopRecognition(getModelUuidFromIntent(intent)); |
| 124 | } else if (command.equals("play_trigger")) { |
| 125 | playTriggerAudio(getModelUuidFromIntent(intent)); |
| 126 | } else if (command.equals("play_captured")) { |
| 127 | playCapturedAudio(getModelUuidFromIntent(intent)); |
| 128 | } else if (command.equals("set_capture")) { |
| 129 | setCaptureAudio(getModelUuidFromIntent(intent), |
| 130 | intent.getBooleanExtra("enabled", true)); |
| 131 | } else if (command.equals("set_capture_timeout")) { |
| 132 | setCaptureAudioTimeout(getModelUuidFromIntent(intent), |
| 133 | intent.getIntExtra("timeout", 5000)); |
| 134 | } else { |
| 135 | Log.e(TAG, "Unknown command '" + command + "'"); |
| 136 | } |
| 137 | } catch (Exception e) { |
| 138 | Log.e(TAG, "Failed to process " + command, e); |
| 139 | } |
| 140 | } |
| 141 | } |
| 142 | } |
| 143 | }; |
| 144 | |
| 145 | private UUID getModelUuidFromIntent(Intent intent) { |
| 146 | // First, see if the specified the UUID straight up. |
| 147 | String value = intent.getStringExtra("modelUuid"); |
| 148 | if (value != null) { |
| 149 | return UUID.fromString(value); |
| 150 | } |
| 151 | |
| 152 | // If they specified a name, use that to iterate through the map of models and find it. |
| 153 | value = intent.getStringExtra("name"); |
| 154 | if (value != null) { |
| 155 | for (ModelInfo modelInfo : mModelInfoMap.values()) { |
| 156 | if (value.equals(modelInfo.name)) { |
| 157 | return modelInfo.modelUuid; |
| 158 | } |
| 159 | } |
| 160 | Log.e(TAG, "Failed to find a matching model with name '" + value + "'"); |
| 161 | } |
| 162 | |
| 163 | // We couldn't figure out what they were asking for. |
| 164 | throw new RuntimeException("Failed to get model from intent - specify either " + |
| 165 | "'modelUuid' or 'name'"); |
| 166 | } |
| 167 | |
| 168 | /** |
| 169 | * Will be called when the service is killed (through swipe aways, not if we're force killed). |
| 170 | */ |
| 171 | @Override |
| 172 | public void onTaskRemoved(Intent rootIntent) { |
| 173 | super.onTaskRemoved(rootIntent); |
Chris Thornton | e492e49 | 2017-03-22 13:50:04 -0700 | [diff] [blame^] | 174 | stopAllRecognitionsAndUnload(); |
Chris Thornton | dfa7c3b | 2016-06-30 22:05:51 -0700 | [diff] [blame] | 175 | stopSelf(); |
| 176 | } |
| 177 | |
| 178 | @Override |
| 179 | public synchronized IBinder onBind(Intent intent) { |
| 180 | return mBinder; |
| 181 | } |
| 182 | |
| 183 | public class SoundTriggerTestBinder extends Binder { |
| 184 | SoundTriggerTestService getService() { |
| 185 | // Return instance of our parent so clients can call public methods. |
| 186 | return SoundTriggerTestService.this; |
| 187 | } |
| 188 | } |
| 189 | |
| 190 | public synchronized void setUserActivity(UserActivity activity) { |
| 191 | mUserActivity = activity; |
| 192 | if (mUserActivity != null) { |
| 193 | for (Map.Entry<UUID, ModelInfo> entry : mModelInfoMap.entrySet()) { |
| 194 | mUserActivity.addModel(entry.getKey(), entry.getValue().name); |
| 195 | mUserActivity.setModelState(entry.getKey(), entry.getValue().state); |
| 196 | } |
| 197 | } |
| 198 | } |
| 199 | |
Chris Thornton | e492e49 | 2017-03-22 13:50:04 -0700 | [diff] [blame^] | 200 | private synchronized void stopAllRecognitionsAndUnload() { |
| 201 | Log.e(TAG, "Stop all recognitions"); |
Chris Thornton | dfa7c3b | 2016-06-30 22:05:51 -0700 | [diff] [blame] | 202 | for (ModelInfo modelInfo : mModelInfoMap.values()) { |
Chris Thornton | e492e49 | 2017-03-22 13:50:04 -0700 | [diff] [blame^] | 203 | Log.e(TAG, "Loop " + modelInfo.modelUuid); |
Chris Thornton | dfa7c3b | 2016-06-30 22:05:51 -0700 | [diff] [blame] | 204 | if (modelInfo.detector != null) { |
| 205 | Log.i(TAG, "Stopping recognition for " + modelInfo.name); |
| 206 | try { |
| 207 | modelInfo.detector.stopRecognition(); |
| 208 | } catch (Exception e) { |
| 209 | Log.e(TAG, "Failed to stop recognition", e); |
| 210 | } |
Chris Thornton | e492e49 | 2017-03-22 13:50:04 -0700 | [diff] [blame^] | 211 | try { |
| 212 | mSoundTriggerUtil.deleteSoundModel(modelInfo.modelUuid); |
| 213 | modelInfo.detector = null; |
| 214 | } catch (Exception e) { |
| 215 | Log.e(TAG, "Failed to unload sound model", e); |
| 216 | } |
Chris Thornton | dfa7c3b | 2016-06-30 22:05:51 -0700 | [diff] [blame] | 217 | } |
| 218 | } |
| 219 | } |
| 220 | |
| 221 | // Helper struct for holding information about a model. |
| 222 | public static class ModelInfo { |
| 223 | public String name; |
| 224 | public String state; |
| 225 | public UUID modelUuid; |
| 226 | public UUID vendorUuid; |
| 227 | public MediaPlayer triggerAudioPlayer; |
| 228 | public SoundTriggerDetector detector; |
| 229 | public byte modelData[]; |
| 230 | public boolean captureAudio; |
| 231 | public int captureAudioMs; |
| 232 | public AudioTrack captureAudioTrack; |
| 233 | } |
| 234 | |
| 235 | private GenericSoundModel createNewSoundModel(ModelInfo modelInfo) { |
| 236 | return new GenericSoundModel(modelInfo.modelUuid, modelInfo.vendorUuid, |
| 237 | modelInfo.modelData); |
| 238 | } |
| 239 | |
| 240 | public synchronized void loadModel(UUID modelUuid) { |
| 241 | ModelInfo modelInfo = mModelInfoMap.get(modelUuid); |
| 242 | if (modelInfo == null) { |
| 243 | postError("Could not find model for: " + modelUuid.toString()); |
| 244 | return; |
| 245 | } |
| 246 | |
| 247 | postMessage("Loading model: " + modelInfo.name); |
| 248 | |
| 249 | GenericSoundModel soundModel = createNewSoundModel(modelInfo); |
| 250 | |
| 251 | boolean status = mSoundTriggerUtil.addOrUpdateSoundModel(soundModel); |
| 252 | if (status) { |
| 253 | postToast("Successfully loaded " + modelInfo.name + ", UUID=" + soundModel.uuid); |
| 254 | setModelState(modelInfo, "Loaded"); |
| 255 | } else { |
| 256 | postErrorToast("Failed to load " + modelInfo.name + ", UUID=" + soundModel.uuid + "!"); |
| 257 | setModelState(modelInfo, "Failed to load"); |
| 258 | } |
| 259 | } |
| 260 | |
| 261 | public synchronized void unloadModel(UUID modelUuid) { |
| 262 | ModelInfo modelInfo = mModelInfoMap.get(modelUuid); |
| 263 | if (modelInfo == null) { |
| 264 | postError("Could not find model for: " + modelUuid.toString()); |
| 265 | return; |
| 266 | } |
| 267 | |
| 268 | postMessage("Unloading model: " + modelInfo.name); |
| 269 | |
| 270 | GenericSoundModel soundModel = mSoundTriggerUtil.getSoundModel(modelUuid); |
| 271 | if (soundModel == null) { |
| 272 | postErrorToast("Sound model not found for " + modelInfo.name + "!"); |
| 273 | return; |
| 274 | } |
| 275 | modelInfo.detector = null; |
| 276 | boolean status = mSoundTriggerUtil.deleteSoundModel(modelUuid); |
| 277 | if (status) { |
| 278 | postToast("Successfully unloaded " + modelInfo.name + ", UUID=" + soundModel.uuid); |
| 279 | setModelState(modelInfo, "Unloaded"); |
| 280 | } else { |
| 281 | postErrorToast("Failed to unload " + |
| 282 | modelInfo.name + ", UUID=" + soundModel.uuid + "!"); |
| 283 | setModelState(modelInfo, "Failed to unload"); |
| 284 | } |
| 285 | } |
| 286 | |
| 287 | public synchronized void reloadModel(UUID modelUuid) { |
| 288 | ModelInfo modelInfo = mModelInfoMap.get(modelUuid); |
| 289 | if (modelInfo == null) { |
| 290 | postError("Could not find model for: " + modelUuid.toString()); |
| 291 | return; |
| 292 | } |
| 293 | postMessage("Reloading model: " + modelInfo.name); |
| 294 | GenericSoundModel soundModel = mSoundTriggerUtil.getSoundModel(modelUuid); |
| 295 | if (soundModel == null) { |
| 296 | postErrorToast("Sound model not found for " + modelInfo.name + "!"); |
| 297 | return; |
| 298 | } |
| 299 | GenericSoundModel updated = createNewSoundModel(modelInfo); |
| 300 | boolean status = mSoundTriggerUtil.addOrUpdateSoundModel(updated); |
| 301 | if (status) { |
| 302 | postToast("Successfully reloaded " + modelInfo.name + ", UUID=" + modelInfo.modelUuid); |
| 303 | setModelState(modelInfo, "Reloaded"); |
| 304 | } else { |
| 305 | postErrorToast("Failed to reload " |
| 306 | + modelInfo.name + ", UUID=" + modelInfo.modelUuid + "!"); |
| 307 | setModelState(modelInfo, "Failed to reload"); |
| 308 | } |
| 309 | } |
| 310 | |
| 311 | public synchronized void startRecognition(UUID modelUuid) { |
| 312 | ModelInfo modelInfo = mModelInfoMap.get(modelUuid); |
| 313 | if (modelInfo == null) { |
| 314 | postError("Could not find model for: " + modelUuid.toString()); |
| 315 | return; |
| 316 | } |
| 317 | |
| 318 | if (modelInfo.detector == null) { |
| 319 | postMessage("Creating SoundTriggerDetector for " + modelInfo.name); |
| 320 | modelInfo.detector = mSoundTriggerUtil.createSoundTriggerDetector( |
| 321 | modelUuid, new DetectorCallback(modelInfo)); |
| 322 | } |
| 323 | |
| 324 | postMessage("Starting recognition for " + modelInfo.name + ", UUID=" + modelInfo.modelUuid); |
| 325 | if (modelInfo.detector.startRecognition(modelInfo.captureAudio ? |
| 326 | SoundTriggerDetector.RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO : |
| 327 | SoundTriggerDetector.RECOGNITION_FLAG_ALLOW_MULTIPLE_TRIGGERS)) { |
| 328 | setModelState(modelInfo, "Started"); |
| 329 | } else { |
| 330 | postErrorToast("Fast failure attempting to start recognition for " + |
| 331 | modelInfo.name + ", UUID=" + modelInfo.modelUuid); |
| 332 | setModelState(modelInfo, "Failed to start"); |
| 333 | } |
| 334 | } |
| 335 | |
| 336 | public synchronized void stopRecognition(UUID modelUuid) { |
| 337 | ModelInfo modelInfo = mModelInfoMap.get(modelUuid); |
| 338 | if (modelInfo == null) { |
| 339 | postError("Could not find model for: " + modelUuid.toString()); |
| 340 | return; |
| 341 | } |
| 342 | |
| 343 | if (modelInfo.detector == null) { |
| 344 | postErrorToast("Stop called on null detector for " + |
| 345 | modelInfo.name + ", UUID=" + modelInfo.modelUuid); |
| 346 | return; |
| 347 | } |
| 348 | postMessage("Triggering stop recognition for " + |
| 349 | modelInfo.name + ", UUID=" + modelInfo.modelUuid); |
| 350 | if (modelInfo.detector.stopRecognition()) { |
| 351 | setModelState(modelInfo, "Stopped"); |
| 352 | } else { |
| 353 | postErrorToast("Fast failure attempting to stop recognition for " + |
| 354 | modelInfo.name + ", UUID=" + modelInfo.modelUuid); |
| 355 | setModelState(modelInfo, "Failed to stop"); |
| 356 | } |
| 357 | } |
| 358 | |
| 359 | public synchronized void playTriggerAudio(UUID modelUuid) { |
| 360 | ModelInfo modelInfo = mModelInfoMap.get(modelUuid); |
| 361 | if (modelInfo == null) { |
| 362 | postError("Could not find model for: " + modelUuid.toString()); |
| 363 | return; |
| 364 | } |
| 365 | if (modelInfo.triggerAudioPlayer != null) { |
| 366 | postMessage("Playing trigger audio for " + modelInfo.name); |
| 367 | modelInfo.triggerAudioPlayer.start(); |
| 368 | } else { |
| 369 | postMessage("No trigger audio for " + modelInfo.name); |
| 370 | } |
| 371 | } |
| 372 | |
| 373 | public synchronized void playCapturedAudio(UUID modelUuid) { |
| 374 | ModelInfo modelInfo = mModelInfoMap.get(modelUuid); |
| 375 | if (modelInfo == null) { |
| 376 | postError("Could not find model for: " + modelUuid.toString()); |
| 377 | return; |
| 378 | } |
| 379 | if (modelInfo.captureAudioTrack != null) { |
| 380 | postMessage("Playing captured audio for " + modelInfo.name); |
| 381 | modelInfo.captureAudioTrack.stop(); |
| 382 | modelInfo.captureAudioTrack.reloadStaticData(); |
| 383 | modelInfo.captureAudioTrack.play(); |
| 384 | } else { |
| 385 | postMessage("No captured audio for " + modelInfo.name); |
| 386 | } |
| 387 | } |
| 388 | |
| 389 | public synchronized void setCaptureAudioTimeout(UUID modelUuid, int captureTimeoutMs) { |
| 390 | ModelInfo modelInfo = mModelInfoMap.get(modelUuid); |
| 391 | if (modelInfo == null) { |
| 392 | postError("Could not find model for: " + modelUuid.toString()); |
| 393 | return; |
| 394 | } |
| 395 | modelInfo.captureAudioMs = captureTimeoutMs; |
| 396 | Log.i(TAG, "Set " + modelInfo.name + " capture audio timeout to " + |
| 397 | captureTimeoutMs + "ms"); |
| 398 | } |
| 399 | |
| 400 | public synchronized void setCaptureAudio(UUID modelUuid, boolean captureAudio) { |
| 401 | ModelInfo modelInfo = mModelInfoMap.get(modelUuid); |
| 402 | if (modelInfo == null) { |
| 403 | postError("Could not find model for: " + modelUuid.toString()); |
| 404 | return; |
| 405 | } |
| 406 | modelInfo.captureAudio = captureAudio; |
| 407 | Log.i(TAG, "Set " + modelInfo.name + " capture audio to " + captureAudio); |
| 408 | } |
| 409 | |
| 410 | public synchronized boolean hasMicrophonePermission() { |
| 411 | return getBaseContext().checkSelfPermission(Manifest.permission.RECORD_AUDIO) |
| 412 | == PackageManager.PERMISSION_GRANTED; |
| 413 | } |
| 414 | |
| 415 | public synchronized boolean modelHasTriggerAudio(UUID modelUuid) { |
| 416 | ModelInfo modelInfo = mModelInfoMap.get(modelUuid); |
| 417 | return modelInfo != null && modelInfo.triggerAudioPlayer != null; |
| 418 | } |
| 419 | |
| 420 | public synchronized boolean modelWillCaptureTriggerAudio(UUID modelUuid) { |
| 421 | ModelInfo modelInfo = mModelInfoMap.get(modelUuid); |
| 422 | return modelInfo != null && modelInfo.captureAudio; |
| 423 | } |
| 424 | |
| 425 | public synchronized boolean modelHasCapturedAudio(UUID modelUuid) { |
| 426 | ModelInfo modelInfo = mModelInfoMap.get(modelUuid); |
| 427 | return modelInfo != null && modelInfo.captureAudioTrack != null; |
| 428 | } |
| 429 | |
| 430 | private void loadModelsInDataDir() { |
| 431 | // Load all the models in the data dir. |
| 432 | boolean loadedModel = false; |
| 433 | for (File file : getFilesDir().listFiles()) { |
| 434 | // Find meta-data in .properties files, ignore everything else. |
| 435 | if (!file.getName().endsWith(".properties")) { |
| 436 | continue; |
| 437 | } |
| 438 | try { |
| 439 | Properties properties = new Properties(); |
| 440 | properties.load(new FileInputStream(file)); |
| 441 | createModelInfo(properties); |
| 442 | loadedModel = true; |
| 443 | } catch (Exception e) { |
| 444 | Log.e(TAG, "Failed to load properties file " + file.getName()); |
| 445 | } |
| 446 | } |
| 447 | |
| 448 | // Create a few dummy models if we didn't load anything. |
| 449 | if (!loadedModel) { |
| 450 | Properties dummyModelProperties = new Properties(); |
| 451 | for (String name : new String[]{"1", "2", "3"}) { |
| 452 | dummyModelProperties.setProperty("name", "Model " + name); |
| 453 | createModelInfo(dummyModelProperties); |
| 454 | } |
| 455 | } |
| 456 | } |
| 457 | |
| 458 | /** Parses a Properties collection to generate a sound model. |
| 459 | * |
| 460 | * Missing keys are filled in with default/random values. |
| 461 | * @param properties Has the required 'name' property, but the remaining 'modelUuid', |
| 462 | * 'vendorUuid', 'triggerAudio', and 'dataFile' optional properties. |
| 463 | * |
| 464 | */ |
| 465 | private synchronized void createModelInfo(Properties properties) { |
| 466 | try { |
| 467 | ModelInfo modelInfo = new ModelInfo(); |
| 468 | |
| 469 | if (!properties.containsKey("name")) { |
| 470 | throw new RuntimeException("must have a 'name' property"); |
| 471 | } |
| 472 | modelInfo.name = properties.getProperty("name"); |
| 473 | |
| 474 | if (properties.containsKey("modelUuid")) { |
| 475 | modelInfo.modelUuid = UUID.fromString(properties.getProperty("modelUuid")); |
| 476 | } else { |
| 477 | modelInfo.modelUuid = UUID.randomUUID(); |
| 478 | } |
| 479 | |
| 480 | if (properties.containsKey("vendorUuid")) { |
| 481 | modelInfo.vendorUuid = UUID.fromString(properties.getProperty("vendorUuid")); |
| 482 | } else { |
| 483 | modelInfo.vendorUuid = UUID.randomUUID(); |
| 484 | } |
| 485 | |
| 486 | if (properties.containsKey("triggerAudio")) { |
| 487 | modelInfo.triggerAudioPlayer = MediaPlayer.create(this, Uri.parse( |
| 488 | getFilesDir().getPath() + "/" + properties.getProperty("triggerAudio"))); |
| 489 | if (modelInfo.triggerAudioPlayer.getDuration() == 0) { |
| 490 | modelInfo.triggerAudioPlayer.release(); |
| 491 | modelInfo.triggerAudioPlayer = null; |
| 492 | } |
| 493 | } |
| 494 | |
| 495 | if (properties.containsKey("dataFile")) { |
| 496 | File modelDataFile = new File( |
| 497 | getFilesDir().getPath() + "/" + properties.getProperty("dataFile")); |
| 498 | modelInfo.modelData = new byte[(int) modelDataFile.length()]; |
| 499 | FileInputStream input = new FileInputStream(modelDataFile); |
| 500 | input.read(modelInfo.modelData, 0, modelInfo.modelData.length); |
| 501 | } else { |
| 502 | modelInfo.modelData = new byte[1024]; |
| 503 | mRandom.nextBytes(modelInfo.modelData); |
| 504 | } |
| 505 | |
| 506 | modelInfo.captureAudioMs = Integer.parseInt((String) properties.getOrDefault( |
| 507 | "captureAudioDurationMs", "5000")); |
| 508 | |
| 509 | // TODO: Add property support for keyphrase models when they're exposed by the |
| 510 | // service. |
| 511 | |
| 512 | // Update our maps containing the button -> id and id -> modelInfo. |
| 513 | mModelInfoMap.put(modelInfo.modelUuid, modelInfo); |
| 514 | if (mUserActivity != null) { |
| 515 | mUserActivity.addModel(modelInfo.modelUuid, modelInfo.name); |
| 516 | mUserActivity.setModelState(modelInfo.modelUuid, modelInfo.state); |
| 517 | } |
| 518 | } catch (IOException e) { |
| 519 | Log.e(TAG, "Error parsing properties for " + properties.getProperty("name"), e); |
| 520 | } |
| 521 | } |
| 522 | |
| 523 | private class CaptureAudioRecorder implements Runnable { |
| 524 | private final ModelInfo mModelInfo; |
| 525 | private final SoundTriggerDetector.EventPayload mEvent; |
| 526 | |
| 527 | public CaptureAudioRecorder(ModelInfo modelInfo, SoundTriggerDetector.EventPayload event) { |
| 528 | mModelInfo = modelInfo; |
| 529 | mEvent = event; |
| 530 | } |
| 531 | |
| 532 | @Override |
| 533 | public void run() { |
| 534 | AudioFormat format = mEvent.getCaptureAudioFormat(); |
| 535 | if (format == null) { |
| 536 | postErrorToast("No audio format in recognition event."); |
| 537 | return; |
| 538 | } |
| 539 | |
| 540 | AudioRecord audioRecord = null; |
| 541 | AudioTrack playbackTrack = null; |
| 542 | try { |
| 543 | // Inform the audio flinger that we really do want the stream from the soundtrigger. |
| 544 | AudioAttributes.Builder attributesBuilder = new AudioAttributes.Builder(); |
| 545 | attributesBuilder.setInternalCapturePreset(1999); |
| 546 | AudioAttributes attributes = attributesBuilder.build(); |
| 547 | |
| 548 | // Make sure we understand this kind of playback so we know how many bytes to read. |
| 549 | String encoding; |
| 550 | int bytesPerSample; |
| 551 | switch (format.getEncoding()) { |
| 552 | case AudioFormat.ENCODING_PCM_8BIT: |
| 553 | encoding = "8bit"; |
| 554 | bytesPerSample = 1; |
| 555 | break; |
| 556 | case AudioFormat.ENCODING_PCM_16BIT: |
| 557 | encoding = "16bit"; |
| 558 | bytesPerSample = 2; |
| 559 | break; |
| 560 | case AudioFormat.ENCODING_PCM_FLOAT: |
| 561 | encoding = "float"; |
| 562 | bytesPerSample = 4; |
| 563 | break; |
| 564 | default: |
| 565 | throw new RuntimeException("Unhandled audio format in event"); |
| 566 | } |
| 567 | |
| 568 | int bytesRequired = format.getSampleRate() * format.getChannelCount() * |
| 569 | bytesPerSample * mModelInfo.captureAudioMs / 1000; |
| 570 | int minBufferSize = AudioRecord.getMinBufferSize( |
| 571 | format.getSampleRate(), format.getChannelMask(), format.getEncoding()); |
| 572 | if (minBufferSize > bytesRequired) { |
| 573 | bytesRequired = minBufferSize; |
| 574 | } |
| 575 | |
| 576 | // Make an AudioTrack so we can play the data back out after it's finished |
| 577 | // recording. |
| 578 | try { |
| 579 | int channelConfig = AudioFormat.CHANNEL_OUT_MONO; |
| 580 | if (format.getChannelCount() == 2) { |
| 581 | channelConfig = AudioFormat.CHANNEL_OUT_STEREO; |
| 582 | } else if (format.getChannelCount() >= 3) { |
| 583 | throw new RuntimeException( |
| 584 | "Too many channels in captured audio for playback"); |
| 585 | } |
| 586 | |
| 587 | playbackTrack = new AudioTrack(AudioManager.STREAM_MUSIC, |
| 588 | format.getSampleRate(), channelConfig, format.getEncoding(), |
| 589 | bytesRequired, AudioTrack.MODE_STATIC); |
| 590 | } catch (Exception e) { |
| 591 | Log.e(TAG, "Exception creating playback track", e); |
| 592 | postErrorToast("Failed to create playback track: " + e.getMessage()); |
| 593 | } |
| 594 | |
| 595 | audioRecord = new AudioRecord(attributes, format, bytesRequired, |
| 596 | mEvent.getCaptureSession()); |
| 597 | |
| 598 | byte[] buffer = new byte[bytesRequired]; |
| 599 | |
| 600 | // Create a file so we can save the output data there for analysis later. |
| 601 | FileOutputStream fos = null; |
| 602 | try { |
| 603 | fos = new FileOutputStream( new File( |
| 604 | getFilesDir() + File.separator + mModelInfo.name.replace(' ', '_') + |
| 605 | "_capture_" + format.getChannelCount() + "ch_" + |
| 606 | format.getSampleRate() + "hz_" + encoding + ".pcm")); |
| 607 | } catch (IOException e) { |
| 608 | Log.e(TAG, "Failed to open output for saving PCM data", e); |
| 609 | postErrorToast("Failed to open output for saving PCM data: " + e.getMessage()); |
| 610 | } |
| 611 | |
| 612 | // Inform the user we're recording. |
| 613 | setModelState(mModelInfo, "Recording"); |
| 614 | audioRecord.startRecording(); |
| 615 | while (bytesRequired > 0) { |
| 616 | int bytesRead = audioRecord.read(buffer, 0, buffer.length); |
| 617 | if (bytesRead == -1) { |
| 618 | break; |
| 619 | } |
| 620 | if (fos != null) { |
| 621 | fos.write(buffer, 0, bytesRead); |
| 622 | } |
| 623 | if (playbackTrack != null) { |
| 624 | playbackTrack.write(buffer, 0, bytesRead); |
| 625 | } |
| 626 | bytesRequired -= bytesRead; |
| 627 | } |
| 628 | audioRecord.stop(); |
| 629 | } catch (Exception e) { |
| 630 | Log.e(TAG, "Error recording trigger audio", e); |
| 631 | postErrorToast("Error recording trigger audio: " + e.getMessage()); |
| 632 | } finally { |
| 633 | if (audioRecord != null) { |
| 634 | audioRecord.release(); |
| 635 | } |
| 636 | synchronized (SoundTriggerTestService.this) { |
| 637 | if (mModelInfo.captureAudioTrack != null) { |
| 638 | mModelInfo.captureAudioTrack.release(); |
| 639 | } |
| 640 | mModelInfo.captureAudioTrack = playbackTrack; |
| 641 | } |
| 642 | setModelState(mModelInfo, "Recording finished"); |
| 643 | } |
| 644 | } |
| 645 | } |
| 646 | |
| 647 | // Implementation of SoundTriggerDetector.Callback. |
| 648 | private class DetectorCallback extends SoundTriggerDetector.Callback { |
| 649 | private final ModelInfo mModelInfo; |
| 650 | |
| 651 | public DetectorCallback(ModelInfo modelInfo) { |
| 652 | mModelInfo = modelInfo; |
| 653 | } |
| 654 | |
| 655 | public void onAvailabilityChanged(int status) { |
Chris Thornton | 2d2ba9d0 | 2016-07-12 15:05:18 -0700 | [diff] [blame] | 656 | postMessage(mModelInfo.name + " availability changed to: " + status); |
Chris Thornton | dfa7c3b | 2016-06-30 22:05:51 -0700 | [diff] [blame] | 657 | } |
| 658 | |
| 659 | public void onDetected(SoundTriggerDetector.EventPayload event) { |
Chris Thornton | 2d2ba9d0 | 2016-07-12 15:05:18 -0700 | [diff] [blame] | 660 | postMessage(mModelInfo.name + " onDetected(): " + eventPayloadToString(event)); |
Chris Thornton | dfa7c3b | 2016-06-30 22:05:51 -0700 | [diff] [blame] | 661 | synchronized (SoundTriggerTestService.this) { |
| 662 | if (mUserActivity != null) { |
| 663 | mUserActivity.handleDetection(mModelInfo.modelUuid); |
| 664 | } |
| 665 | if (mModelInfo.captureAudio) { |
| 666 | new Thread(new CaptureAudioRecorder(mModelInfo, event)).start(); |
| 667 | } |
| 668 | } |
| 669 | } |
| 670 | |
| 671 | public void onError() { |
Chris Thornton | 2d2ba9d0 | 2016-07-12 15:05:18 -0700 | [diff] [blame] | 672 | postMessage(mModelInfo.name + " onError()"); |
Chris Thornton | dfa7c3b | 2016-06-30 22:05:51 -0700 | [diff] [blame] | 673 | setModelState(mModelInfo, "Error"); |
| 674 | } |
| 675 | |
| 676 | public void onRecognitionPaused() { |
| 677 | postMessage(mModelInfo.name + " onRecognitionPaused()"); |
| 678 | setModelState(mModelInfo, "Paused"); |
| 679 | } |
| 680 | |
| 681 | public void onRecognitionResumed() { |
Chris Thornton | 2d2ba9d0 | 2016-07-12 15:05:18 -0700 | [diff] [blame] | 682 | postMessage(mModelInfo.name + " onRecognitionResumed()"); |
Chris Thornton | dfa7c3b | 2016-06-30 22:05:51 -0700 | [diff] [blame] | 683 | setModelState(mModelInfo, "Resumed"); |
| 684 | } |
| 685 | } |
| 686 | |
| 687 | private String eventPayloadToString(SoundTriggerDetector.EventPayload event) { |
| 688 | String result = "EventPayload("; |
| 689 | AudioFormat format = event.getCaptureAudioFormat(); |
| 690 | result = result + "AudioFormat: " + ((format == null) ? "null" : format.toString()); |
| 691 | byte[] triggerAudio = event.getTriggerAudio(); |
| 692 | result = result + "TriggerAudio: " + (triggerAudio == null ? "null" : triggerAudio.length); |
| 693 | result = result + "CaptureSession: " + event.getCaptureSession(); |
| 694 | result += " )"; |
| 695 | return result; |
| 696 | } |
| 697 | |
| 698 | private void postMessage(String msg) { |
| 699 | showMessage(msg, Log.INFO, false); |
| 700 | } |
| 701 | |
| 702 | private void postError(String msg) { |
| 703 | showMessage(msg, Log.ERROR, false); |
| 704 | } |
| 705 | |
| 706 | private void postToast(String msg) { |
| 707 | showMessage(msg, Log.INFO, true); |
| 708 | } |
| 709 | |
| 710 | private void postErrorToast(String msg) { |
| 711 | showMessage(msg, Log.ERROR, true); |
| 712 | } |
| 713 | |
| 714 | /** Logs the message at the specified level, then forwards it to the activity if present. */ |
| 715 | private synchronized void showMessage(String msg, int logLevel, boolean showToast) { |
| 716 | Log.println(logLevel, TAG, msg); |
| 717 | if (mUserActivity != null) { |
| 718 | mUserActivity.showMessage(msg, showToast); |
| 719 | } |
| 720 | } |
| 721 | |
| 722 | private synchronized void setModelState(ModelInfo modelInfo, String state) { |
| 723 | modelInfo.state = state; |
| 724 | if (mUserActivity != null) { |
| 725 | mUserActivity.setModelState(modelInfo.modelUuid, modelInfo.state); |
| 726 | } |
| 727 | } |
| 728 | } |