Update voice interaction layer for new UI design.
Can switch from a pure overlay at the top of the screen,
to interactive mode with the voice UI drawing at the bottom
and pushing its target activity up like an IME.
Add mechanism to get assist data to the voice interaction UI.
Add some basic visualization of the assist data, outlining
where on the screen we have text.
Add a test ACTION_ASSIST handler, which can propagate the
assist data it gets to the voice interaction session so
you can see what kind of data we are getting from different
apps.
Change-Id: I18312fe1601d7926d1fb96a817638d60f6263771
diff --git a/core/java/android/app/ActivityManagerNative.java b/core/java/android/app/ActivityManagerNative.java
index 47f57ea..fa10893 100644
--- a/core/java/android/app/ActivityManagerNative.java
+++ b/core/java/android/app/ActivityManagerNative.java
@@ -51,6 +51,7 @@
import android.util.Log;
import android.util.Singleton;
import com.android.internal.app.IVoiceInteractor;
+import com.android.internal.os.IResultReceiver;
import java.util.ArrayList;
import java.util.List;
@@ -2114,6 +2115,15 @@
return true;
}
+ case REQUEST_ASSIST_CONTEXT_EXTRAS_TRANSACTION: {
+ data.enforceInterface(IActivityManager.descriptor);
+ int requestType = data.readInt();
+ IResultReceiver receiver = IResultReceiver.Stub.asInterface(data.readStrongBinder());
+ requestAssistContextExtras(requestType, receiver);
+ reply.writeNoException();
+ return true;
+ }
+
case REPORT_ASSIST_CONTEXT_EXTRAS_TRANSACTION: {
data.enforceInterface(IActivityManager.descriptor);
IBinder token = data.readStrongBinder();
@@ -5146,6 +5156,19 @@
return res;
}
+ public void requestAssistContextExtras(int requestType, IResultReceiver receiver)
+ throws RemoteException {
+ Parcel data = Parcel.obtain();
+ Parcel reply = Parcel.obtain();
+ data.writeInterfaceToken(IActivityManager.descriptor);
+ data.writeInt(requestType);
+ data.writeStrongBinder(receiver.asBinder());
+ mRemote.transact(REQUEST_ASSIST_CONTEXT_EXTRAS_TRANSACTION, data, reply, 0);
+ reply.readException();
+ data.recycle();
+ reply.recycle();
+ }
+
public void reportAssistContextExtras(IBinder token, Bundle extras)
throws RemoteException {
Parcel data = Parcel.obtain();
diff --git a/core/java/android/app/AssistData.java b/core/java/android/app/AssistData.java
index 8d3d348..7b5eb6d 100644
--- a/core/java/android/app/AssistData.java
+++ b/core/java/android/app/AssistData.java
@@ -16,6 +16,7 @@
package android.app;
+import android.content.ComponentName;
import android.graphics.Rect;
import android.os.Bundle;
import android.os.Parcel;
@@ -44,6 +45,8 @@
*/
public static final String ASSIST_KEY = "android:assist";
+ final ComponentName mActivityComponent;
+
final ArrayList<ViewNodeImpl> mRootViews = new ArrayList<>();
ViewAssistDataImpl mTmpViewAssistDataImpl = new ViewAssistDataImpl();
@@ -400,6 +403,7 @@
}
AssistData(Activity activity) {
+ mActivityComponent = activity.getComponentName();
ArrayList<ViewRootImpl> views = WindowManagerGlobal.getInstance().getRootViews(
activity.getActivityToken());
for (int i=0; i<views.size(); i++) {
@@ -414,6 +418,7 @@
}
AssistData(Parcel in) {
+ mActivityComponent = ComponentName.readFromParcel(in);
final int N = in.readInt();
for (int i=0; i<N; i++) {
mRootViews.add(new ViewNodeImpl(in));
@@ -421,7 +426,9 @@
//dump();
}
- void dump() {
+ /** @hide */
+ public void dump() {
+ Log.i(TAG, "Activity: " + mActivityComponent.flattenToShortString());
ViewNode node = new ViewNode();
final int N = getWindowCount();
for (int i=0; i<N; i++) {
@@ -445,7 +452,7 @@
}
String text = node.getText();
if (text != null) {
- Log.i(TAG, prefix + " Text (sel " + node.getTextSelectionStart() + "-"
+ Log.i(TAG, prefix + " Text (sel " + node.getTextSelectionStart() + "-"
+ node.getTextSelectionEnd() + "): " + text);
}
String hint = node.getHint();
@@ -476,6 +483,10 @@
return assistBundle.getParcelable(ASSIST_KEY);
}
+ public ComponentName getActivityComponent() {
+ return mActivityComponent;
+ }
+
/**
* Return the number of window contents that have been collected in this assist data.
*/
@@ -498,6 +509,7 @@
public void writeToParcel(Parcel out, int flags) {
int start = out.dataPosition();
+ ComponentName.writeToParcel(mActivityComponent, out);
final int N = mRootViews.size();
out.writeInt(N);
for (int i=0; i<N; i++) {
diff --git a/core/java/android/app/IActivityManager.java b/core/java/android/app/IActivityManager.java
index 467c99a..341a2d7 100644
--- a/core/java/android/app/IActivityManager.java
+++ b/core/java/android/app/IActivityManager.java
@@ -51,6 +51,7 @@
import android.os.StrictMode;
import android.service.voice.IVoiceInteractionSession;
import com.android.internal.app.IVoiceInteractor;
+import com.android.internal.os.IResultReceiver;
import java.util.List;
@@ -419,6 +420,9 @@
public Bundle getAssistContextExtras(int requestType) throws RemoteException;
+ public void requestAssistContextExtras(int requestType, IResultReceiver receiver)
+ throws RemoteException;
+
public void reportAssistContextExtras(IBinder token, Bundle extras) throws RemoteException;
public boolean launchAssistIntent(Intent intent, int requestType, String hint, int userHandle)
@@ -804,4 +808,5 @@
int CREATE_STACK_ON_DISPLAY = IBinder.FIRST_CALL_TRANSACTION+281;
int GET_FOCUSED_STACK_ID_TRANSACTION = IBinder.FIRST_CALL_TRANSACTION+282;
int SET_TASK_RESIZEABLE_TRANSACTION = IBinder.FIRST_CALL_TRANSACTION+283;
+ int REQUEST_ASSIST_CONTEXT_EXTRAS_TRANSACTION = IBinder.FIRST_CALL_TRANSACTION+284;
}
diff --git a/core/java/android/service/dreams/DreamService.java b/core/java/android/service/dreams/DreamService.java
index fa5ac42..d410622 100644
--- a/core/java/android/service/dreams/DreamService.java
+++ b/core/java/android/service/dreams/DreamService.java
@@ -1039,10 +1039,10 @@
protected void dump(final FileDescriptor fd, PrintWriter pw, final String[] args) {
DumpUtils.dumpAsync(mHandler, new Dump() {
@Override
- public void dump(PrintWriter pw) {
+ public void dump(PrintWriter pw, String prefix) {
dumpOnHandler(fd, pw, args);
}
- }, pw, 1000);
+ }, pw, "", 1000);
}
/** @hide */
diff --git a/core/java/android/service/voice/IVoiceInteractionSession.aidl b/core/java/android/service/voice/IVoiceInteractionSession.aidl
index 9f9c312..a8c0c4c 100644
--- a/core/java/android/service/voice/IVoiceInteractionSession.aidl
+++ b/core/java/android/service/voice/IVoiceInteractionSession.aidl
@@ -17,11 +17,13 @@
package android.service.voice;
import android.content.Intent;
+import android.os.Bundle;
/**
* @hide
*/
oneway interface IVoiceInteractionSession {
+ void handleAssist(in Bundle assistData);
void taskStarted(in Intent intent, int taskId);
void taskFinished(in Intent intent, int taskId);
void closeSystemDialogs();
diff --git a/core/java/android/service/voice/IVoiceInteractionSessionService.aidl b/core/java/android/service/voice/IVoiceInteractionSessionService.aidl
index 2519442..7f8158f 100644
--- a/core/java/android/service/voice/IVoiceInteractionSessionService.aidl
+++ b/core/java/android/service/voice/IVoiceInteractionSessionService.aidl
@@ -24,5 +24,5 @@
* @hide
*/
oneway interface IVoiceInteractionSessionService {
- void newSession(IBinder token, in Bundle args);
+ void newSession(IBinder token, in Bundle args, int startFlags);
}
diff --git a/core/java/android/service/voice/VoiceInteractionService.java b/core/java/android/service/voice/VoiceInteractionService.java
index 65e6988..54a3a0a 100644
--- a/core/java/android/service/voice/VoiceInteractionService.java
+++ b/core/java/android/service/voice/VoiceInteractionService.java
@@ -70,6 +70,12 @@
*/
public static final String SERVICE_META_DATA = "android.voice_interaction";
+ /**
+ * Flag for use with {@link #startSession}: request that the session be started with
+ * assist data from the currently focused activity.
+ */
+ public static final int START_WITH_ASSIST = 1<<0;
+
IVoiceInteractionService mInterface = new IVoiceInteractionService.Stub() {
@Override public void ready() {
mHandler.sendEmptyMessage(MSG_READY);
@@ -136,16 +142,21 @@
* Initiate the execution of a new {@link android.service.voice.VoiceInteractionSession}.
* @param args Arbitrary arguments that will be propagated to the session.
*/
- public void startSession(Bundle args) {
+ public void startSession(Bundle args, int flags) {
if (mSystemService == null) {
throw new IllegalStateException("Not available until onReady() is called");
}
try {
- mSystemService.startSession(mInterface, args);
+ mSystemService.startSession(mInterface, args, flags);
} catch (RemoteException e) {
}
}
+ /** @hide */
+ public void startSession(Bundle args) {
+ startSession(args, 0);
+ }
+
@Override
public void onCreate() {
super.onCreate();
@@ -163,8 +174,8 @@
/**
* Called during service initialization to tell you when the system is ready
* to receive interaction from it. You should generally do initialization here
- * rather than in {@link #onCreate()}. Methods such as {@link #startSession(Bundle)} and
- * {@link #createAlwaysOnHotwordDetector(String, Locale, android.service.voice.AlwaysOnHotwordDetector.Callback)}
+ * rather than in {@link #onCreate}. Methods such as {@link #startSession} and
+ * {@link #createAlwaysOnHotwordDetector}
* will not be operational until this point.
*/
public void onReady() {
diff --git a/core/java/android/service/voice/VoiceInteractionServiceInfo.java b/core/java/android/service/voice/VoiceInteractionServiceInfo.java
index e6e9413..ebc7507 100644
--- a/core/java/android/service/voice/VoiceInteractionServiceInfo.java
+++ b/core/java/android/service/voice/VoiceInteractionServiceInfo.java
@@ -99,6 +99,10 @@
mParseError = "No sessionService specified";
return;
}
+ if (mRecognitionService == null) {
+ mParseError = "No recognitionService specified";
+ return;
+ }
/* Not yet time
if (mRecognitionService == null) {
mParseError = "No recogitionService specified";
diff --git a/core/java/android/service/voice/VoiceInteractionSession.java b/core/java/android/service/voice/VoiceInteractionSession.java
index 19d14bf..a3a2ca1 100644
--- a/core/java/android/service/voice/VoiceInteractionSession.java
+++ b/core/java/android/service/voice/VoiceInteractionSession.java
@@ -50,7 +50,6 @@
import java.lang.ref.WeakReference;
import static android.view.ViewGroup.LayoutParams.MATCH_PARENT;
-import static android.view.ViewGroup.LayoutParams.WRAP_CONTENT;
/**
* An active voice interaction session, providing a facility for the implementation
@@ -91,7 +90,6 @@
final ArrayMap<IBinder, Request> mActiveRequests = new ArrayMap<IBinder, Request>();
final Insets mTmpInsets = new Insets();
- final int[] mTmpLocation = new int[2];
final WeakReference<VoiceInteractionSession> mWeakRef
= new WeakReference<VoiceInteractionSession>(this);
@@ -153,6 +151,12 @@
final IVoiceInteractionSession mSession = new IVoiceInteractionSession.Stub() {
@Override
+ public void handleAssist(Bundle assistBundle) {
+ mHandlerCaller.sendMessage(mHandlerCaller.obtainMessageO(MSG_HANDLE_ASSIST,
+ assistBundle));
+ }
+
+ @Override
public void taskStarted(Intent intent, int taskId) {
mHandlerCaller.sendMessage(mHandlerCaller.obtainMessageIO(MSG_TASK_STARTED,
taskId, intent));
@@ -279,6 +283,7 @@
static final int MSG_TASK_FINISHED = 101;
static final int MSG_CLOSE_SYSTEM_DIALOGS = 102;
static final int MSG_DESTROY = 103;
+ static final int MSG_HANDLE_ASSIST = 104;
class MyCallbacks implements HandlerCaller.Callback, SoftInputWindow.Callback {
@Override
@@ -341,6 +346,10 @@
if (DEBUG) Log.d(TAG, "doDestroy");
doDestroy();
break;
+ case MSG_HANDLE_ASSIST:
+ if (DEBUG) Log.d(TAG, "onHandleAssist: " + (Bundle)msg.obj);
+ onHandleAssist((Bundle) msg.obj);
+ break;
}
}
@@ -441,10 +450,11 @@
}
}
- void doCreate(IVoiceInteractionManagerService service, IBinder token, Bundle args) {
+ void doCreate(IVoiceInteractionManagerService service, IBinder token, Bundle args,
+ int startFlags) {
mSystemService = service;
mToken = token;
- onCreate(args);
+ onCreate(args, startFlags);
}
void doDestroy() {
@@ -585,12 +595,7 @@
}
}
- /**
- * Initiatize a new session.
- *
- * @param args The arguments that were supplied to
- * {@link VoiceInteractionService#startSession VoiceInteractionService.startSession}.
- */
+ /** @hide */
public void onCreate(Bundle args) {
mTheme = mTheme != 0 ? mTheme
: com.android.internal.R.style.Theme_DeviceDefault_VoiceInteractionSession;
@@ -598,14 +603,26 @@
Context.LAYOUT_INFLATER_SERVICE);
mWindow = new SoftInputWindow(mContext, "VoiceInteractionSession", mTheme,
mCallbacks, this, mDispatcherState,
- WindowManager.LayoutParams.TYPE_VOICE_INTERACTION, Gravity.TOP, true);
+ WindowManager.LayoutParams.TYPE_VOICE_INTERACTION, Gravity.BOTTOM, true);
mWindow.getWindow().addFlags(WindowManager.LayoutParams.FLAG_HARDWARE_ACCELERATED);
initViews();
- mWindow.getWindow().setLayout(MATCH_PARENT, WRAP_CONTENT);
+ mWindow.getWindow().setLayout(MATCH_PARENT, MATCH_PARENT);
mWindow.setToken(mToken);
}
/**
+ * Initiatize a new session.
+ *
+ * @param args The arguments that were supplied to
+ * {@link VoiceInteractionService#startSession VoiceInteractionService.startSession}.
+ * @param startFlags The start flags originally provided to
+ * {@link VoiceInteractionService#startSession VoiceInteractionService.startSession}.
+ */
+ public void onCreate(Bundle args, int startFlags) {
+ onCreate(args);
+ }
+
+ /**
* Last callback to the session as it is being finished.
*/
public void onDestroy() {
@@ -622,10 +639,13 @@
mContentFrame.removeAllViews();
mContentFrame.addView(view, new FrameLayout.LayoutParams(
ViewGroup.LayoutParams.MATCH_PARENT,
- ViewGroup.LayoutParams.WRAP_CONTENT));
+ ViewGroup.LayoutParams.MATCH_PARENT));
}
+ public void onHandleAssist(Bundle assistBundle) {
+ }
+
public boolean onKeyDown(int keyCode, KeyEvent event) {
return false;
}
@@ -657,19 +677,19 @@
/**
* Compute the interesting insets into your UI. The default implementation
- * uses the entire window frame as the insets. The default touchable
- * insets are {@link Insets#TOUCHABLE_INSETS_FRAME}.
+ * sets {@link Insets#contentInsets outInsets.contentInsets.top} to the height
+ * of the window, meaning it should not adjust content underneath. The default touchable
+ * insets are {@link Insets#TOUCHABLE_INSETS_FRAME}, meaning it consumes all touch
+ * events within its window frame.
*
* @param outInsets Fill in with the current UI insets.
*/
public void onComputeInsets(Insets outInsets) {
- int[] loc = mTmpLocation;
- View decor = getWindow().getWindow().getDecorView();
- decor.getLocationInWindow(loc);
- outInsets.contentInsets.top = 0;
outInsets.contentInsets.left = 0;
- outInsets.contentInsets.right = 0;
outInsets.contentInsets.bottom = 0;
+ outInsets.contentInsets.right = 0;
+ View decor = getWindow().getWindow().getDecorView();
+ outInsets.contentInsets.top = decor.getHeight();
outInsets.touchableInsets = Insets.TOUCHABLE_INSETS_FRAME;
outInsets.touchableRegion.setEmpty();
}
diff --git a/core/java/android/service/voice/VoiceInteractionSessionService.java b/core/java/android/service/voice/VoiceInteractionSessionService.java
index e793849..008d55f 100644
--- a/core/java/android/service/voice/VoiceInteractionSessionService.java
+++ b/core/java/android/service/voice/VoiceInteractionSessionService.java
@@ -40,9 +40,9 @@
VoiceInteractionSession mSession;
IVoiceInteractionSessionService mInterface = new IVoiceInteractionSessionService.Stub() {
- public void newSession(IBinder token, Bundle args) {
- mHandlerCaller.sendMessage(mHandlerCaller.obtainMessageOO(MSG_NEW_SESSION,
- token, args));
+ public void newSession(IBinder token, Bundle args, int startFlags) {
+ mHandlerCaller.sendMessage(mHandlerCaller.obtainMessageIOO(MSG_NEW_SESSION,
+ startFlags, token, args));
}
};
@@ -54,7 +54,7 @@
SomeArgs args = (SomeArgs)msg.obj;
switch (msg.what) {
case MSG_NEW_SESSION:
- doNewSession((IBinder)args.arg1, (Bundle)args.arg2);
+ doNewSession((IBinder)args.arg1, (Bundle)args.arg2, args.argi1);
break;
}
}
@@ -76,7 +76,7 @@
return mInterface.asBinder();
}
- void doNewSession(IBinder token, Bundle args) {
+ void doNewSession(IBinder token, Bundle args, int startFlags) {
if (mSession != null) {
mSession.doDestroy();
mSession = null;
@@ -84,7 +84,7 @@
mSession = onNewSession(args);
try {
mSystemService.deliverNewSession(token, mSession.mSession, mSession.mInteractor);
- mSession.doCreate(mSystemService, token, args);
+ mSession.doCreate(mSystemService, token, args, startFlags);
} catch (RemoteException e) {
}
}
diff --git a/core/java/com/android/internal/app/IVoiceInteractionManagerService.aidl b/core/java/com/android/internal/app/IVoiceInteractionManagerService.aidl
index 5a10524..6d90420 100644
--- a/core/java/com/android/internal/app/IVoiceInteractionManagerService.aidl
+++ b/core/java/com/android/internal/app/IVoiceInteractionManagerService.aidl
@@ -26,7 +26,7 @@
import android.service.voice.IVoiceInteractionSession;
interface IVoiceInteractionManagerService {
- void startSession(IVoiceInteractionService service, in Bundle sessionArgs);
+ void startSession(IVoiceInteractionService service, in Bundle sessionArgs, int flags);
boolean deliverNewSession(IBinder token, IVoiceInteractionSession session,
IVoiceInteractor interactor);
int startVoiceActivity(IBinder token, in Intent intent, String resolvedType);
diff --git a/core/java/com/android/internal/util/DumpUtils.java b/core/java/com/android/internal/util/DumpUtils.java
index 65b56ec..64e1d10 100644
--- a/core/java/com/android/internal/util/DumpUtils.java
+++ b/core/java/com/android/internal/util/DumpUtils.java
@@ -35,13 +35,14 @@
* trying to acquire, we use a short timeout to avoid deadlocks. The process
* is inelegant but this function is only used for debugging purposes.
*/
- public static void dumpAsync(Handler handler, final Dump dump, PrintWriter pw, long timeout) {
+ public static void dumpAsync(Handler handler, final Dump dump, PrintWriter pw,
+ final String prefix, long timeout) {
final StringWriter sw = new StringWriter();
if (handler.runWithScissors(new Runnable() {
@Override
public void run() {
PrintWriter lpw = new FastPrintWriter(sw);
- dump.dump(lpw);
+ dump.dump(lpw, prefix);
lpw.close();
}
}, timeout)) {
@@ -52,6 +53,6 @@
}
public interface Dump {
- void dump(PrintWriter pw);
+ void dump(PrintWriter pw, String prefix);
}
}
diff --git a/core/res/res/values/themes_material.xml b/core/res/res/values/themes_material.xml
index 57041fdd..3fa24c2 100644
--- a/core/res/res/values/themes_material.xml
+++ b/core/res/res/values/themes_material.xml
@@ -972,8 +972,12 @@
{@link android.service.voice.VoiceInteractionSession} class.
this inherits from Theme.Panel, but sets up appropriate animations
and a few custom attributes. -->
- <style name="Theme.Material.VoiceInteractionSession" parent="Theme.Material.Light.Panel">
- <item name="windowAnimationStyle">@style/Animation.VoiceInteractionSession</item>
+ <style name="Theme.Material.VoiceInteractionSession"
+ parent="Theme.Material.Light.NoActionBar.TranslucentDecor">
+ <item name="windowBackground">@color/transparent</item>
+ <item name="colorBackgroundCacheHint">@null</item>
+ <item name="windowIsTranslucent">true</item>
+ <item name="windowAnimationStyle">@style/Animation</item>
</style>
<!-- Theme for the search input bar. -->