Yet another (major) refactoring on Content Capture and Augmented Autofill.
Bunch of changes:
- Split public SmartSuggestionsService info ContentCaptureService and
AugmentedAutofillService
- Renamed 'intelligence' packages to either 'contentcapture' or
'autofil.augmented'
- Renamed internal packages and classes.
- Changed permissions, resource names, etc...
- Moved Augmented Autofill logic from IntelligeceManagerService (R.I.P.) to
Autofill.
- Optimized IPCs by passing a String instead of the InteractionSessionId
(that also solves the view -> service dependency).
Test: atest CtsContentCaptureServiceTestCases \
CtsAutoFillServiceTestCases \
FrameworksCoreTests:SettingsBackupTest
Test: manual verification with Augmented Autofill Service
Bug: 119638877
Bug: 117944706
Change-Id: I787fc2a0dbd9ad53e4d5edb0d2a9242346e4652d
diff --git a/core/java/android/view/contentcapture/ContentCaptureManager.java b/core/java/android/view/contentcapture/ContentCaptureManager.java
new file mode 100644
index 0000000..48831da
--- /dev/null
+++ b/core/java/android/view/contentcapture/ContentCaptureManager.java
@@ -0,0 +1,484 @@
+/*
+ * Copyright (C) 2018 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package android.view.contentcapture;
+
+import static android.view.contentcapture.ContentCaptureEvent.TYPE_VIEW_APPEARED;
+import static android.view.contentcapture.ContentCaptureEvent.TYPE_VIEW_DISAPPEARED;
+import static android.view.contentcapture.ContentCaptureEvent.TYPE_VIEW_TEXT_CHANGED;
+
+import static com.android.internal.util.function.pooled.PooledLambda.obtainMessage;
+
+import android.annotation.NonNull;
+import android.annotation.Nullable;
+import android.annotation.SystemService;
+import android.content.ComponentName;
+import android.content.Context;
+import android.os.Bundle;
+import android.os.Handler;
+import android.os.HandlerThread;
+import android.os.IBinder;
+import android.os.RemoteException;
+import android.util.Log;
+import android.view.View;
+import android.view.ViewStructure;
+import android.view.autofill.AutofillId;
+import android.view.contentcapture.ContentCaptureEvent.EventType;
+
+import com.android.internal.os.IResultReceiver;
+import com.android.internal.util.Preconditions;
+
+import java.io.PrintWriter;
+import java.util.ArrayList;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/*
+ * NOTE: all methods in this class should return right away, or do the real work in a handler
+ * thread.
+ *
+ * Hence, the only field that must be thread-safe is mEnabled, which is called at the beginning
+ * of every method.
+ */
+/**
+ * TODO(b/111276913): add javadocs / implement
+ */
+@SystemService(Context.CONTENT_CAPTURE_MANAGER_SERVICE)
+public final class ContentCaptureManager {
+
+ private static final String TAG = ContentCaptureManager.class.getSimpleName();
+
+ // TODO(b/111276913): define a way to dynamically set them(for example, using settings?)
+ private static final boolean VERBOSE = false;
+ private static final boolean DEBUG = true; // STOPSHIP if not set to false
+
+ /**
+ * Used to indicate that a text change was caused by user input (for example, through IME).
+ */
+ //TODO(b/111276913): link to notifyTextChanged() method once available
+ public static final int FLAG_USER_INPUT = 0x1;
+
+ /**
+ * Initial state, when there is no session.
+ *
+ * @hide
+ */
+ public static final int STATE_UNKNOWN = 0;
+
+ /**
+ * Service's startSession() was called, but server didn't confirm it was created yet.
+ *
+ * @hide
+ */
+ public static final int STATE_WAITING_FOR_SERVER = 1;
+
+ /**
+ * Session is active.
+ *
+ * @hide
+ */
+ public static final int STATE_ACTIVE = 2;
+
+ /**
+ * Session is disabled.
+ *
+ * @hide
+ */
+ public static final int STATE_DISABLED = 3;
+
+ private static final String BG_THREAD_NAME = "intel_svc_streamer_thread";
+
+ /**
+ * Maximum number of events that are buffered before sent to the app.
+ */
+ // TODO(b/111276913): use settings
+ private static final int MAX_BUFFER_SIZE = 100;
+
+ @NonNull
+ private final AtomicBoolean mDisabled = new AtomicBoolean();
+
+ @NonNull
+ private final Context mContext;
+
+ @Nullable
+ private final IContentCaptureManager mService;
+
+ @Nullable
+ private String mId;
+
+ private int mState = STATE_UNKNOWN;
+
+ @Nullable
+ private IBinder mApplicationToken;
+
+ @Nullable
+ private ComponentName mComponentName;
+
+ /**
+ * List of events held to be sent as a batch.
+ */
+ @Nullable
+ private ArrayList<ContentCaptureEvent> mEvents;
+
+ // TODO(b/111276913): use UI Thread directly (as calls are one-way) or a shared thread / handler
+ // held at the Application level
+ private final Handler mHandler;
+
+ /** @hide */
+ public ContentCaptureManager(@NonNull Context context,
+ @Nullable IContentCaptureManager service) {
+ mContext = Preconditions.checkNotNull(context, "context cannot be null");
+ if (VERBOSE) {
+ Log.v(TAG, "Constructor for " + context.getPackageName());
+ }
+ mService = service;
+ // TODO(b/111276913): use an existing bg thread instead...
+ final HandlerThread bgThread = new HandlerThread(BG_THREAD_NAME);
+ bgThread.start();
+ mHandler = Handler.createAsync(bgThread.getLooper());
+ }
+
+ /** @hide */
+ public void onActivityCreated(@NonNull IBinder token, @NonNull ComponentName componentName) {
+ if (!isContentCaptureEnabled()) return;
+
+ mHandler.sendMessage(obtainMessage(ContentCaptureManager::handleStartSession, this,
+ token, componentName));
+ }
+
+ private void handleStartSession(@NonNull IBinder token, @NonNull ComponentName componentName) {
+ if (mState != STATE_UNKNOWN) {
+ // TODO(b/111276913): revisit this scenario
+ Log.w(TAG, "ignoring handleStartSession(" + token + ") while on state "
+ + getStateAsString(mState));
+ return;
+ }
+ mState = STATE_WAITING_FOR_SERVER;
+ mId = UUID.randomUUID().toString();
+ mApplicationToken = token;
+ mComponentName = componentName;
+
+ if (VERBOSE) {
+ Log.v(TAG, "handleStartSession(): token=" + token + ", act="
+ + getActivityDebugName() + ", id=" + mId);
+ }
+ final int flags = 0; // TODO(b/111276913): get proper flags
+
+ try {
+ mService.startSession(mContext.getUserId(), mApplicationToken, componentName,
+ mId, flags, new IResultReceiver.Stub() {
+ @Override
+ public void send(int resultCode, Bundle resultData) {
+ handleSessionStarted(resultCode);
+ }
+ });
+ } catch (RemoteException e) {
+ Log.w(TAG, "Error starting session for " + componentName.flattenToShortString() + ": "
+ + e);
+ }
+ }
+
+ private void handleSessionStarted(int resultCode) {
+ mState = resultCode;
+ mDisabled.set(mState == STATE_DISABLED);
+ if (VERBOSE) {
+ Log.v(TAG, "onActivityStarted() result: code=" + resultCode + ", id=" + mId
+ + ", state=" + getStateAsString(mState) + ", disabled=" + mDisabled.get());
+ }
+ }
+
+ private void handleSendEvent(@NonNull ContentCaptureEvent event, boolean forceFlush) {
+ if (mEvents == null) {
+ if (VERBOSE) {
+ Log.v(TAG, "Creating buffer for " + MAX_BUFFER_SIZE + " events");
+ }
+ mEvents = new ArrayList<>(MAX_BUFFER_SIZE);
+ }
+ mEvents.add(event);
+ final int numberEvents = mEvents.size();
+ if (numberEvents < MAX_BUFFER_SIZE && !forceFlush) {
+ // Buffering events, return right away...
+ return;
+ }
+
+ if (mState != STATE_ACTIVE) {
+ // Callback from startSession hasn't been called yet - typically happens on system
+ // apps that are started before the system service
+ // TODO(b/111276913): try to ignore session while system is not ready / boot
+ // not complete instead. Similarly, the manager service should return right away
+ // when the user does not have a service set
+ if (VERBOSE) {
+ Log.v(TAG, "Closing session for " + getActivityDebugName()
+ + " after " + numberEvents + " delayed events and state "
+ + getStateAsString(mState));
+ }
+ handleResetState();
+ // TODO(b/111276913): blacklist activity / use special flag to indicate that
+ // when it's launched again
+ return;
+ }
+
+ if (mId == null) {
+ // Sanity check - should not happen
+ Log.wtf(TAG, "null session id for " + getActivityDebugName());
+ return;
+ }
+
+ try {
+ if (DEBUG) {
+ Log.d(TAG, "Flushing " + numberEvents + " event(s) for " + getActivityDebugName());
+ }
+ mService.sendEvents(mContext.getUserId(), mId, mEvents);
+ // TODO(b/111276913): decide whether we should clear or set it to null, as each has
+ // its own advantages: clearing will save extra allocations while the session is
+ // active, while setting to null would save memory if there's no more event coming.
+ mEvents.clear();
+ } catch (RemoteException e) {
+ Log.w(TAG, "Error sending " + numberEvents + " for " + getActivityDebugName()
+ + ": " + e);
+ }
+ }
+
+ /**
+ * Used for intermediate events (i.e, other than created and destroyed).
+ *
+ * @hide
+ */
+ public void onActivityLifecycleEvent(@EventType int type) {
+ if (!isContentCaptureEnabled()) return;
+ if (VERBOSE) {
+ Log.v(TAG, "onActivityLifecycleEvent() for " + getActivityDebugName()
+ + ": " + ContentCaptureEvent.getTypeAsString(type));
+ }
+ mHandler.sendMessage(obtainMessage(ContentCaptureManager::handleSendEvent, this,
+ new ContentCaptureEvent(type), /* forceFlush= */ true));
+ }
+
+ /** @hide */
+ public void onActivityDestroyed() {
+ if (!isContentCaptureEnabled()) return;
+
+ //TODO(b/111276913): check state (for example, how to handle if it's waiting for remote
+ // id) and send it to the cache of batched commands
+ if (VERBOSE) {
+ Log.v(TAG, "onActivityDestroyed(): state=" + getStateAsString(mState)
+ + ", mId=" + mId);
+ }
+
+ mHandler.sendMessage(obtainMessage(ContentCaptureManager::handleFinishSession, this));
+ }
+
+ private void handleFinishSession() {
+ //TODO(b/111276913): right now both the ContentEvents and lifecycle sessions are sent
+ // to system_server, so it's ok to call both in sequence here. But once we split
+ // them so the events are sent directly to the service, we need to make sure they're
+ // sent in order.
+ try {
+ if (DEBUG) {
+ Log.d(TAG, "Finishing session " + mId + " with "
+ + (mEvents == null ? 0 : mEvents.size()) + " event(s) for "
+ + getActivityDebugName());
+ }
+
+ mService.finishSession(mContext.getUserId(), mId, mEvents);
+ } catch (RemoteException e) {
+ Log.e(TAG, "Error finishing session " + mId + " for " + getActivityDebugName()
+ + ": " + e);
+ } finally {
+ handleResetState();
+ }
+ }
+
+ private void handleResetState() {
+ mState = STATE_UNKNOWN;
+ mId = null;
+ mApplicationToken = null;
+ mComponentName = null;
+ mEvents = null;
+ }
+
+ /**
+ * Notifies the Intelligence Service that a node has been added to the view structure.
+ *
+ * <p>Typically called "manually" by views that handle their own virtual view hierarchy, or
+ * automatically by the Android System for views that return {@code true} on
+ * {@link View#onProvideContentCaptureStructure(ViewStructure, int)}.
+ *
+ * @param node node that has been added.
+ */
+ public void notifyViewAppeared(@NonNull ViewStructure node) {
+ Preconditions.checkNotNull(node);
+ if (!isContentCaptureEnabled()) return;
+
+ if (!(node instanceof ViewNode.ViewStructureImpl)) {
+ throw new IllegalArgumentException("Invalid node class: " + node.getClass());
+ }
+
+ mHandler.sendMessage(obtainMessage(ContentCaptureManager::handleSendEvent, this,
+ new ContentCaptureEvent(TYPE_VIEW_APPEARED)
+ .setViewNode(((ViewNode.ViewStructureImpl) node).mNode),
+ /* forceFlush= */ false));
+ }
+
+ /**
+ * Notifies the Intelligence Service that a node has been removed from the view structure.
+ *
+ * <p>Typically called "manually" by views that handle their own virtual view hierarchy, or
+ * automatically by the Android System for standard views.
+ *
+ * @param id id of the node that has been removed.
+ */
+ public void notifyViewDisappeared(@NonNull AutofillId id) {
+ Preconditions.checkNotNull(id);
+ if (!isContentCaptureEnabled()) return;
+
+ mHandler.sendMessage(obtainMessage(ContentCaptureManager::handleSendEvent, this,
+ new ContentCaptureEvent(TYPE_VIEW_DISAPPEARED).setAutofillId(id),
+ /* forceFlush= */ false));
+ }
+
+ /**
+ * Notifies the Intelligence Service that the value of a text node has been changed.
+ *
+ * @param id of the node.
+ * @param text new text.
+ * @param flags either {@code 0} or {@link #FLAG_USER_INPUT} when the value was explicitly
+ * changed by the user (for example, through the keyboard).
+ */
+ public void notifyViewTextChanged(@NonNull AutofillId id, @Nullable CharSequence text,
+ int flags) {
+ Preconditions.checkNotNull(id);
+
+ if (!isContentCaptureEnabled()) return;
+
+ mHandler.sendMessage(obtainMessage(ContentCaptureManager::handleSendEvent, this,
+ new ContentCaptureEvent(TYPE_VIEW_TEXT_CHANGED, flags).setAutofillId(id)
+ .setText(text), /* forceFlush= */ false));
+ }
+
+ /**
+ * Creates a {@link ViewStructure} for a "standard" view.
+ *
+ * @hide
+ */
+ @NonNull
+ public ViewStructure newViewStructure(@NonNull View view) {
+ return new ViewNode.ViewStructureImpl(view);
+ }
+
+ /**
+ * Creates a {@link ViewStructure} for a "virtual" view, so it can be passed to
+ * {@link #notifyViewAppeared(ViewStructure)} by the view managing the virtual view hierarchy.
+ *
+ * @param parentId id of the virtual view parent (it can be obtained by calling
+ * {@link ViewStructure#getAutofillId()} on the parent).
+ * @param virtualId id of the virtual child, relative to the parent.
+ *
+ * @return a new {@link ViewStructure} that can be used for Content Capture purposes.
+ */
+ @NonNull
+ public ViewStructure newVirtualViewStructure(@NonNull AutofillId parentId, int virtualId) {
+ return new ViewNode.ViewStructureImpl(parentId, virtualId);
+ }
+
+ /**
+ * Returns the component name of the system service that is consuming the captured events for
+ * the current user.
+ */
+ @Nullable
+ public ComponentName getServiceComponentName() {
+ //TODO(b/111276913): implement
+ return null;
+ }
+
+ /**
+ * Checks whether content capture is enabled for this activity.
+ */
+ public boolean isContentCaptureEnabled() {
+ return mService != null && !mDisabled.get();
+ }
+
+ /**
+ * Called by apps to explicitly enable or disable content capture.
+ *
+ * <p><b>Note: </b> this call is not persisted accross reboots, so apps should typically call
+ * it on {@link android.app.Activity#onCreate(android.os.Bundle, android.os.PersistableBundle)}.
+ */
+ public void setContentCaptureEnabled(boolean enabled) {
+ //TODO(b/111276913): implement
+ }
+
+ /** @hide */
+ public void dump(String prefix, PrintWriter pw) {
+ pw.print(prefix); pw.println("IntelligenceManager");
+ final String prefix2 = prefix + " ";
+ pw.print(prefix2); pw.print("mContext: "); pw.println(mContext);
+ pw.print(prefix2); pw.print("user: "); pw.println(mContext.getUserId());
+ if (mService != null) {
+ pw.print(prefix2); pw.print("mService: "); pw.println(mService);
+ }
+ pw.print(prefix2); pw.print("mDisabled: "); pw.println(mDisabled.get());
+ pw.print(prefix2); pw.print("isEnabled(): "); pw.println(isContentCaptureEnabled());
+ if (mId != null) {
+ pw.print(prefix2); pw.print("id: "); pw.println(mId);
+ }
+ pw.print(prefix2); pw.print("state: "); pw.print(mState); pw.print(" (");
+ pw.print(getStateAsString(mState)); pw.println(")");
+ if (mApplicationToken != null) {
+ pw.print(prefix2); pw.print("app token: "); pw.println(mApplicationToken);
+ }
+ if (mComponentName != null) {
+ pw.print(prefix2); pw.print("component name: ");
+ pw.println(mComponentName.flattenToShortString());
+ }
+ if (mEvents != null) {
+ final int numberEvents = mEvents.size();
+ pw.print(prefix2); pw.print("buffered events: "); pw.print(numberEvents);
+ pw.print('/'); pw.println(MAX_BUFFER_SIZE);
+ if (VERBOSE && numberEvents > 0) {
+ final String prefix3 = prefix2 + " ";
+ for (int i = 0; i < numberEvents; i++) {
+ final ContentCaptureEvent event = mEvents.get(i);
+ pw.print(prefix3); pw.print(i); pw.print(": "); event.dump(pw);
+ pw.println();
+ }
+ }
+ }
+ }
+
+ /**
+ * Gets a string that can be used to identify the activity on logging statements.
+ */
+ private String getActivityDebugName() {
+ return mComponentName == null ? mContext.getPackageName()
+ : mComponentName.flattenToShortString();
+ }
+
+ @NonNull
+ private static String getStateAsString(int state) {
+ switch (state) {
+ case STATE_UNKNOWN:
+ return "UNKNOWN";
+ case STATE_WAITING_FOR_SERVER:
+ return "WAITING_FOR_SERVER";
+ case STATE_ACTIVE:
+ return "ACTIVE";
+ case STATE_DISABLED:
+ return "DISABLED";
+ default:
+ return "INVALID:" + state;
+ }
+ }
+}