Tony Mak | 09db2ea | 2018-06-27 18:12:48 +0100 | [diff] [blame] | 1 | /** |
| 2 | * Copyright (C) 2018 The Android Open Source Project |
| 3 | * |
| 4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | * you may not use this file except in compliance with the License. |
| 6 | * You may obtain a copy of the License at |
| 7 | * |
| 8 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | * |
| 10 | * Unless required by applicable law or agreed to in writing, software |
| 11 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | * See the License for the specific language governing permissions and |
| 14 | * limitations under the License. |
| 15 | */ |
| 16 | package android.ext.services.notification; |
| 17 | |
| 18 | import android.annotation.NonNull; |
| 19 | import android.annotation.Nullable; |
| 20 | import android.app.Notification; |
| 21 | import android.app.RemoteAction; |
| 22 | import android.content.Context; |
| 23 | import android.os.Bundle; |
| 24 | import android.os.Parcelable; |
Tony Mak | c68b280 | 2018-07-13 15:59:29 +0100 | [diff] [blame] | 25 | import android.os.Process; |
Tony Mak | 09db2ea | 2018-06-27 18:12:48 +0100 | [diff] [blame] | 26 | import android.text.TextUtils; |
| 27 | import android.util.ArrayMap; |
Tony Mak | d287485 | 2018-10-23 14:27:40 +0100 | [diff] [blame] | 28 | import android.view.textclassifier.ConversationActions; |
Tony Mak | 09db2ea | 2018-06-27 18:12:48 +0100 | [diff] [blame] | 29 | import android.view.textclassifier.TextClassification; |
| 30 | import android.view.textclassifier.TextClassificationManager; |
| 31 | import android.view.textclassifier.TextClassifier; |
| 32 | import android.view.textclassifier.TextLinks; |
| 33 | |
Tony Mak | 09db2ea | 2018-06-27 18:12:48 +0100 | [diff] [blame] | 34 | import java.util.ArrayList; |
| 35 | import java.util.Collections; |
Tony Mak | d287485 | 2018-10-23 14:27:40 +0100 | [diff] [blame] | 36 | import java.util.List; |
| 37 | import java.util.stream.Collectors; |
Tony Mak | 09db2ea | 2018-06-27 18:12:48 +0100 | [diff] [blame] | 38 | |
| 39 | public class SmartActionsHelper { |
Tony Mak | c9acf67 | 2018-07-20 13:58:24 +0200 | [diff] [blame] | 40 | private static final ArrayList<Notification.Action> EMPTY_ACTION_LIST = new ArrayList<>(); |
| 41 | private static final ArrayList<CharSequence> EMPTY_REPLY_LIST = new ArrayList<>(); |
Tony Mak | 09db2ea | 2018-06-27 18:12:48 +0100 | [diff] [blame] | 42 | |
| 43 | // If a notification has any of these flags set, it's inelgibile for actions being added. |
| 44 | private static final int FLAG_MASK_INELGIBILE_FOR_ACTIONS = |
| 45 | Notification.FLAG_ONGOING_EVENT |
| 46 | | Notification.FLAG_FOREGROUND_SERVICE |
| 47 | | Notification.FLAG_GROUP_SUMMARY |
| 48 | | Notification.FLAG_NO_CLEAR; |
| 49 | private static final int MAX_ACTION_EXTRACTION_TEXT_LENGTH = 400; |
| 50 | private static final int MAX_ACTIONS_PER_LINK = 1; |
| 51 | private static final int MAX_SMART_ACTIONS = Notification.MAX_ACTION_BUTTONS; |
Tony Mak | d287485 | 2018-10-23 14:27:40 +0100 | [diff] [blame] | 52 | private static final int MAX_SUGGESTED_REPLIES = 3; |
Tony Mak | 09db2ea | 2018-06-27 18:12:48 +0100 | [diff] [blame] | 53 | |
Tony Mak | d287485 | 2018-10-23 14:27:40 +0100 | [diff] [blame] | 54 | private static final ConversationActions.TypeConfig TYPE_CONFIG = |
| 55 | new ConversationActions.TypeConfig.Builder().setIncludedTypes( |
| 56 | Collections.singletonList(ConversationActions.TYPE_TEXT_REPLY)) |
| 57 | .includeTypesFromTextClassifier(false) |
| 58 | .build(); |
| 59 | private static final List<String> HINTS = |
| 60 | Collections.singletonList(ConversationActions.HINT_FOR_NOTIFICATION); |
| 61 | |
| 62 | SmartActionsHelper() { |
| 63 | } |
Tony Mak | 09db2ea | 2018-06-27 18:12:48 +0100 | [diff] [blame] | 64 | |
| 65 | /** |
| 66 | * Adds action adjustments based on the notification contents. |
| 67 | * |
| 68 | * TODO: Once we have a API in {@link TextClassificationManager} to predict smart actions |
| 69 | * from notification text / message, we can replace most of the code here by consuming that API. |
| 70 | */ |
| 71 | @NonNull |
| 72 | ArrayList<Notification.Action> suggestActions( |
Julia Reynolds | 901bf28 | 2018-08-14 10:09:36 -0400 | [diff] [blame] | 73 | @Nullable Context context, @NonNull NotificationEntry entry) { |
| 74 | if (!isEligibleForActionAdjustment(entry)) { |
Tony Mak | c9acf67 | 2018-07-20 13:58:24 +0200 | [diff] [blame] | 75 | return EMPTY_ACTION_LIST; |
Tony Mak | 09db2ea | 2018-06-27 18:12:48 +0100 | [diff] [blame] | 76 | } |
| 77 | if (context == null) { |
Tony Mak | c9acf67 | 2018-07-20 13:58:24 +0200 | [diff] [blame] | 78 | return EMPTY_ACTION_LIST; |
Tony Mak | 09db2ea | 2018-06-27 18:12:48 +0100 | [diff] [blame] | 79 | } |
| 80 | TextClassificationManager tcm = context.getSystemService(TextClassificationManager.class); |
| 81 | if (tcm == null) { |
Tony Mak | c9acf67 | 2018-07-20 13:58:24 +0200 | [diff] [blame] | 82 | return EMPTY_ACTION_LIST; |
Tony Mak | 09db2ea | 2018-06-27 18:12:48 +0100 | [diff] [blame] | 83 | } |
Julia Reynolds | 901bf28 | 2018-08-14 10:09:36 -0400 | [diff] [blame] | 84 | Notification.Action[] actions = entry.getNotification().actions; |
Tony Mak | 09db2ea | 2018-06-27 18:12:48 +0100 | [diff] [blame] | 85 | int numOfExistingActions = actions == null ? 0: actions.length; |
| 86 | int maxSmartActions = MAX_SMART_ACTIONS - numOfExistingActions; |
| 87 | return suggestActionsFromText( |
| 88 | tcm, |
Julia Reynolds | 901bf28 | 2018-08-14 10:09:36 -0400 | [diff] [blame] | 89 | getMostSalientActionText(entry.getNotification()), maxSmartActions); |
Tony Mak | 09db2ea | 2018-06-27 18:12:48 +0100 | [diff] [blame] | 90 | } |
| 91 | |
Tony Mak | c9acf67 | 2018-07-20 13:58:24 +0200 | [diff] [blame] | 92 | ArrayList<CharSequence> suggestReplies( |
Julia Reynolds | 901bf28 | 2018-08-14 10:09:36 -0400 | [diff] [blame] | 93 | @Nullable Context context, @NonNull NotificationEntry entry) { |
| 94 | if (!isEligibleForReplyAdjustment(entry)) { |
Tony Mak | c9acf67 | 2018-07-20 13:58:24 +0200 | [diff] [blame] | 95 | return EMPTY_REPLY_LIST; |
| 96 | } |
| 97 | if (context == null) { |
| 98 | return EMPTY_REPLY_LIST; |
| 99 | } |
Tony Mak | d287485 | 2018-10-23 14:27:40 +0100 | [diff] [blame] | 100 | TextClassificationManager tcm = context.getSystemService(TextClassificationManager.class); |
| 101 | if (tcm == null) { |
| 102 | return EMPTY_REPLY_LIST; |
| 103 | } |
| 104 | CharSequence text = getMostSalientActionText(entry.getNotification()); |
| 105 | ConversationActions.Message message = |
| 106 | new ConversationActions.Message.Builder() |
| 107 | .setText(text) |
| 108 | .build(); |
| 109 | |
| 110 | ConversationActions.Request request = |
| 111 | new ConversationActions.Request.Builder(Collections.singletonList(message)) |
| 112 | .setMaxSuggestions(MAX_SUGGESTED_REPLIES) |
| 113 | .setHints(HINTS) |
| 114 | .setTypeConfig(TYPE_CONFIG) |
| 115 | .build(); |
| 116 | |
| 117 | TextClassifier textClassifier = tcm.getTextClassifier(); |
| 118 | List<ConversationActions.ConversationAction> conversationActions = |
| 119 | textClassifier.suggestConversationActions(request).getConversationActions(); |
| 120 | |
| 121 | return conversationActions.stream() |
| 122 | .map(conversationAction -> conversationAction.getTextReply()) |
| 123 | .filter(textReply -> !TextUtils.isEmpty(textReply)) |
| 124 | .collect(Collectors.toCollection(ArrayList::new)); |
Tony Mak | c9acf67 | 2018-07-20 13:58:24 +0200 | [diff] [blame] | 125 | } |
| 126 | |
Tony Mak | 09db2ea | 2018-06-27 18:12:48 +0100 | [diff] [blame] | 127 | /** |
| 128 | * Returns whether a notification is eligible for action adjustments. |
| 129 | * |
| 130 | * <p>We exclude system notifications, those that get refreshed frequently, or ones that relate |
| 131 | * to fundamental phone functionality where any error would result in a very negative user |
| 132 | * experience. |
| 133 | */ |
Julia Reynolds | 901bf28 | 2018-08-14 10:09:36 -0400 | [diff] [blame] | 134 | private boolean isEligibleForActionAdjustment(@NonNull NotificationEntry entry) { |
| 135 | Notification notification = entry.getNotification(); |
| 136 | String pkg = entry.getSbn().getPackageName(); |
| 137 | if (!Process.myUserHandle().equals(entry.getSbn().getUser())) { |
Tony Mak | c68b280 | 2018-07-13 15:59:29 +0100 | [diff] [blame] | 138 | return false; |
| 139 | } |
Tony Mak | 09db2ea | 2018-06-27 18:12:48 +0100 | [diff] [blame] | 140 | if (notification.actions != null |
| 141 | && notification.actions.length >= Notification.MAX_ACTION_BUTTONS) { |
| 142 | return false; |
| 143 | } |
Julia Reynolds | 901bf28 | 2018-08-14 10:09:36 -0400 | [diff] [blame] | 144 | if ((notification.flags & FLAG_MASK_INELGIBILE_FOR_ACTIONS) != 0) { |
Tony Mak | 09db2ea | 2018-06-27 18:12:48 +0100 | [diff] [blame] | 145 | return false; |
| 146 | } |
| 147 | if (TextUtils.isEmpty(pkg) || pkg.equals("android")) { |
| 148 | return false; |
| 149 | } |
| 150 | // For now, we are only interested in messages. |
Julia Reynolds | 901bf28 | 2018-08-14 10:09:36 -0400 | [diff] [blame] | 151 | return entry.isMessaging(); |
Tony Mak | 9145fb6 | 2018-07-13 13:48:53 +0100 | [diff] [blame] | 152 | } |
| 153 | |
Julia Reynolds | 901bf28 | 2018-08-14 10:09:36 -0400 | [diff] [blame] | 154 | private boolean isEligibleForReplyAdjustment(@NonNull NotificationEntry entry) { |
Tony Mak | d287485 | 2018-10-23 14:27:40 +0100 | [diff] [blame] | 155 | if (!Process.myUserHandle().equals(entry.getSbn().getUser())) { |
Tony Mak | c9acf67 | 2018-07-20 13:58:24 +0200 | [diff] [blame] | 156 | return false; |
| 157 | } |
Tony Mak | d287485 | 2018-10-23 14:27:40 +0100 | [diff] [blame] | 158 | String pkg = entry.getSbn().getPackageName(); |
| 159 | if (TextUtils.isEmpty(pkg) || pkg.equals("android")) { |
Tony Mak | c9acf67 | 2018-07-20 13:58:24 +0200 | [diff] [blame] | 160 | return false; |
| 161 | } |
Tony Mak | d287485 | 2018-10-23 14:27:40 +0100 | [diff] [blame] | 162 | // For now, we are only interested in messages. |
| 163 | if (!entry.isMessaging()) { |
| 164 | return false; |
| 165 | } |
| 166 | // Does not make sense to provide suggested replies if it is not something that can be |
| 167 | // replied. |
| 168 | if (!entry.hasInlineReply()) { |
| 169 | return false; |
| 170 | } |
| 171 | return true; |
Tony Mak | 09db2ea | 2018-06-27 18:12:48 +0100 | [diff] [blame] | 172 | } |
| 173 | |
| 174 | /** Returns the text most salient for action extraction in a notification. */ |
| 175 | @Nullable |
| 176 | private CharSequence getMostSalientActionText(@NonNull Notification notification) { |
| 177 | /* If it's messaging style, use the most recent message. */ |
Tony Mak | d287485 | 2018-10-23 14:27:40 +0100 | [diff] [blame] | 178 | // TODO: Use the last few X messages instead and take the Person object into consideration. |
Tony Mak | 09db2ea | 2018-06-27 18:12:48 +0100 | [diff] [blame] | 179 | Parcelable[] messages = notification.extras.getParcelableArray(Notification.EXTRA_MESSAGES); |
| 180 | if (messages != null && messages.length != 0) { |
| 181 | Bundle lastMessage = (Bundle) messages[messages.length - 1]; |
| 182 | CharSequence lastMessageText = |
| 183 | lastMessage.getCharSequence(Notification.MessagingStyle.Message.KEY_TEXT); |
| 184 | if (!TextUtils.isEmpty(lastMessageText)) { |
| 185 | return lastMessageText; |
| 186 | } |
| 187 | } |
| 188 | |
| 189 | // Fall back to using the normal text. |
| 190 | return notification.extras.getCharSequence(Notification.EXTRA_TEXT); |
| 191 | } |
| 192 | |
| 193 | /** Returns a list of actions to act on entities in a given piece of text. */ |
| 194 | @NonNull |
| 195 | private ArrayList<Notification.Action> suggestActionsFromText( |
| 196 | @NonNull TextClassificationManager tcm, @Nullable CharSequence text, |
| 197 | int maxSmartActions) { |
| 198 | if (TextUtils.isEmpty(text)) { |
Tony Mak | c9acf67 | 2018-07-20 13:58:24 +0200 | [diff] [blame] | 199 | return EMPTY_ACTION_LIST; |
Tony Mak | 09db2ea | 2018-06-27 18:12:48 +0100 | [diff] [blame] | 200 | } |
| 201 | TextClassifier textClassifier = tcm.getTextClassifier(); |
| 202 | |
| 203 | // We want to process only text visible to the user to avoid confusing suggestions, so we |
| 204 | // truncate the text to a reasonable length. This is particularly important for e.g. |
| 205 | // email apps that sometimes include the text for the entire thread. |
| 206 | text = text.subSequence(0, Math.min(text.length(), MAX_ACTION_EXTRACTION_TEXT_LENGTH)); |
| 207 | |
| 208 | // Extract all entities. |
| 209 | TextLinks.Request textLinksRequest = new TextLinks.Request.Builder(text) |
| 210 | .setEntityConfig( |
| 211 | TextClassifier.EntityConfig.createWithHints( |
| 212 | Collections.singletonList( |
| 213 | TextClassifier.HINT_TEXT_IS_NOT_EDITABLE))) |
| 214 | .build(); |
| 215 | TextLinks links = textClassifier.generateLinks(textLinksRequest); |
| 216 | ArrayMap<String, Integer> entityTypeFrequency = getEntityTypeFrequency(links); |
| 217 | |
| 218 | ArrayList<Notification.Action> actions = new ArrayList<>(); |
| 219 | for (TextLinks.TextLink link : links.getLinks()) { |
| 220 | // Ignore any entity type for which we have too many entities. This is to handle the |
| 221 | // case where a notification contains e.g. a list of phone numbers. In such cases, the |
| 222 | // user likely wants to act on the whole list rather than an individual entity. |
| 223 | if (link.getEntityCount() == 0 |
| 224 | || entityTypeFrequency.get(link.getEntity(0)) != 1) { |
| 225 | continue; |
| 226 | } |
| 227 | |
| 228 | // Generate the actions, and add the most prominent ones to the action bar. |
| 229 | TextClassification classification = |
| 230 | textClassifier.classifyText( |
| 231 | new TextClassification.Request.Builder( |
| 232 | text, link.getStart(), link.getEnd()).build()); |
| 233 | int numOfActions = Math.min( |
| 234 | MAX_ACTIONS_PER_LINK, classification.getActions().size()); |
| 235 | for (int i = 0; i < numOfActions; ++i) { |
| 236 | RemoteAction action = classification.getActions().get(i); |
| 237 | actions.add( |
| 238 | new Notification.Action.Builder( |
| 239 | action.getIcon(), |
| 240 | action.getTitle(), |
| 241 | action.getActionIntent()) |
| 242 | .build()); |
| 243 | // We have enough smart actions. |
| 244 | if (actions.size() >= maxSmartActions) { |
| 245 | return actions; |
| 246 | } |
| 247 | } |
| 248 | } |
| 249 | return actions; |
| 250 | } |
| 251 | |
| 252 | /** |
| 253 | * Given the links extracted from a piece of text, returns the frequency of each entity |
| 254 | * type. |
| 255 | */ |
| 256 | @NonNull |
| 257 | private ArrayMap<String, Integer> getEntityTypeFrequency(@NonNull TextLinks links) { |
| 258 | ArrayMap<String, Integer> entityTypeCount = new ArrayMap<>(); |
| 259 | for (TextLinks.TextLink link : links.getLinks()) { |
| 260 | if (link.getEntityCount() == 0) { |
| 261 | continue; |
| 262 | } |
| 263 | String entityType = link.getEntity(0); |
| 264 | if (entityTypeCount.containsKey(entityType)) { |
| 265 | entityTypeCount.put(entityType, entityTypeCount.get(entityType) + 1); |
| 266 | } else { |
| 267 | entityTypeCount.put(entityType, 1); |
| 268 | } |
| 269 | } |
| 270 | return entityTypeCount; |
| 271 | } |
| 272 | } |