diff --git a/src/com/android/ex/chips/RecipientAlternatesAdapter.java b/src/com/android/ex/chips/RecipientAlternatesAdapter.java
index 6569d36..35ff2a7 100644
--- a/src/com/android/ex/chips/RecipientAlternatesAdapter.java
+++ b/src/com/android/ex/chips/RecipientAlternatesAdapter.java
@@ -411,7 +411,7 @@
             selection.append(Queries.PHONE.getProjection()[Queries.Query.CONTACT_ID]);
             selection.append(" = ?");
 
-            if (lookupKey == null) {
+            if (directoryId == null || lookupKey == null) {
                 uri = Queries.PHONE.getContentUri();
                 desiredMimeType = null;
             } else {