Migrate playlists on OS upgrade
1) During the OS upgrade and MediaProvider database migration, we don't
migrate "FORMAT" column which determines MTP format of the file. This
makes ModernMediaScanner#reconcileAndClean() delete all stale rows
unless FORMAT is MtpConstants.FORMAT_ABSTRACT_AV_PLAYLIST.
2) Database migration doesn't migrate audio_playlists_map table, hence
we lose all playlist->audio_files mapping.
3) In Q, playlist files can be only "virtual" playlists which only
exists in MediaProvider database. We changed this logic in R, we force
create "real" playlist files for all playlists. Playlist files are
source of truth in R, and MediaProvider database is updated from
playlist details from playlist files.
All these together causes playlists to be lost after the OS upgrade from
Q to R.
This CL achieves migration of virtual playlist files on OS upgrade with
following steps:
1) Migrate playlist rows with right playlist file names.
2) Create "real" playlist files for virtaul playlists during the
migration and let ModernMediaScanner resolve the playlist and create
audio_playlists_map entries.
Also adds tests for verifying playlist files are migrated on
LegacyMigration.
Bug: 165904660
Test: LegacyProviderMigrationTest#testLegacy_PlaylistMap
Change-Id: Ib96e63bf33a27679465b05c8baa4acf7cc4977b2
diff --git a/Android.bp b/Android.bp
index e1ac31d..1039d55 100644
--- a/Android.bp
+++ b/Android.bp
@@ -83,6 +83,7 @@
"src/com/android/providers/media/util/HandlerExecutor.java",
"src/com/android/providers/media/util/Logging.java",
"src/com/android/providers/media/util/MimeUtils.java",
+ "src/com/android/providers/media/playlist/*.java",
],
}
diff --git a/src/com/android/providers/media/DatabaseHelper.java b/src/com/android/providers/media/DatabaseHelper.java
index a624d88..5877d3e 100644
--- a/src/com/android/providers/media/DatabaseHelper.java
+++ b/src/com/android/providers/media/DatabaseHelper.java
@@ -22,6 +22,7 @@
import android.content.ContentProviderClient;
import android.content.ContentResolver;
+import android.content.ContentUris;
import android.content.ContentValues;
import android.content.Context;
import android.content.Intent;
@@ -35,6 +36,7 @@
import android.net.Uri;
import android.os.Bundle;
import android.os.Environment;
+import android.os.RemoteException;
import android.os.SystemClock;
import android.os.Trace;
import android.provider.MediaStore;
@@ -58,6 +60,7 @@
import androidx.annotation.Nullable;
import androidx.annotation.VisibleForTesting;
+import com.android.providers.media.playlist.Playlist;
import com.android.providers.media.util.BackgroundThread;
import com.android.providers.media.util.DatabaseUtils;
import com.android.providers.media.util.FileUtils;
@@ -66,6 +69,7 @@
import com.android.providers.media.util.MimeUtils;
import java.io.File;
+import java.io.FileNotFoundException;
import java.io.FilenameFilter;
import java.io.IOException;
import java.lang.annotation.Annotation;
@@ -854,9 +858,41 @@
DatabaseUtils.copyFromCursorToContentValues(column, c, values);
}
+ final String volumePath = FileUtils.extractVolumePath(data);
+
+ // Handle playlist files which may need special handling if
+ // there are no "real" playlist files.
+ final int mediaType = c.getInt(c.getColumnIndex(FileColumns.MEDIA_TYPE));
+ if (!mInternal && volumePath != null &&
+ mediaType == FileColumns.MEDIA_TYPE_PLAYLIST) {
+ File playlistFile = new File(data);
+
+ if (!playlistFile.exists()) {
+ if (LOGV) Log.v(TAG, "Migrating playlist file " + playlistFile);
+
+ // Migrate virtual playlists to a "real" playlist file.
+ // Also change playlist file name and path to adapt to new
+ // default primary directory.
+ String playlistFilePath = data;
+ try {
+ playlistFilePath = migratePlaylistFiles(client,
+ c.getLong(c.getColumnIndex(FileColumns._ID)));
+ // Either migration didn't happen or is not necessary because
+ // playlist file already exists
+ if (playlistFilePath == null) playlistFilePath = data;
+ } catch (Exception e) {
+ // We only have one shot to migrate data, so log and
+ // keep marching forward.
+ Log.wtf(TAG, "Couldn't migrate playlist file " + data);
+ }
+
+ values.put(FileColumns.DATA, playlistFilePath);
+ FileUtils.computeValuesFromData(values, /*isForFuse*/ false);
+ }
+ }
+
// When migrating pending or trashed files, we might need to
// rename them on disk to match new schema
- final String volumePath = FileUtils.extractVolumePath(data);
if (volumePath != null) {
FileUtils.computeDataFromValues(values, new File(volumePath),
/*isForFuse*/ false);
@@ -907,6 +943,135 @@
db.endTransaction();
mMigrationListener.onFinished(client, mVolumeName);
}
+
+ }
+
+ @Nullable
+ private String migratePlaylistFiles(ContentProviderClient client, long playlistId)
+ throws IllegalStateException {
+ final String selection = FileColumns.MEDIA_TYPE + "=" + FileColumns.MEDIA_TYPE_PLAYLIST
+ + " AND " + FileColumns._ID + "=" + playlistId;
+ final String[] projection = new String[]{
+ FileColumns._ID,
+ FileColumns.DATA,
+ MediaColumns.MIME_TYPE,
+ MediaStore.Audio.PlaylistsColumns.NAME,
+ };
+ final Uri queryUri = MediaStore
+ .rewriteToLegacy(MediaStore.Files.getContentUri(mVolumeName));
+
+ try (Cursor cursor = client.query(queryUri, projection, selection, null, null)) {
+ if (!cursor.moveToFirst()) {
+ throw new IllegalStateException("Couldn't find database row for playlist file"
+ + playlistId);
+ }
+
+ final String data = cursor.getString(cursor.getColumnIndex(MediaColumns.DATA));
+ File playlistFile = new File(data);
+ if (playlistFile.exists()) {
+ throw new IllegalStateException("Playlist file exists " + data);
+ }
+
+ String mimeType = cursor.getString(cursor.getColumnIndex(MediaColumns.MIME_TYPE));
+ // Sometimes, playlists in Q may have mimeType as
+ // "application/octet-stream". Ensure that playlist rows have the
+ // right playlist mimeType. These rows will be committed to a file
+ // and hence they should have correct playlist mimeType for
+ // Playlist#write to identify the right child playlist class.
+ if (!MimeUtils.isPlaylistMimeType(mimeType)) {
+ // Playlist files should always have right mimeType, default to
+ // audio/mpegurl when mimeType doesn't match playlist media_type.
+ mimeType = "audio/mpegurl";
+ }
+
+ // If the directory is Playlists/ change the directory to Music/
+ // since defaultPrimary for playlists is Music/. This helps
+ // resolve any future app-compat issues around renaming playlist
+ // files.
+ File parentFile = playlistFile.getParentFile();
+ if (parentFile.getName().equalsIgnoreCase("Playlists")) {
+ parentFile = new File(parentFile.getParentFile(), Environment.DIRECTORY_MUSIC);
+ }
+ final String playlistName = cursor.getString(
+ cursor.getColumnIndex(MediaStore.Audio.PlaylistsColumns.NAME));
+
+ try {
+ // Build playlist file path with a file extension that matches
+ // playlist mimeType.
+ playlistFile = FileUtils.buildUniqueFile(parentFile, mimeType, playlistName);
+ } catch(FileNotFoundException e) {
+ Log.e(TAG, "Couldn't create unique file for " + playlistFile +
+ ", using actual playlist file name", e);
+ }
+
+ final long rowId = cursor.getLong(cursor.getColumnIndex(FileColumns._ID));
+ final Uri playlistMemberUri = MediaStore.rewriteToLegacy(
+ MediaStore.Audio.Playlists.Members.getContentUri(mVolumeName, rowId));
+ createPlaylistFile(client, playlistMemberUri, playlistFile);
+ return playlistFile.getAbsolutePath();
+ } catch (RemoteException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ /**
+ * Creates "real" playlist files on disk from the playlist data from the database.
+ */
+ private void createPlaylistFile(ContentProviderClient client, @NonNull Uri playlistMemberUri,
+ @NonNull File playlistFile) throws IllegalStateException {
+ final String[] projection = new String[] {
+ MediaStore.Audio.Playlists.Members.AUDIO_ID,
+ MediaStore.Audio.Playlists.Members.PLAY_ORDER,
+ };
+
+ final Playlist playlist = new Playlist();
+ // Migrating music->playlist association.
+ try (Cursor c = client.query(playlistMemberUri, projection, null, null,
+ Audio.Playlists.Members.DEFAULT_SORT_ORDER)) {
+ while (c.moveToNext()) {
+ // Write these values to the playlist file
+ final long audioId = c.getLong(0);
+ final int playOrder = c.getInt(1);
+
+ final Uri audioFileUri = MediaStore.rewriteToLegacy(ContentUris.withAppendedId(
+ MediaStore.Files.getContentUri(mVolumeName), audioId));
+ final String audioFilePath = queryForData(client, audioFileUri);
+ if (audioFilePath == null) {
+ // This shouldn't happen, we should always find audio file
+ // unless audio file is removed, and database has stale db
+ // row. However this shouldn't block creating playlist
+ // files;
+ Log.e(TAG, "Couldn't find audio file for " + audioId + ", continuing..");
+ continue;
+ }
+ playlist.add(playOrder, playlistFile.toPath().getParent().
+ relativize(new File(audioFilePath).toPath()));
+ }
+
+ try {
+ writeToPlaylistFileWithRetry(playlistFile, playlist);
+ } catch (IOException e) {
+ // We only have one shot to migrate data, so log and
+ // keep marching forward.
+ Log.wtf(TAG, "Couldn't migrate playlist file " + playlistFile);
+ }
+ } catch (RemoteException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ /**
+ * Return the {@link MediaColumns#DATA} field for the given {@code uri}.
+ */
+ private String queryForData(ContentProviderClient client, @NonNull Uri uri) {
+ try (Cursor c = client.query(uri, new String[] {FileColumns.DATA}, Bundle.EMPTY, null)) {
+ if (c.moveToFirst()) {
+ return c.getString(0);
+ }
+ } catch (Exception e) {
+ Log.wtf(TAG, "Exception occurred while querying for data file for " + uri, e);
+ }
+ return null;
}
/**
@@ -1552,7 +1717,33 @@
}
}
- private static final long RENAME_TIMEOUT = 10 * DateUtils.SECOND_IN_MILLIS;
+ private static final long PASSTHROUGH_WAIT_TIMEOUT = 10 * DateUtils.SECOND_IN_MILLIS;
+
+ /**
+ * When writing to playlist files during migration, the underlying
+ * pass-through view of storage may not be mounted yet, so we're willing
+ * to retry several times before giving up.
+ * The retry logic is mainly added to avoid test flakiness.
+ */
+ private static String writeToPlaylistFileWithRetry(@NonNull File playlistFile,
+ @NonNull Playlist playlist) throws IOException {
+ final long start = SystemClock.elapsedRealtime();
+ while (true) {
+ if (SystemClock.elapsedRealtime() - start > PASSTHROUGH_WAIT_TIMEOUT) {
+ throw new IOException("Passthrough failed to mount");
+ }
+
+ try {
+ playlistFile.getParentFile().mkdirs();
+ playlistFile.createNewFile();
+ playlist.write(playlistFile);
+ } catch (IOException e) {
+ Log.i(TAG, "Failed to migrate playlist file, retrying " + e);
+ }
+ Log.i(TAG, "Waiting for passthrough to be mounted...");
+ SystemClock.sleep(100);
+ }
+ }
/**
* When renaming files during migration, the underlying pass-through view of
@@ -1563,7 +1754,7 @@
throws IOException {
final long start = SystemClock.elapsedRealtime();
while (true) {
- if (SystemClock.elapsedRealtime() - start > RENAME_TIMEOUT) {
+ if (SystemClock.elapsedRealtime() - start > PASSTHROUGH_WAIT_TIMEOUT) {
throw new IOException("Passthrough failed to mount");
}
diff --git a/tests/client/src/com/android/providers/media/client/LegacyProviderMigrationTest.java b/tests/client/src/com/android/providers/media/client/LegacyProviderMigrationTest.java
index 0c1fc65..08b69a1 100644
--- a/tests/client/src/com/android/providers/media/client/LegacyProviderMigrationTest.java
+++ b/tests/client/src/com/android/providers/media/client/LegacyProviderMigrationTest.java
@@ -18,7 +18,9 @@
import static android.provider.MediaStore.rewriteToLegacy;
+import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -26,6 +28,7 @@
import android.content.ContentProviderClient;
import android.content.ContentProviderOperation;
import android.content.ContentResolver;
+import android.content.ContentUris;
import android.content.ContentValues;
import android.content.Context;
import android.content.pm.ProviderInfo;
@@ -96,6 +99,7 @@
private Uri mExternalVideo;
private Uri mExternalImages;
private Uri mExternalDownloads;
+ private Uri mExternalPlaylists;
@Before
public void setUp() throws Exception {
@@ -104,6 +108,10 @@
mExternalVideo = MediaStore.Video.Media.getContentUri(mVolumeName);
mExternalImages = MediaStore.Images.Media.getContentUri(mVolumeName);
mExternalDownloads = MediaStore.Downloads.getContentUri(mVolumeName);
+
+ Uri playlists = MediaStore.Audio.Playlists.getContentUri(mVolumeName);
+ mExternalPlaylists = playlists.buildUpon()
+ .appendQueryParameter("silent", "true").build();
}
private ContentValues generateValues(int mediaType, String mimeType, String dirName) {
@@ -194,15 +202,143 @@
doLegacy(mExternalDownloads, values);
}
- /**
- * Verify that a legacy database with thousands of media entries can be
- * successfully migrated.
- */
@Test
- public void testLegacy_Extreme() throws Exception {
+ public void testLegacy_PlaylistMap() throws Exception {
final Context context = InstrumentationRegistry.getTargetContext();
final UiAutomation ui = InstrumentationRegistry.getInstrumentation().getUiAutomation();
+ final ContentValues audios[] = new ContentValues[] {
+ generateValues(FileColumns.MEDIA_TYPE_AUDIO, "audio/mpeg",
+ Environment.DIRECTORY_MUSIC),
+ generateValues(FileColumns.MEDIA_TYPE_AUDIO, "audio/mpeg",
+ Environment.DIRECTORY_MUSIC),
+ };
+
+ final String playlistMimeType = "audio/mpegurl";
+ final ContentValues playlist = generateValues(FileColumns.MEDIA_TYPE_PLAYLIST,
+ playlistMimeType, "Playlists");
+ final String playlistName = "LegacyPlaylistName_" + System.nanoTime();
+ playlist.put(MediaStore.Audio.PlaylistsColumns.NAME, playlistName);
+ File playlistFile = new File(playlist.getAsString(MediaColumns.DATA));
+
+ playlistFile.delete();
+
+ final ContentValues playlistMap = new ContentValues();
+ playlistMap.put(MediaStore.Audio.Playlists.Members.PLAY_ORDER, 1);
+
+ prepareProviders(context, ui);
+
+ try (ContentProviderClient legacy = context.getContentResolver()
+ .acquireContentProviderClient(MediaStore.AUTHORITY_LEGACY)) {
+
+ // Step 1: Insert the playlist entry into the playlists table.
+ final Uri playlistUri = rewriteToLegacy(legacy.insert(
+ rewriteToLegacy(mExternalPlaylists), playlist));
+ long playlistId = ContentUris.parseId(playlistUri);
+ final Uri playlistMemberUri = MediaStore.rewriteToLegacy(
+ MediaStore.Audio.Playlists.Members.getContentUri(mVolumeName, playlistId)
+ .buildUpon()
+ .appendQueryParameter("silent", "true").build());
+
+
+ for (ContentValues values : audios) {
+ // Step 2: Write the audio file to the legacy mediastore.
+ final Uri audioUri =
+ rewriteToLegacy(legacy.insert(rewriteToLegacy(mExternalAudio), values));
+ // Remember our ID to check it later
+ values.put(MediaColumns._ID, audioUri.getLastPathSegment());
+
+
+ long audioId = ContentUris.parseId(audioUri);
+ playlistMap.put(MediaStore.Audio.Playlists.Members.PLAYLIST_ID, playlistId);
+ playlistMap.put(MediaStore.Audio.Playlists.Members.AUDIO_ID, audioId);
+
+ // Step 3: Add a mapping to playlist members.
+ legacy.insert(playlistMemberUri, playlistMap);
+ }
+
+ // Insert a stale row, We only have 3 items in the database. #4 is a stale row
+ // and will be skipped from the playlist during the migration.
+ playlistMap.put(MediaStore.Audio.Playlists.Members.AUDIO_ID, 4);
+ legacy.insert(playlistMemberUri, playlistMap);
+
+ }
+
+ // This will delete MediaProvider data and restarts MediaProvider, and mounts storage.
+ clearProviders(context, ui);
+
+ // Verify scan on DEMAND doesn't delete any virtual playlist files.
+ MediaStore.scanFile(context.getContentResolver(),
+ Environment.getExternalStorageDirectory());
+
+ // Playlist files are created from playlist NAME
+ final File musicDir = new File(context.getSystemService(StorageManager.class)
+ .getStorageVolume(MediaStore.Files.getContentUri(mVolumeName)).getDirectory(),
+ Environment.DIRECTORY_MUSIC);
+ playlistFile = new File(musicDir, playlistName + "."
+ + MimeTypeMap.getSingleton().getExtensionFromMimeType(playlistMimeType));
+ // Wait for scan on MEDIA_MOUNTED to create "real" playlist files.
+ pollForFile(playlistFile);
+
+ // Scan again to verify updated playlist metadata
+ MediaStore.scanFile(context.getContentResolver(), playlistFile);
+
+ try (ContentProviderClient modern = context.getContentResolver()
+ .acquireContentProviderClient(MediaStore.AUTHORITY)) {
+ long legacyPlaylistId =
+ playlistMap.getAsLong(MediaStore.Audio.Playlists.Members.PLAYLIST_ID);
+ long legacyAudioId1 = audios[0].getAsLong(MediaColumns._ID);
+ long legacyAudioId2 = audios[1].getAsLong(MediaColumns._ID);
+
+ // Verify that playlist_id matches with legacy playlist_id
+ {
+ Uri playlists = MediaStore.Audio.Playlists.getContentUri(mVolumeName);
+ final String[] project = {FileColumns._ID, MediaStore.Audio.PlaylistsColumns.NAME};
+
+ try (Cursor cursor = modern.query(playlists, project, null, null, null)) {
+ boolean found = false;
+ while(cursor.moveToNext()) {
+ if (cursor.getLong(0) == legacyPlaylistId) {
+ found = true;
+ assertEquals(playlistName, cursor.getString(1));
+ break;
+ }
+ }
+ assertTrue(found);
+ }
+ }
+
+ // Verify that playlist_members map matches legacy playlist_members map.
+ {
+ Uri members = MediaStore.Audio.Playlists.Members.getContentUri(
+ mVolumeName, legacyPlaylistId);
+ final String[] project = { MediaStore.Audio.Playlists.Members.AUDIO_ID };
+
+ try (Cursor cursor = modern.query(members, project, null, null,
+ MediaStore.Audio.Playlists.Members.DEFAULT_SORT_ORDER)) {
+ assertTrue(cursor.moveToNext());
+ assertEquals(legacyAudioId1, cursor.getLong(0));
+ assertTrue(cursor.moveToNext());
+ assertEquals(legacyAudioId2, cursor.getLong(0));
+ assertFalse(cursor.moveToNext());
+ }
+ }
+
+ // Verify that migrated playlist audio_id refers to legacy audio file.
+ {
+ Uri modernAudioUri = ContentUris.withAppendedId(
+ MediaStore.Audio.Media.getContentUri(mVolumeName), legacyAudioId1);
+ final String[] project = {FileColumns.DATA};
+
+ try (Cursor cursor = modern.query(modernAudioUri, project, null, null, null)) {
+ assertTrue(cursor.moveToFirst());
+ assertEquals(audios[0].getAsString(MediaColumns.DATA), cursor.getString(0));
+ }
+ }
+ }
+ }
+
+ private static void prepareProviders(Context context, UiAutomation ui) throws Exception {
final ProviderInfo legacyProvider = context.getPackageManager()
.resolveContentProvider(MediaStore.AUTHORITY_LEGACY, 0);
final ProviderInfo modernProvider = context.getPackageManager()
@@ -217,6 +353,30 @@
executeShellCommand("sync", ui);
executeShellCommand("pm clear " + legacyProvider.applicationInfo.packageName, ui);
waitForMountedAndIdle(context.getContentResolver());
+ }
+
+ private static void clearProviders(Context context, UiAutomation ui) throws Exception {
+ final ProviderInfo modernProvider = context.getPackageManager()
+ .resolveContentProvider(MediaStore.AUTHORITY, 0);
+
+ // Clear data on the modern provider so that the initial scan recovers
+ // metadata from the legacy provider
+ waitForMountedAndIdle(context.getContentResolver());
+ executeShellCommand("sync", ui);
+ executeShellCommand("pm clear " + modernProvider.applicationInfo.packageName, ui);
+ waitForMountedAndIdle(context.getContentResolver());
+ }
+
+ /**
+ * Verify that a legacy database with thousands of media entries can be
+ * successfully migrated.
+ */
+ @Test
+ public void testLegacy_Extreme() throws Exception {
+ final Context context = InstrumentationRegistry.getTargetContext();
+ final UiAutomation ui = InstrumentationRegistry.getInstrumentation().getUiAutomation();
+
+ prepareProviders(context, ui);
// Create thousands of items in the legacy provider
try (ContentProviderClient legacy = context.getContentResolver()
@@ -241,12 +401,7 @@
}
}
- // Clear data on the modern provider so that the initial scan recovers
- // metadata from the legacy provider
- waitForMountedAndIdle(context.getContentResolver());
- executeShellCommand("sync", ui);
- executeShellCommand("pm clear " + modernProvider.applicationInfo.packageName, ui);
- waitForMountedAndIdle(context.getContentResolver());
+ clearProviders(context, ui);
// Confirm that details from legacy provider have migrated
try (ContentProviderClient modern = context.getContentResolver()
@@ -261,20 +416,7 @@
final Context context = InstrumentationRegistry.getTargetContext();
final UiAutomation ui = InstrumentationRegistry.getInstrumentation().getUiAutomation();
- final ProviderInfo legacyProvider = context.getPackageManager()
- .resolveContentProvider(MediaStore.AUTHORITY_LEGACY, 0);
- final ProviderInfo modernProvider = context.getPackageManager()
- .resolveContentProvider(MediaStore.AUTHORITY, 0);
-
- // Only continue if we have both providers to test against
- Assume.assumeNotNull(legacyProvider);
- Assume.assumeNotNull(modernProvider);
-
- // Clear data on the legacy provider so that we create a database
- waitForMountedAndIdle(context.getContentResolver());
- executeShellCommand("sync", ui);
- executeShellCommand("pm clear " + legacyProvider.applicationInfo.packageName, ui);
- waitForMountedAndIdle(context.getContentResolver());
+ prepareProviders(context, ui);
// Create a well-known entry in legacy provider, and write data into
// place to ensure the file is created on disk
@@ -296,12 +438,7 @@
values.remove(FileColumns.DATA);
}
- // Clear data on the modern provider so that the initial scan recovers
- // metadata from the legacy provider
- waitForMountedAndIdle(context.getContentResolver());
- executeShellCommand("sync", ui);
- executeShellCommand("pm clear " + modernProvider.applicationInfo.packageName, ui);
- waitForMountedAndIdle(context.getContentResolver());
+ clearProviders(context, ui);
// And force a scan to confirm upgraded data survives
MediaStore.scanVolume(context.getContentResolver(),
@@ -341,6 +478,16 @@
MediaStore.waitForIdle(resolver);
}
+ private static void pollForFile(File file) {
+ for (int i = 0; i < POLLING_TIMEOUT_MILLIS / POLLING_SLEEP_MILLIS; i++) {
+ if (file.exists()) return;
+
+ Log.v(TAG, "Waiting for..." + file);
+ SystemClock.sleep(POLLING_SLEEP_MILLIS);
+ }
+ fail("Timed out while waiting for file " + file);
+ }
+
private static void pollForExternalStorageState() {
final File target = Environment.getExternalStorageDirectory();
for (int i = 0; i < POLLING_TIMEOUT_MILLIS / POLLING_SLEEP_MILLIS; i++) {