ext4_utils: Add support for >2G input files
Change-Id: I6af69d676610099d3912e90fcab3cbdc27ace4e2
diff --git a/ext4_utils/allocate.c b/ext4_utils/allocate.c
index 5f3f12b..9377e3c 100644
--- a/ext4_utils/allocate.c
+++ b/ext4_utils/allocate.c
@@ -14,14 +14,14 @@
* limitations under the License.
*/
-#include <stdio.h>
-#include <stdlib.h>
-
#include "ext4_utils.h"
#include "allocate.h"
#include "backed_block.h"
#include "ext4.h"
+#include <stdio.h>
+#include <stdlib.h>
+
struct region_list {
struct region *first;
struct region *last;
diff --git a/ext4_utils/allocate.h b/ext4_utils/allocate.h
index f4bed41..16dc33c 100644
--- a/ext4_utils/allocate.h
+++ b/ext4_utils/allocate.h
@@ -14,6 +14,9 @@
* limitations under the License.
*/
+#ifndef _ALLOCATE_H_
+#define _ALLOCATE_H_
+
#define EXT4_ALLOCATE_FAILED (u32)(~0)
#include "ext4_utils.h"
@@ -49,3 +52,5 @@
u32 block, u32 len, int bg);
struct block_allocation *create_allocation();
int append_oob_allocation(struct block_allocation *alloc, u32 len);
+
+#endif
diff --git a/ext4_utils/backed_block.c b/ext4_utils/backed_block.c
index c1a2f20..82c4bcc 100644
--- a/ext4_utils/backed_block.c
+++ b/ext4_utils/backed_block.c
@@ -14,17 +14,17 @@
* limitations under the License.
*/
-#include <stdlib.h>
-
#include "ext4_utils.h"
#include "backed_block.h"
+#include <stdlib.h>
+
struct data_block {
u32 block;
u32 len;
u8 *data;
const char *filename;
- off_t offset;
+ off64_t offset;
struct data_block *next;
};
@@ -83,7 +83,7 @@
}
/* Queues a chunk of a file on disk to be written to the specified data blocks */
-void queue_data_file(const char *filename, off_t offset, u32 len,
+void queue_data_file(const char *filename, off64_t offset, u32 len,
u32 block)
{
struct data_block *db = malloc(sizeof(struct data_block));
diff --git a/ext4_utils/backed_block.h b/ext4_utils/backed_block.h
index 5304f00..61a1b1c 100644
--- a/ext4_utils/backed_block.h
+++ b/ext4_utils/backed_block.h
@@ -17,19 +17,17 @@
#ifndef _BACKED_BLOCK_H_
#define _BACKED_BLOCK_H_
-#include <sys/types.h>
-#include <unistd.h>
#include "ext4_utils.h"
#include "output_file.h"
typedef void (*data_block_callback_t)(struct output_file *out, u64 off,
u8 *data, int len);
typedef void (*data_block_file_callback_t)(struct output_file *out, u64 off,
- const char *file, off_t offset,
+ const char *file, off64_t offset,
int len);
void queue_data_block(u8 *data, u32 len, u32 block);
-void queue_data_file(const char *filename, off_t offset, u32 len,
+void queue_data_file(const char *filename, off64_t offset, u32 len,
u32 block);
void for_each_data_block(data_block_callback_t data_func,
data_block_file_callback_t file_func, struct output_file *out);
diff --git a/ext4_utils/ext4_utils.c b/ext4_utils/ext4_utils.c
index 82d1116..1551633 100644
--- a/ext4_utils/ext4_utils.c
+++ b/ext4_utils/ext4_utils.c
@@ -14,6 +14,14 @@
* limitations under the License.
*/
+#include "ext4_utils.h"
+#include "output_file.h"
+#include "backed_block.h"
+#include "uuid.h"
+#include "allocate.h"
+#include "indirect.h"
+#include "extent.h"
+
#include <fcntl.h>
#include <arpa/inet.h>
#include <sys/ioctl.h>
@@ -27,14 +35,6 @@
#include <sys/disk.h>
#endif
-#include "ext4_utils.h"
-#include "output_file.h"
-#include "backed_block.h"
-#include "uuid.h"
-#include "allocate.h"
-#include "indirect.h"
-#include "extent.h"
-
#include "ext4.h"
#include "jbd2.h"
@@ -78,7 +78,6 @@
{
int ret = 0;
struct output_file *out = open_output_file(filename, gz, sparse);
- off_t off;
if (!out)
return;
diff --git a/ext4_utils/ext4_utils.h b/ext4_utils/ext4_utils.h
index 712fea6..8a6c3c4 100644
--- a/ext4_utils/ext4_utils.h
+++ b/ext4_utils/ext4_utils.h
@@ -17,6 +17,12 @@
#ifndef _EXT4_UTILS_H_
#define _EXT4_UTILS_H_
+#define _GNU_SOURCE
+#define _FILE_OFFSET_BITS 64
+#define _LARGEFILE64_SOURCE
+#include <sys/types.h>
+#include <unistd.h>
+
#include <sys/types.h>
#include <errno.h>
#include <stdarg.h>
@@ -24,6 +30,20 @@
#include <stdlib.h>
#include <string.h>
+#if defined(__APPLE__) && defined(__MACH__)
+#define lseek64 lseek
+#define off64_t off_t
+#endif
+
+#ifdef __BIONIC__
+extern void* __mmap2(void *, size_t, int, int, int, off_t);
+static inline void *mmap64(void *addr, size_t length, int prot, int flags,
+ int fd, off64_t offset)
+{
+ return __mmap2(addr, length, prot, flags, fd, offset >> 12);
+}
+#endif
+
extern int force;
#define warn(fmt, args...) do { fprintf(stderr, "warning: %s: " fmt "\n", __func__, ## args); } while (0)
diff --git a/ext4_utils/extent.c b/ext4_utils/extent.c
index 1d2581d..daa2541 100644
--- a/ext4_utils/extent.c
+++ b/ext4_utils/extent.c
@@ -14,16 +14,16 @@
* limitations under the License.
*/
-#include <stdlib.h>
-#include <stdio.h>
-
#include "ext4_utils.h"
#include "ext4.h"
#include "ext4_extents.h"
#include "backed_block.h"
-
#include "extent.h"
+#include <stdlib.h>
+#include <stdio.h>
+
+
/* Creates data buffers for the first backing_len bytes of a block allocation
and queues them to be written */
static u8 *extent_create_backing(struct block_allocation *alloc,
@@ -55,7 +55,7 @@
static void extent_create_backing_file(struct block_allocation *alloc,
u64 backing_len, const char *filename)
{
- off_t offset = 0;
+ off64_t offset = 0;
for (; alloc != NULL && backing_len > 0; get_next_region(alloc)) {
u32 region_block;
u32 region_len;
diff --git a/ext4_utils/indirect.c b/ext4_utils/indirect.c
index d776d57..4e768a1 100644
--- a/ext4_utils/indirect.c
+++ b/ext4_utils/indirect.c
@@ -14,9 +14,6 @@
* limitations under the License.
*/
-#include <stdlib.h>
-#include <stdio.h>
-
#include "ext4_utils.h"
#include "ext4.h"
#include "ext4_extents.h"
@@ -24,6 +21,9 @@
#include "indirect.h"
#include "allocate.h"
+#include <stdlib.h>
+#include <stdio.h>
+
/* Creates data buffers for the first backing_len bytes of a block allocation
and queues them to be written */
static u8 *create_backing(struct block_allocation *alloc,
diff --git a/ext4_utils/make_ext4fs.c b/ext4_utils/make_ext4fs.c
index d8eb1a3..cbec2ad 100644
--- a/ext4_utils/make_ext4fs.c
+++ b/ext4_utils/make_ext4fs.c
@@ -14,7 +14,12 @@
* limitations under the License.
*/
-#define _GNU_SOURCE
+#include "make_ext4fs.h"
+#include "output_file.h"
+#include "ext4_utils.h"
+#include "allocate.h"
+#include "contents.h"
+#include "uuid.h"
#include <dirent.h>
#include <libgen.h>
@@ -25,13 +30,6 @@
#include <sys/stat.h>
#include <sys/types.h>
-#include "make_ext4fs.h"
-#include "output_file.h"
-#include "ext4_utils.h"
-#include "allocate.h"
-#include "contents.h"
-#include "uuid.h"
-
#ifdef ANDROID
#include <private/android_filesystem_config.h>
#endif
diff --git a/ext4_utils/output_file.c b/ext4_utils/output_file.c
index 3785b06..efb1307 100644
--- a/ext4_utils/output_file.c
+++ b/ext4_utils/output_file.c
@@ -13,7 +13,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-#define _LARGEFILE64_SOURCE
+
+#include "ext4_utils.h"
+#include "output_file.h"
+#include "sparse_format.h"
+#include "sparse_crc32.h"
#include <sys/types.h>
#include <sys/stat.h>
@@ -24,11 +28,6 @@
#include <zlib.h>
-#include "ext4_utils.h"
-#include "output_file.h"
-#include "sparse_format.h"
-#include "sparse_crc32.h"
-
#if defined(__APPLE__) && defined(__MACH__)
#define lseek64 lseek
#define off64_t off_t
@@ -384,9 +383,11 @@
/* Write a contiguous region of data blocks from a file */
void write_data_file(struct output_file *out, u64 off, const char *file,
- off_t offset, int len)
+ off64_t offset, int len)
{
int ret;
+ off64_t aligned_offset;
+ int aligned_diff;
if (off + len >= info.len) {
error("attempted to write block %llu past end of filesystem",
@@ -400,21 +401,25 @@
return;
}
- u8 *data = mmap(NULL, len, PROT_READ, MAP_SHARED, file_fd, offset);
+ aligned_offset = offset & ~(4096 - 1);
+ aligned_diff = offset - aligned_offset;
+
+ u8 *data = mmap64(NULL, len + aligned_diff, PROT_READ, MAP_SHARED, file_fd,
+ aligned_offset);
if (data == MAP_FAILED) {
- error_errno("mmap");
+ error_errno("mmap64");
close(file_fd);
return;
}
if (out->sparse) {
- write_chunk_raw(out, off, data, len);
+ write_chunk_raw(out, off, data + aligned_diff, len);
} else {
ret = out->ops->seek(out, off);
if (ret < 0)
goto err;
- ret = out->ops->write(out, data, len);
+ ret = out->ops->write(out, data + aligned_diff, len);
if (ret < 0)
goto err;
}
diff --git a/ext4_utils/output_file.h b/ext4_utils/output_file.h
index 82b0952..393bc9b 100644
--- a/ext4_utils/output_file.h
+++ b/ext4_utils/output_file.h
@@ -14,11 +14,16 @@
* limitations under the License.
*/
+#ifndef _OUTPUT_FILE_H_
+#define _OUTPUT_FILE_H_
+
struct output_file;
struct output_file *open_output_file(const char *filename, int gz, int sparse);
void write_data_block(struct output_file *out, u64 off, u8 *data, int len);
void write_data_file(struct output_file *out, u64 off, const char *file,
- off_t offset, int len);
+ off64_t offset, int len);
void pad_output_file(struct output_file *out, u64 len);
void close_output_file(struct output_file *out);
+
+#endif
diff --git a/ext4_utils/simg2img.c b/ext4_utils/simg2img.c
index 88c67b4..9ef2509 100644
--- a/ext4_utils/simg2img.c
+++ b/ext4_utils/simg2img.c
@@ -13,8 +13,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-#define _LARGEFILE64_SOURCE
-#define _FILE_OFFSET_BITS 64
+
+#include "ext4_utils.h"
+#include "output_file.h"
+#include "sparse_format.h"
+#include "sparse_crc32.h"
#include <sys/types.h>
#include <sys/stat.h>
@@ -24,11 +27,6 @@
#include <fcntl.h>
#include <stdio.h>
-#include "ext4_utils.h"
-#include "output_file.h"
-#include "sparse_format.h"
-#include "sparse_crc32.h"
-
#if defined(__APPLE__) && defined(__MACH__)
#define lseek64 lseek
#define off64_t off_t