fastboot: add support for auto-resparsing large files
Add support to fastboot for automatically using libsparse to break large
files, whether they are in sparse or normal format, into multiple sparse
files that can each fit into the target's memory. Allows flashing
images that are larger than the size of the available memory on the
target.
By default, any file over 512MB will be sparsed into 512MB chunks. The
limit can be modified with the -m argument, or sparsing can be forced
with -S or avoided with -N. If -m is not specified, the target can
override the default by implementing getvar:max-download-size
Change-Id: I6c59381c3d24475c4f2587ea877200b96971cbd7
diff --git a/fastboot/Android.mk b/fastboot/Android.mk
index 089b9bb..e3261a7 100644
--- a/fastboot/Android.mk
+++ b/fastboot/Android.mk
@@ -48,7 +48,7 @@
LOCAL_C_INCLUDES += development/host/windows/usb/api
endif
-LOCAL_STATIC_LIBRARIES := $(EXTRA_STATIC_LIBS) libzipfile libunz libext4_utils libz
+LOCAL_STATIC_LIBRARIES := $(EXTRA_STATIC_LIBS) libzipfile libunz libext4_utils libsparse libz
ifneq ($(HOST_OS),windows)
ifeq ($(HAVE_SELINUX), true)
diff --git a/fastboot/engine.c b/fastboot/engine.c
index 7dc29e4..93be3de 100644
--- a/fastboot/engine.c
+++ b/fastboot/engine.c
@@ -28,7 +28,6 @@
#include "fastboot.h"
#include "make_ext4fs.h"
-#include "ext4_utils.h"
#include <stdio.h>
#include <stdlib.h>
@@ -77,6 +76,7 @@
#define OP_QUERY 3
#define OP_NOTICE 4
#define OP_FORMAT 5
+#define OP_DOWNLOAD_SPARSE 6
typedef struct Action Action;
@@ -382,6 +382,19 @@
a->msg = mkmsg("writing '%s'", ptn);
}
+void fb_queue_flash_sparse(const char *ptn, struct sparse_file *s, unsigned sz)
+{
+ Action *a;
+
+ a = queue_action(OP_DOWNLOAD_SPARSE, "");
+ a->data = s;
+ a->size = 0;
+ a->msg = mkmsg("sending sparse '%s' (%d KB)", ptn, sz / 1024);
+
+ a = queue_action(OP_COMMAND, "flash:%s", ptn);
+ a->msg = mkmsg("writing '%s'", ptn);
+}
+
static int match(char *str, const char **value, unsigned count)
{
const char *val;
@@ -580,6 +593,10 @@
status = fb_format(a, usb, (int)a->data);
status = a->func(a, status, status ? fb_get_error() : "");
if (status) break;
+ } else if (a->op == OP_DOWNLOAD_SPARSE) {
+ status = fb_download_data_sparse(usb, a->data);
+ status = a->func(a, status, status ? fb_get_error() : "");
+ if (status) break;
} else {
die("bogus action");
}
diff --git a/fastboot/fastboot.c b/fastboot/fastboot.c
index 2dc79e9..ff99173 100644
--- a/fastboot/fastboot.c
+++ b/fastboot/fastboot.c
@@ -26,9 +26,13 @@
* SUCH DAMAGE.
*/
+#define _LARGEFILE64_SOURCE
+
#include <stdio.h>
#include <stdlib.h>
#include <stdarg.h>
+#include <stdbool.h>
+#include <stdint.h>
#include <string.h>
#include <errno.h>
#include <fcntl.h>
@@ -38,11 +42,20 @@
#include <getopt.h>
#include <sys/time.h>
+#include <sys/types.h>
+
#include <bootimg.h>
+#include <sparse/sparse.h>
#include <zipfile/zipfile.h>
#include "fastboot.h"
+#ifndef O_BINARY
+#define O_BINARY 0
+#endif
+
+#define DEFAULT_SPARSE_LIMIT (256 * 1024 * 1024)
+
char cur_product[FB_RESPONSE_SZ + 1];
void bootimg_set_cmdline(boot_img_hdr *h, const char *cmdline);
@@ -59,6 +72,8 @@
static const char *cmdline = 0;
static int wipe_data = 0;
static unsigned short vendor_id = 0;
+static int64_t sparse_limit = -1;
+static int64_t target_sparse_limit = -1;
static unsigned base_addr = 0x10000000;
@@ -117,7 +132,27 @@
#ifdef _WIN32
void *load_file(const char *fn, unsigned *_sz);
+int64_t file_size(const char *fn);
#else
+#if defined(__APPLE__) && defined(__MACH__)
+#define lseek64 lseek
+#define off64_t off_t
+#endif
+
+int64_t file_size(const char *fn)
+{
+ off64_t off;
+ int fd;
+
+ fd = open(fn, O_RDONLY);
+ if (fd < 0) return -1;
+
+ off = lseek64(fd, 0, SEEK_END);
+ close(fd);
+
+ return off;
+}
+
void *load_file(const char *fn, unsigned *_sz)
{
char *data;
@@ -245,6 +280,8 @@
" -i <vendor id> specify a custom USB vendor id\n"
" -b <base_addr> specify a custom kernel base address\n"
" -n <page size> specify the nand page size. default: 2048\n"
+ " -S <size>[K|M|G] automatically sparse files greater than\n"
+ " size. default: 256M, 0 to disable\n"
);
}
@@ -430,6 +467,110 @@
fb_queue_notice("--------------------------------------------");
}
+
+struct sparse_file **load_sparse_files(const char *fname, int max_size)
+{
+ int fd;
+ struct sparse_file *s;
+ int files;
+ struct sparse_file **out_s;
+
+ fd = open(fname, O_RDONLY | O_BINARY);
+ if (fd < 0) {
+ die("cannot open '%s'\n", fname);
+ }
+
+ s = sparse_file_import_auto(fd, false);
+ if (!s) {
+ die("cannot sparse read file '%s'\n", fname);
+ }
+
+ files = sparse_file_resparse(s, max_size, NULL, 0);
+ if (files < 0) {
+ die("Failed to resparse '%s'\n", fname);
+ }
+
+ out_s = calloc(sizeof(struct sparse_file *), files + 1);
+ if (!out_s) {
+ die("Failed to allocate sparse file array\n");
+ }
+
+ files = sparse_file_resparse(s, max_size, out_s, files);
+ if (files < 0) {
+ die("Failed to resparse '%s'\n", fname);
+ }
+
+ return out_s;
+}
+
+static int64_t get_target_sparse_limit(struct usb_handle *usb)
+{
+ int64_t limit = 0;
+ char response[FB_RESPONSE_SZ + 1];
+ int status = fb_getvar(usb, response, "max-download-size");
+
+ if (!status) {
+ limit = strtoul(response, NULL, 0);
+ if (limit > 0) {
+ fprintf(stderr, "target reported max download size of %lld bytes\n",
+ limit);
+ }
+ }
+
+ return limit;
+}
+
+static int64_t get_sparse_limit(struct usb_handle *usb, int64_t size)
+{
+ int64_t limit;
+
+ if (sparse_limit == 0) {
+ return 0;
+ } else if (sparse_limit > 0) {
+ limit = sparse_limit;
+ } else {
+ if (target_sparse_limit == -1) {
+ target_sparse_limit = get_target_sparse_limit(usb);
+ }
+ if (target_sparse_limit > 0) {
+ limit = target_sparse_limit;
+ } else {
+ limit = DEFAULT_SPARSE_LIMIT;
+ }
+ }
+
+ if (size > limit) {
+ return limit;
+ }
+
+ return 0;
+}
+
+void do_flash(usb_handle *usb, const char *pname, const char *fname)
+{
+ int64_t sz64;
+ void *data;
+ int64_t limit;
+
+ sz64 = file_size(fname);
+ limit = get_sparse_limit(usb, sz64);
+ if (limit) {
+ struct sparse_file **s = load_sparse_files(fname, limit);
+ if (s == NULL) {
+ die("cannot sparse load '%s'\n", fname);
+ }
+ while (*s) {
+ sz64 = sparse_file_len(*s, true, false);
+ fb_queue_flash_sparse(pname, *s++, sz64);
+ }
+ } else {
+ unsigned int sz;
+ data = load_file(fname, &sz);
+ if (data == 0) die("cannot load '%s'\n", fname);
+ fb_queue_flash(pname, data, sz);
+ }
+}
+
void do_update_signature(zipfile_t zip, char *fn)
{
void *data;
@@ -567,6 +708,47 @@
return 0;
}
+static int64_t parse_num(const char *arg)
+{
+ char *endptr;
+ unsigned long long num;
+
+ num = strtoull(arg, &endptr, 0);
+ if (endptr == arg) {
+ return -1;
+ }
+
+ if (*endptr == 'k' || *endptr == 'K') {
+ if (num >= (-1ULL) / 1024) {
+ return -1;
+ }
+ num *= 1024LL;
+ endptr++;
+ } else if (*endptr == 'm' || *endptr == 'M') {
+ if (num >= (-1ULL) / (1024 * 1024)) {
+ return -1;
+ }
+ num *= 1024LL * 1024LL;
+ endptr++;
+ } else if (*endptr == 'g' || *endptr == 'G') {
+ if (num >= (-1ULL) / (1024 * 1024 * 1024)) {
+ return -1;
+ }
+ num *= 1024LL * 1024LL * 1024LL;
+ endptr++;
+ }
+
+ if (*endptr != '\0') {
+ return -1;
+ }
+
+ if (num > INT64_MAX) {
+ return -1;
+ }
+
+ return num;
+}
+
int main(int argc, char **argv)
{
int wants_wipe = 0;
@@ -577,13 +759,14 @@
unsigned page_size = 2048;
int status;
int c;
+ int r;
- struct option longopts = { 0, 0, 0, 0 };
+ const struct option longopts = { 0, 0, 0, 0 };
serial = getenv("ANDROID_SERIAL");
while (1) {
- c = getopt_long(argc, argv, "wb:n:s:p:c:i:h", &longopts, NULL);
+ c = getopt_long(argc, argv, "wb:n:s:S:p:c:i:m:h", &longopts, NULL);
if (c < 0) {
break;
}
@@ -602,6 +785,12 @@
case 's':
serial = optarg;
break;
+ case 'S':
+ sparse_limit = parse_num(optarg);
+ if (sparse_limit < 0) {
+ die("invalid sparse limit");
+ }
+ break;
case 'p':
product = optarg;
break;
@@ -702,9 +891,7 @@
skip(2);
}
if (fname == 0) die("cannot determine image filename for '%s'", pname);
- data = load_file(fname, &sz);
- if (data == 0) die("cannot load '%s'\n", fname);
- fb_queue_flash(pname, data, sz);
+ do_flash(usb, pname, fname);
} else if(!strcmp(*argv, "flash:raw")) {
char *pname = argv[1];
char *kname = argv[2];
diff --git a/fastboot/fastboot.h b/fastboot/fastboot.h
index a84b0be..9177932 100644
--- a/fastboot/fastboot.h
+++ b/fastboot/fastboot.h
@@ -31,10 +31,13 @@
#include "usb.h"
+struct sparse_file;
+
/* protocol.c - fastboot protocol */
int fb_command(usb_handle *usb, const char *cmd);
int fb_command_response(usb_handle *usb, const char *cmd, char *response);
int fb_download_data(usb_handle *usb, const void *data, unsigned size);
+int fb_download_data_sparse(usb_handle *usb, struct sparse_file *s);
char *fb_get_error(void);
#define FB_COMMAND_SZ 64
@@ -43,6 +46,7 @@
/* engine.c - high level command queue engine */
int fb_getvar(struct usb_handle *usb, char *response, const char *fmt, ...);
void fb_queue_flash(const char *ptn, void *data, unsigned sz);;
+void fb_queue_flash_sparse(const char *ptn, struct sparse_file *s, unsigned sz);
void fb_queue_erase(const char *ptn);
void fb_queue_format(const char *ptn, int skip_if_not_supported);
void fb_queue_require(const char *prod, const char *var, int invert,
diff --git a/fastboot/protocol.c b/fastboot/protocol.c
index e871113..a0e0fd4 100644
--- a/fastboot/protocol.c
+++ b/fastboot/protocol.c
@@ -26,11 +26,18 @@
* SUCH DAMAGE.
*/
+#define min(a, b) \
+ ({ typeof(a) _a = (a); typeof(b) _b = (b); (_a < _b) ? _a : _b; })
+#define round_down(a, b) \
+ ({ typeof(a) _a = (a); typeof(b) _b = (b); _a - (_a % _b); })
+
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>
+#include <sparse/sparse.h>
+
#include "fastboot.h"
static char ERROR[128];
@@ -40,8 +47,7 @@
return ERROR;
}
-static int check_response(usb_handle *usb, unsigned size,
- unsigned data_okay, char *response)
+static int check_response(usb_handle *usb, unsigned int size, char *response)
{
unsigned char status[65];
int r;
@@ -82,7 +88,7 @@
return -1;
}
- if(!memcmp(status, "DATA", 4) && data_okay){
+ if(!memcmp(status, "DATA", 4) && size > 0){
unsigned dsize = strtoul((char*) status + 4, 0, 16);
if(dsize > size) {
strcpy(ERROR, "data size too large");
@@ -100,9 +106,8 @@
return -1;
}
-static int _command_send(usb_handle *usb, const char *cmd,
- const void *data, unsigned size,
- char *response)
+static int _command_start(usb_handle *usb, const char *cmd, unsigned size,
+ char *response)
{
int cmdsize = strlen(cmd);
int r;
@@ -122,46 +127,81 @@
return -1;
}
- if(data == 0) {
- return check_response(usb, size, 0, response);
+ return check_response(usb, size, response);
+}
+
+static int _command_data(usb_handle *usb, const void *data, unsigned size)
+{
+ int r;
+
+ r = usb_write(usb, data, size);
+ if(r < 0) {
+ sprintf(ERROR, "data transfer failure (%s)", strerror(errno));
+ usb_close(usb);
+ return -1;
+ }
+ if(r != ((int) size)) {
+ sprintf(ERROR, "data transfer failure (short transfer)");
+ usb_close(usb);
+ return -1;
}
- r = check_response(usb, size, 1, 0);
+ return r;
+}
+
+static int _command_end(usb_handle *usb)
+{
+ int r;
+ r = check_response(usb, 0, 0);
if(r < 0) {
return -1;
}
- size = r;
+ return 0;
+}
- if(size) {
- r = usb_write(usb, data, size);
- if(r < 0) {
- sprintf(ERROR, "data transfer failure (%s)", strerror(errno));
- usb_close(usb);
- return -1;
- }
- if(r != ((int) size)) {
- sprintf(ERROR, "data transfer failure (short transfer)");
- usb_close(usb);
- return -1;
- }
+static int _command_send(usb_handle *usb, const char *cmd,
+ const void *data, unsigned size,
+ char *response)
+{
+ int r;
+ if (size == 0) {
+ return -1;
}
- r = check_response(usb, 0, 0, 0);
+ r = _command_start(usb, cmd, size, response);
+ if (r < 0) {
+ return -1;
+ }
+
+ r = _command_data(usb, data, size);
+ if (r < 0) {
+ return -1;
+ }
+
+ r = _command_end(usb);
if(r < 0) {
return -1;
- } else {
- return size;
}
+
+ return size;
+}
+
+static int _command_send_no_data(usb_handle *usb, const char *cmd,
+ char *response)
+{
+ int r;
+
+ return _command_start(usb, cmd, 0, response);
}
int fb_command(usb_handle *usb, const char *cmd)
{
- return _command_send(usb, cmd, 0, 0, 0);
+ return _command_send_no_data(usb, cmd, 0);
}
int fb_command_response(usb_handle *usb, const char *cmd, char *response)
{
- return _command_send(usb, cmd, 0, 0, response);
+ return _command_send_no_data(usb, cmd, response);
}
int fb_download_data(usb_handle *usb, const void *data, unsigned size)
@@ -179,3 +219,96 @@
}
}
+#define USB_BUF_SIZE 512
+static char usb_buf[USB_BUF_SIZE];
+static int usb_buf_len;
+
+static int fb_download_data_sparse_write(void *priv, const void *data, int len)
+{
+ int r;
+ usb_handle *usb = priv;
+ int to_write;
+ const char *ptr = data;
+
+ if (usb_buf_len) {
+ to_write = min(USB_BUF_SIZE - usb_buf_len, len);
+
+ memcpy(usb_buf + usb_buf_len, ptr, to_write);
+ usb_buf_len += to_write;
+ ptr += to_write;
+ len -= to_write;
+ }
+
+ if (usb_buf_len == USB_BUF_SIZE) {
+ r = _command_data(usb, usb_buf, USB_BUF_SIZE);
+ if (r != USB_BUF_SIZE) {
+ return -1;
+ }
+ usb_buf_len = 0;
+ }
+
+ if (len > USB_BUF_SIZE) {
+ if (usb_buf_len > 0) {
+ sprintf(ERROR, "internal error: usb_buf not empty\n");
+ return -1;
+ }
+ to_write = round_down(len, USB_BUF_SIZE);
+ r = _command_data(usb, ptr, to_write);
+ if (r != to_write) {
+ return -1;
+ }
+ ptr += to_write;
+ len -= to_write;
+ }
+
+ if (len > 0) {
+ if (len > USB_BUF_SIZE) {
+ sprintf(ERROR, "internal error: too much left for usb_buf\n");
+ return -1;
+ }
+ memcpy(usb_buf, ptr, len);
+ usb_buf_len = len;
+ }
+
+ return 0;
+}
+
+static int fb_download_data_sparse_flush(usb_handle *usb)
+{
+ int r;
+
+ if (usb_buf_len > 0) {
+ r = _command_data(usb, usb_buf, usb_buf_len);
+ if (r != usb_buf_len) {
+ return -1;
+ }
+ usb_buf_len = 0;
+ }
+
+ return 0;
+}
+
+int fb_download_data_sparse(usb_handle *usb, struct sparse_file *s)
+{
+ char cmd[64];
+ int r;
+ int size = sparse_file_len(s, true, false);
+ if (size <= 0) {
+ return -1;
+ }
+
+ sprintf(cmd, "download:%08x", size);
+ r = _command_start(usb, cmd, size, 0);
+ if (r < 0) {
+ return -1;
+ }
+
+ r = sparse_file_callback(s, true, false, fb_download_data_sparse_write, usb);
+ if (r < 0) {
+ return -1;
+ }
+
+ fb_download_data_sparse_flush(usb);
+
+ return _command_end(usb);
+}
diff --git a/fastboot/util_windows.c b/fastboot/util_windows.c
index c3d545c..9e029fd 100644
--- a/fastboot/util_windows.c
+++ b/fastboot/util_windows.c
@@ -36,6 +36,29 @@
#include <windows.h>
+int64_t file_size(const char *fn)
+{
+ HANDLE file;
+ char *data;
+ DWORD sz;
+
+ file = CreateFile( fn,
+ GENERIC_READ,
+ FILE_SHARE_READ,
+ NULL,
+ OPEN_EXISTING,
+ 0,
+ NULL );
+
+ if (file == INVALID_HANDLE_VALUE)
+ return -1;
+
+ sz = GetFileSize( file, NULL );
+ CloseHandle( file );
+
+ return sz;
+}
+
void get_my_path(char exe[PATH_MAX])
{
char* r;
@@ -52,7 +75,7 @@
{
HANDLE file;
char *data;
- DWORD file_size;
+ DWORD sz;
file = CreateFile( fn,
GENERIC_READ,
@@ -65,29 +88,29 @@
if (file == INVALID_HANDLE_VALUE)
return NULL;
- file_size = GetFileSize( file, NULL );
+ sz = GetFileSize( file, NULL );
data = NULL;
- if (file_size > 0) {
- data = (char*) malloc( file_size );
+ if (sz > 0) {
+ data = (char*) malloc( sz );
if (data == NULL) {
- fprintf(stderr, "load_file: could not allocate %ld bytes\n", file_size );
- file_size = 0;
+ fprintf(stderr, "load_file: could not allocate %ld bytes\n", sz );
+ sz = 0;
} else {
DWORD out_bytes;
- if ( !ReadFile( file, data, file_size, &out_bytes, NULL ) ||
- out_bytes != file_size )
+ if ( !ReadFile( file, data, sz, &out_bytes, NULL ) ||
+ out_bytes != sz )
{
- fprintf(stderr, "load_file: could not read %ld bytes from '%s'\n", file_size, fn);
+ fprintf(stderr, "load_file: could not read %ld bytes from '%s'\n", sz, fn);
free(data);
data = NULL;
- file_size = 0;
+ sz = 0;
}
}
}
CloseHandle( file );
- *_sz = (unsigned) file_size;
+ *_sz = (unsigned) sz;
return data;
}