Allow compressed flash updates

.gitmodules

@@ -19,3 +19,6 @@
 [submodule "tools/esptool"]
 	path = tools/esptool
 	url = https://github.com/espressif/esptool.git
+[submodule "tools/sdk/uzlib"]
+	path = tools/sdk/uzlib
+	url = https://github.com/earlephilhower/uzlib.git

README.md

@@ -124,6 +124,8 @@ ESP8266 core files are licensed under LGPL.
 
 [LittleFS](https://github.com/ARMmbed/littlefs) library written by ARM Limited and released under the [BSD 3-clause license](https://github.com/ARMmbed/littlefs/blob/master/LICENSE.md).
 
+[uzlib](https://github.com/pfalcon/uzlib) library written and (c) 2014-2018 Paul Sokolovsky, licensed under the ZLib license (https://www.zlib.net/zlib_license.html).
+
 ### Other useful links ###
 
 [Toolchain repo](https://github.com/earlephilhower/esp-quick-toolchain)

bootloaders/eboot/Makefile

@@ -6,44 +6,51 @@ TARGET_DIR := ./
 
 TARGET_OBJ_FILES := \
 	eboot.o \
-	eboot_command.o \
-
+	eboot_command.o
 
 TARGET_OBJ_PATHS := $(addprefix $(TARGET_DIR)/,$(TARGET_OBJ_FILES))
 
+UZLIB_PATH := ../../tools/sdk/uzlib/src
+UZLIB_FLAGS := -DRUNTIME_BITS_TABLES
+
 CC := $(XTENSA_TOOLCHAIN)xtensa-lx106-elf-gcc
 CXX := $(XTENSA_TOOLCHAIN)xtensa-lx106-elf-g++
 AR := $(XTENSA_TOOLCHAIN)xtensa-lx106-elf-ar
 LD := $(XTENSA_TOOLCHAIN)xtensa-lx106-elf-gcc
 OBJDUMP := $(XTENSA_TOOLCHAIN)xtensa-lx106-elf-objdump
 
-INC += -I../../tools/sdk/include
+INC += -I../../tools/sdk/include -I../../tools/sdk/uzlib/src
+
 CFLAGS += -std=gnu99
 
-CFLAGS += -O0 -g -Wpointer-arith -Wno-implicit-function-declaration -Wl,-EL -fno-inline-functions -nostdlib -mlongcalls -mno-text-section-literals
+CFLAGS += -Os -g -Wall -Wpointer-arith -Wno-implicit-function-declaration -Wl,-EL -fno-inline-functions -nostdlib -mlongcalls -mno-text-section-literals -ffunction-sections -fdata-sections
 
 CFLAGS += $(INC)
 
-LDFLAGS	+= -nostdlib -Wl,--no-check-sections -umain
+CFLAGS += $(UZLIB_FLAGS)
+
+LDFLAGS	+= -nostdlib -Wl,--no-check-sections -Wl,--gc-sections -umain
 
 LD_SCRIPT := -Teboot.ld
 
 APP_OUT:= eboot.elf
 APP_AR := eboot.a
 APP_FW := eboot.bin
 
-all: $(APP_FW)
 
-$(APP_AR): $(TARGET_OBJ_PATHS)
-	$(AR) cru $@ $^
+all: $(APP_OUT)
 
+tinflate.o: $(UZLIB_PATH)/tinflate.c $(UZLIB_PATH)/uzlib.h $(UZLIB_PATH)/uzlib_conf.h
+	$(CC) $(CFLAGS) -c -o tinflate.o $(UZLIB_PATH)/tinflate.c
 
-$(APP_OUT): $(APP_AR)
-	$(LD) $(LD_SCRIPT) $(LDFLAGS) -Wl,--start-group -Wl,--whole-archive $(APP_AR) -Wl,--end-group -o $@
+tinfgzip.o: $(UZLIB_PATH)/tinfgzip.c $(UZLIB_PATH)/uzlib.h $(UZLIB_PATH)/uzlib_conf.h
+	$(CC) $(CFLAGS) -c -o tinfgzip.o $(UZLIB_PATH)/tinfgzip.c
 
-$(APP_FW): $(APP_OUT)
-	$(ESPTOOL) -vvv -eo $(APP_OUT) -bo $@ -bs .text -bs .data -bs .rodata -bc -ec || true
+$(APP_AR): $(TARGET_OBJ_PATHS) tinflate.o tinfgzip.o
+	$(AR) cru $@ $^
 
+$(APP_OUT): $(APP_AR)
+	$(LD) $(LD_SCRIPT) $(LDFLAGS) -Wl,--start-group -Wl,--whole-archive $(APP_AR) -Wl,--end-group -o $@
 
 clean:
 	rm -f *.o

bootloaders/eboot/eboot.c

@@ -12,6 +12,9 @@
 #include <string.h>
 #include "flash.h"
 #include "eboot_command.h"
+#include <uzlib.h>
+
+extern unsigned char _gzip_dict;
 
 #define SWRST do { (*((volatile uint32_t*) 0x60000700)) |= 0x80000000; } while(0);
 
@@ -24,10 +27,14 @@ int print_version(const uint32_t flash_addr)
     if (SPIRead(flash_addr + APP_START_OFFSET + sizeof(image_header_t) + sizeof(section_header_t), &ver, sizeof(ver))) {
         return 1;
     }
-    const char* __attribute__ ((aligned (4))) fmtt = "v%08x\n\0\0";
-    uint32_t fmt[2];
-    fmt[0] = ((uint32_t*) fmtt)[0];
-    fmt[1] = ((uint32_t*) fmtt)[1];
+    char fmt[16];
+    fmt[0] = 'v';
+    fmt[1] = '%';
+    fmt[2] = '0';
+    fmt[3] = '8';
+    fmt[4] = 'x';
+    fmt[5] = '\n';
+    fmt[6] = '0';
     ets_printf((const char*) fmt, ver);
     return 0;
 }
@@ -80,37 +87,96 @@ int load_app_from_flash_raw(const uint32_t flash_addr)
         pos += section_header.size;
     }
 
-    register uint32_t sp asm("a1") = 0x3ffffff0;
-    register uint32_t pc asm("a3") = image_header.entry;
-    __asm__  __volatile__ ("jx a3");
+    asm volatile("" ::: "memory");
+    asm volatile ("mov.n a1, %0\n"
+        "mov.n a3, %1\n"
+        "jx a3\n" : : "r" (0x3ffffff0), "r" (image_header.entry) );
 
+    __builtin_unreachable(); // Save a few bytes by letting GCC know no need to pop regs/return
     return 0;
 }
 
+uint8_t read_flash_byte(const uint32_t addr)
+{
+    uint8_t __attribute__((aligned(4))) buff[4];
+    SPIRead(addr & ~3, buff, 4);
+    return buff[addr & 3];
+}
+unsigned char __attribute__((aligned(4))) uzlib_flash_read_cb_buff[4096];
+uint32_t uzlib_flash_read_cb_addr;
+int uzlib_flash_read_cb(struct uzlib_uncomp *m)
+{
+    m->source = uzlib_flash_read_cb_buff;
+    m->source_limit = uzlib_flash_read_cb_buff + sizeof(uzlib_flash_read_cb_buff);
+    SPIRead(uzlib_flash_read_cb_addr, uzlib_flash_read_cb_buff, sizeof(uzlib_flash_read_cb_buff));
+    uzlib_flash_read_cb_addr += sizeof(uzlib_flash_read_cb_buff);
+    return *(m->source++);
+}
 
+unsigned char gzip_dict[32768];
 
 int copy_raw(const uint32_t src_addr,
              const uint32_t dst_addr,
              const uint32_t size)
 {
     // require regions to be aligned
-    if (src_addr & 0xfff != 0 ||
-        dst_addr & 0xfff != 0) {
+    if ((src_addr & 0xfff) != 0 ||
+        (dst_addr & 0xfff) != 0) {
         return 1;
     }
 
     const uint32_t buffer_size = FLASH_SECTOR_SIZE;
     uint8_t buffer[buffer_size];
-    uint32_t left = ((size+buffer_size-1) & ~(buffer_size-1));
+    int32_t left = ((size+buffer_size-1) & ~(buffer_size-1));
     uint32_t saddr = src_addr;
     uint32_t daddr = dst_addr;
-
-    while (left) {
+    struct uzlib_uncomp m_uncomp;
+    bool gzip = false;
+
+    // Check if we are uncompressing a GZIP upload or not
+    if ((read_flash_byte(saddr) == 0x1f) && (read_flash_byte(saddr + 1) == 0x8b)) {
+        // GZIP signature matched.  Find real size as encoded at the end
+        left = read_flash_byte(saddr + size - 4);
+        left += read_flash_byte(saddr + size - 3)<<8;
+        left += read_flash_byte(saddr + size - 2)<<16;
+        left += read_flash_byte(saddr + size - 1)<<24;
+
+        uzlib_init();
+
+        /* all 3 fields below must be initialized by user */
+        m_uncomp.source = NULL;
+        m_uncomp.source_limit = NULL;
+        uzlib_flash_read_cb_addr = src_addr;
+        m_uncomp.source_read_cb = uzlib_flash_read_cb;
+        uzlib_uncompress_init(&m_uncomp, gzip_dict, sizeof(gzip_dict));
+
+        int res = uzlib_gzip_parse_header(&m_uncomp);
+        if (res != TINF_OK) {
+            return 5; // Error uncompress header read
+        }
+	gzip = true;
+    }
+    while (left > 0) {
         if (SPIEraseSector(daddr/buffer_size)) {
             return 2;
         }
-        if (SPIRead(saddr, buffer, buffer_size)) {
-            return 3;
+        if (!gzip) {
+            if (SPIRead(saddr, buffer, buffer_size)) {
+                return 3;
+            }
+        } else {
+            m_uncomp.dest_start = buffer;
+            m_uncomp.dest = buffer;
+            int to_read = (left > buffer_size) ? buffer_size : left;
+            m_uncomp.dest_limit = buffer + to_read;
+            int res = uzlib_uncompress(&m_uncomp);
+            if ((res != TINF_DONE) && (res != TINF_OK)) {
+                return 6;
+            }
+            // Fill any remaining with 0xff
+            for (int i = to_read; i < buffer_size; i++) {
+                buffer[i] = 0xff;
+            }
         }
         if (SPIWrite(daddr, buffer, buffer_size)) {
             return 4;
@@ -124,13 +190,12 @@ int copy_raw(const uint32_t src_addr,
 }
 
 
-
-void main()
+int main()
 {
     int res = 9;
     bool clear_cmd = false;
     struct eboot_command cmd;
-    
+
     print_version(0);
 
     if (eboot_command_read(&cmd) == 0) {
@@ -172,4 +237,7 @@ void main()
     }
 
     while(true){}
+
+    __builtin_unreachable();
+    return 0;
 }

bootloaders/eboot/eboot.ld

@@ -68,6 +68,8 @@ SECTIONS
 
   .data : ALIGN(4)
   {
+    *(COMMON)  /* Global vars */
+    . = ALIGN(4);
     _heap_start = ABSOLUTE(.);
 /*    _stack_sentry = ALIGN(0x8); */
   } >dram0_0_seg :dram0_0_bss_phdr
@@ -150,7 +152,6 @@ SECTIONS
     *(.bss)
     *(.bss.*)
     *(.gnu.linkonce.b.*)
-    *(COMMON)
     . = ALIGN (8);
     _bss_end = ABSOLUTE(.);
   } >iram1_0_seg :iram1_0_phdr

bootloaders/eboot/eboot_command.c

@@ -37,7 +37,7 @@ int eboot_command_read(struct eboot_command* cmd)
     }
 
     uint32_t crc32 = eboot_command_calculate_crc32(cmd);
-    if (cmd->magic & EBOOT_MAGIC_MASK != EBOOT_MAGIC || 
+    if ((cmd->magic & EBOOT_MAGIC_MASK) != EBOOT_MAGIC ||
         cmd->crc32 != crc32) {
         return 1;
     }

cores/esp8266/Updater.cpp

@@ -329,7 +329,8 @@ bool UpdaterClass::_writeBuffer(){
   bool modifyFlashMode = false;
   FlashMode_t flashMode = FM_QIO;
   FlashMode_t bufferFlashMode = FM_QIO;
-  if ((_currentAddress == _startAddress + FLASH_MODE_PAGE) && (_command == U_FLASH)) {
+  //TODO - GZIP can't do this
+  if ((_currentAddress == _startAddress + FLASH_MODE_PAGE) && (_buffer[0] != 0x1f) && (_command == U_FLASH)) {
     flashMode = ESP.getFlashChipMode();
     #ifdef DEBUG_UPDATER
       DEBUG_UPDATER.printf_P(PSTR("Header: 0x%1X %1X %1X %1X\n"), _buffer[0], _buffer[1], _buffer[2], _buffer[3]);
@@ -411,7 +412,7 @@ size_t UpdaterClass::write(uint8_t *data, size_t len) {
 bool UpdaterClass::_verifyHeader(uint8_t data) {
     if(_command == U_FLASH) {
         // check for valid first magic byte (is always 0xE9)
-        if(data != 0xE9) {
+        if ((data != 0xE9) && (data != 0x1f)) {
             _currentAddress = (_startAddress + _size);
             _setError(UPDATE_ERROR_MAGIC_BYTE);
             return false;
@@ -435,7 +436,12 @@ bool UpdaterClass::_verifyEnd() {
         }
 
         // check for valid first magic byte
-        if(buf[0] != 0xE9) {
+	//
+	// TODO: GZIP compresses the chipsize flags, so can't do check here
+	if ((buf[0] == 0x1f) && (buf[1] == 0x8b)) {
+            // GZIP, just assume OK
+            return true;
+        } else if (buf[0] != 0xE9) {
             _currentAddress = (_startAddress);
             _setError(UPDATE_ERROR_MAGIC_BYTE);            
             return false;

libraries/ESP8266httpUpdate/src/ESP8266httpUpdate.cpp

@@ -370,22 +370,24 @@ HTTPUpdateResult ESP8266HTTPUpdate::handleUpdate(HTTPClient& http, const String&
                     }
 
                     // check for valid first magic byte
-                    if(buf[0] != 0xE9) {
+                    if(buf[0] != 0xE9 && buf[0] != 0x1f) {
                         DEBUG_HTTP_UPDATE("[httpUpdate] Magic header does not start with 0xE9\n");
                         _setLastError(HTTP_UE_BIN_VERIFY_HEADER_FAILED);
                         http.end();
                         return HTTP_UPDATE_FAILED;
 
                     }
 
-                    uint32_t bin_flash_size = ESP.magicFlashChipSize((buf[3] & 0xf0) >> 4);
+                    if (buf[0] == 0xe9) {
+                        uint32_t bin_flash_size = ESP.magicFlashChipSize((buf[3] & 0xf0) >> 4);
 
-                    // check if new bin fits to SPI flash
-                    if(bin_flash_size > ESP.getFlashChipRealSize()) {
-                        DEBUG_HTTP_UPDATE("[httpUpdate] New binary does not fit SPI Flash size\n");
-                        _setLastError(HTTP_UE_BIN_FOR_WRONG_FLASH);
-                        http.end();
-                        return HTTP_UPDATE_FAILED;
+                        // check if new bin fits to SPI flash
+                        if(bin_flash_size > ESP.getFlashChipRealSize()) {
+                            DEBUG_HTTP_UPDATE("[httpUpdate] New binary does not fit SPI Flash size\n");
+                            _setLastError(HTTP_UE_BIN_FOR_WRONG_FLASH);
+                            http.end();
+                            return HTTP_UPDATE_FAILED;
+                        }
                     }
                 }
                 if(runUpdate(*tcp, len, http.header("x-MD5"), command)) {