Fix VM Address mask (#8440)

* Fix VM Address mask

Adjust VM Address mask to allow up to 8M Byte parts.
Allow for free heap size values over 64K
  ESP.getHeapStats(, uint32_t,)
  uint32_t getMaxFreeBlockSize();

* Fix example

* Update MockEsp.cpp for uint32_t on EspClass::getMaxFreeBlockSize()

* Added comment about heap size limitation and static_assert to verify.
Updated boards.txt to show correct External memory size and Heap size.

cores/esp8266/Esp-frag.cpp

@@ -23,19 +23,18 @@
 #include "coredecls.h"
 #include "Esp.h"
 
-void EspClass::getHeapStats(uint32_t* hfree, uint16_t* hmax, uint8_t* hfrag)
+void EspClass::getHeapStats(uint32_t* hfree, uint32_t* hmax, uint8_t* hfrag)
 {
     // L2 / Euclidean norm of free block sizes.
     // Having getFreeHeap()=sum(hole-size), fragmentation is given by
     // 100 * (1 - sqrt(sum(hole-size²)) / sum(hole-size))
-
     umm_info(NULL, false);
 
     uint32_t free_size = umm_free_heap_size_core(umm_get_current_heap());
     if (hfree)
         *hfree = free_size;
     if (hmax)
-        *hmax = (uint16_t)umm_max_block_size_core(umm_get_current_heap());
+        *hmax = umm_max_block_size_core(umm_get_current_heap());
     if (hfrag) {
       if (free_size) {
         *hfrag = umm_fragmentation_metric_core(umm_get_current_heap());
@@ -45,6 +44,18 @@ void EspClass::getHeapStats(uint32_t* hfree, uint16_t* hmax, uint8_t* hfrag)
     }
 }
 
+void EspClass::getHeapStats(uint32_t* hfree, uint16_t* hmax, uint8_t* hfrag)
+{
+    uint32_t hmax32;
+    getHeapStats(hfree, &hmax32, hfrag);
+    if (hmax) {
+      // With the MMU_EXTERNAL_HEAP option, hmax could overflow for heaps larger
+      // then 64KB. return UINT16_MAX (saturation) for those cases.
+      // Added deprecated attribute and message.
+      *hmax = (hmax32 > UINT16_MAX) ? UINT16_MAX : hmax32;
+    }
+}
+
 uint8_t EspClass::getHeapFragmentation()
 {
   return (uint8_t)umm_fragmentation_metric();

cores/esp8266/Esp.cpp

@@ -222,7 +222,7 @@ uint32_t EspClass::getFreeHeap(void)
     return system_get_free_heap_size();
 }
 
-uint16_t EspClass::getMaxFreeBlockSize(void)
+uint32_t EspClass::getMaxFreeBlockSize(void)
 {
     return umm_max_block_size();
 }

cores/esp8266/Esp.h

@@ -114,9 +114,10 @@ class EspClass {
         static uint32_t getChipId();
 
         static uint32_t getFreeHeap();
-        static uint16_t getMaxFreeBlockSize();
+        static uint32_t getMaxFreeBlockSize();
         static uint8_t getHeapFragmentation(); // in %
-        static void getHeapStats(uint32_t* free = nullptr, uint16_t* max = nullptr, uint8_t* frag = nullptr);
+        static void getHeapStats(uint32_t* free = nullptr, uint16_t* max = nullptr, uint8_t* frag = nullptr) __attribute__((deprecated("Use 'uint32_t*' on max, 2nd argument")));
+        static void getHeapStats(uint32_t* free = nullptr, uint32_t* max = nullptr, uint8_t* frag = nullptr);
 
         static uint32_t getFreeContStack();
         static void resetFreeContStack();

cores/esp8266/core_esp8266_vm.cpp

@@ -56,7 +56,7 @@
  * Higher density PSRAM (ESP-PSRAM64H/etc.) works as well, but may be too
    large to effectively use with UMM.  Only 256K is available vial malloc,
    but addresses above 256K do work and can be used for fixed buffers.
- 
+
 */
 
 #ifdef MMU_EXTERNAL_HEAP
@@ -71,6 +71,8 @@
 
 extern "C" {
 
+#define VM_OFFSET_MASK  0x007fffffu
+
 #define SHORT_MASK  0x000008u
 #define LOAD_MASK   0x00f00fu
 #define L8UI_MATCH  0x000002u
@@ -324,21 +326,21 @@ static IRAM_ATTR void loadstore_exception_handler(struct __exception_frame *ef,
     uint32_t val = ef->a_reg[regno];
     uint32_t what = insn & STORE_MASK;
     if (what == S8I_MATCH) {
-       spi_ramwrite(spi1, excvaddr & 0x1ffff, 8-1, val);
+       spi_ramwrite(spi1, excvaddr & VM_OFFSET_MASK, 8-1, val);
     } else if (what == S16I_MATCH) {
-      spi_ramwrite(spi1, excvaddr & 0x1ffff, 16-1, val);
+      spi_ramwrite(spi1, excvaddr & VM_OFFSET_MASK, 16-1, val);
     } else {
-      spi_ramwrite(spi1, excvaddr & 0x1ffff, 32-1, val);
+      spi_ramwrite(spi1, excvaddr & VM_OFFSET_MASK, 32-1, val);
     }
   } else {
     if (insn & L32_MASK) {
-      ef->a_reg[regno] = spi_ramread(spi1, excvaddr & 0x1ffff, 32-1);
+      ef->a_reg[regno] = spi_ramread(spi1, excvaddr & VM_OFFSET_MASK, 32-1);
     } else if (insn & L16_MASK) {
-      ef->a_reg[regno] = spi_ramread(spi1, excvaddr & 0x1ffff, 16-1);
+      ef->a_reg[regno] = spi_ramread(spi1, excvaddr & VM_OFFSET_MASK, 16-1);
       if ((insn & SIGNED_MASK ) && (ef->a_reg[regno] & 0x8000))
         ef->a_reg[regno] |= 0xffff0000;
     } else {
-      ef->a_reg[regno] = spi_ramread(spi1, excvaddr & 0x1ffff, 8-1);
+      ef->a_reg[regno] = spi_ramread(spi1, excvaddr & VM_OFFSET_MASK, 8-1);
     }
   }
 }
@@ -389,6 +391,11 @@ void install_vm_exception_handler()
     __vm_cache_line[cache_ways - 1].next = NULL;
   }
 
+  // Our umm_malloc configuration can only support a maximum of 256K RAM. A
+  // change would affect the block size of all heaps, and a larger block size
+  // would result in wasted space in the smaller heaps.
+  static_assert(MMU_EXTERNAL_HEAP <= 256, "Heap size must not exceed 256K");
+
   // Hook into memory manager
   umm_init_vm( (void *)0x10000000, MMU_EXTERNAL_HEAP * 1024);
 }

cores/esp8266/umm_malloc/umm_local.c

@@ -192,15 +192,15 @@ void umm_print_stats(int force) {
     umm_heap_context_t *_context = umm_get_current_heap();
 
     DBGLOG_FORCE(force, "umm heap statistics:\n");
-    DBGLOG_FORCE(force,   "  Heap ID           %5u\n", _context->id);
-    DBGLOG_FORCE(force,   "  Free Space        %5u\n", _context->UMM_FREE_BLOCKS * sizeof(umm_block));
-    DBGLOG_FORCE(force,   "  OOM Count         %5u\n", _context->UMM_OOM_COUNT);
+    DBGLOG_FORCE(force,   "  Heap ID           %7u\n", _context->id);
+    DBGLOG_FORCE(force,   "  Free Space        %7u\n", _context->UMM_FREE_BLOCKS * sizeof(umm_block));
+    DBGLOG_FORCE(force,   "  OOM Count         %7u\n", _context->UMM_OOM_COUNT);
     #if defined(UMM_STATS_FULL)
-    DBGLOG_FORCE(force,   "  Low Watermark     %5u\n", _context->stats.free_blocks_min * sizeof(umm_block));
-    DBGLOG_FORCE(force,   "  Low Watermark ISR %5u\n", _context->stats.free_blocks_isr_min * sizeof(umm_block));
-    DBGLOG_FORCE(force,   "  MAX Alloc Request %5u\n", _context->stats.alloc_max_size);
+    DBGLOG_FORCE(force,   "  Low Watermark     %7u\n", _context->stats.free_blocks_min * sizeof(umm_block));
+    DBGLOG_FORCE(force,   "  Low Watermark ISR %7u\n", _context->stats.free_blocks_isr_min * sizeof(umm_block));
+    DBGLOG_FORCE(force,   "  MAX Alloc Request %7u\n", _context->stats.alloc_max_size);
     #endif
-    DBGLOG_FORCE(force,   "  Size of umm_block %5u\n", sizeof(umm_block));
+    DBGLOG_FORCE(force,   "  Size of umm_block %7u\n", sizeof(umm_block));
     DBGLOG_FORCE(force, "+--------------------------------------------------------------+\n");
 }
 #endif

libraries/esp8266/examples/HeapMetric/HeapMetric.ino

@@ -10,11 +10,11 @@ void stats(const char* what) {
   // we could use getFreeHeap() getMaxFreeBlockSize() and getHeapFragmentation()
   // or all at once:
   uint32_t free;
-  uint16_t max;
+  uint32_t max;
   uint8_t frag;
   ESP.getHeapStats(&free, &max, &frag);
 
-  Serial.printf("free: %5d - max: %5d - frag: %3d%% <- ", free, max, frag);
+  Serial.printf("free: %7u - max: %7u - frag: %3d%% <- ", free, max, frag);
   // %s requires a malloc that could fail, using println instead:
   Serial.println(what);
 }
@@ -109,6 +109,7 @@ void tryit(int blocksize) {
 void setup() {
   Serial.begin(115200);
   WiFi.mode(WIFI_OFF);
+  delay(50);
 
   Serial.printf("\r\nDemo Heap Metrics for DRAM\r\n");
   tryit(8000);
@@ -135,6 +136,26 @@ void setup() {
     tryit(15);
   }
 #endif
+#ifdef MMU_EXTERNAL_HEAP
+  {
+    HeapSelect ephemeral = HeapSelect(UMM_HEAP_EXTERNAL);
+    Serial.printf("\r\nDemo Heap Metrics for External RAM\r\n");
+#if (MMU_EXTERNAL_HEAP > 64)
+    tryit(64000);
+    tryit(32000);
+#endif
+    tryit(16000);
+    tryit(8000);
+    tryit(4000);
+    tryit(2000);
+    tryit(1000);
+    tryit(500);
+    tryit(200);
+    tryit(100);
+    tryit(50);
+    tryit(15);
+  }
+#endif
 }
 
 void loop() {

libraries/esp8266/examples/irammem/irammem.ino

@@ -592,7 +592,7 @@ void setup() {
     size_t free_iram = ESP.getFreeHeap();
     ETS_PRINTF("IRAM free: %6d\n", free_iram);
     uint32_t hfree;
-    uint16_t hmax;
+    uint32_t hmax;
     uint8_t hfrag;
     ESP.getHeapStats(&hfree, &hmax, &hfrag);
     ETS_PRINTF("ESP.getHeapStats(free: %u, max: %u, frag: %u)\n",

tests/host/common/MockEsp.cpp

@@ -108,7 +108,7 @@ uint32_t EspClass::getFreeHeap()
 	return 30000;
 }
 
-uint16_t EspClass::getMaxFreeBlockSize()
+uint32_t EspClass::getMaxFreeBlockSize()
 {
 	return 20000;
 }

tools/boards.txt.py

@@ -1270,10 +1270,10 @@
         ( '.menu.mmu.4816H.build.mmuflags', '-DMMU_IRAM_SIZE=0xC000 -DMMU_ICACHE_SIZE=0x4000 -DMMU_IRAM_HEAP' ),
         ( '.menu.mmu.3216', '16KB cache + 32KB IRAM + 16KB 2nd Heap (not shared)' ),
         ( '.menu.mmu.3216.build.mmuflags', '-DMMU_IRAM_SIZE=0x8000 -DMMU_ICACHE_SIZE=0x4000 -DMMU_SEC_HEAP=0x40108000 -DMMU_SEC_HEAP_SIZE=0x4000' ),
-        ( '.menu.mmu.ext128k', '128K External 23LC1024' ),
+        ( '.menu.mmu.ext128k', '128K Heap External 23LC1024' ),
         ( '.menu.mmu.ext128k.build.mmuflags', '-DMMU_EXTERNAL_HEAP=128 -DMMU_IRAM_SIZE=0x8000 -DMMU_ICACHE_SIZE=0x8000' ),
-        ( '.menu.mmu.ext1024k', '1M External 64 MBit PSRAM' ),
-        ( '.menu.mmu.ext1024k.build.mmuflags', '-DMMU_EXTERNAL_HEAP=256 -DMMU_IRAM_SIZE=0x8000 -DMMU_ICACHE_SIZE=0x8000' ),
+        ( '.menu.mmu.ext8192k', '8M w/256K Heap External 64 MBit PSRAM' ),
+        ( '.menu.mmu.ext8192k.build.mmuflags', '-DMMU_EXTERNAL_HEAP=256 -DMMU_IRAM_SIZE=0x8000 -DMMU_ICACHE_SIZE=0x8000' ),
         ]),
 
     ######################## Non 32-bit load/store exception handler