Add FSTools with examples of how to convert between SPIFFS and LITTLE…

…FS. (#7696) * Add FSTools with examples of how to convert between SPIFFS and LITTLEFS. * Oops. Need to pass layout by reference in order to capture the correct address. Took a while to find that. There maybe a better way to store all these configs * Update FSTools.cpp fix ESP.h to Esp.h * Fix unused variable i * Parsed with restyle.sh. Compile with all errors. * remove unused variable * fix different sign complication error * Fix indentation to spaces Run test/restyle.sh Remove commented code Use #ifdef blocks for debugging. `DEBUG_ESP_CORE` and `DEBUG_ESP_PORT` use `static constexpr layout` which saves ROM ~500B for unused vars * Update FSTools.cpp Add yield in between file copy
esp8266 · Jun 26, 2022 · f0d8f33 · f0d8f33
1 parent 5656035
commit f0d8f33
Show file tree

Hide file tree

Showing 6 changed files with 623 additions and 2 deletions.
diff --git a/cores/esp8266/FS.h b/cores/esp8266/FS.h
@@ -90,9 +90,8 @@ class File : public Stream
       uint8_t obuf[256];
       size_t doneLen = 0;
       size_t sentLen;
-      int i;
 
-      while (src.available() > sizeof(obuf)){
+      while (src.available() > (int)sizeof(obuf)){
         src.read(obuf, sizeof(obuf));
         sentLen = write(obuf, sizeof(obuf));
         doneLen = doneLen + sentLen;

diff --git a/libraries/FSTools/FSTools.cpp b/libraries/FSTools/FSTools.cpp
@@ -0,0 +1,254 @@
+#include "FSTools.h"
+#include "LittleFS.h"
+#include <spiffs_api.h>
+#include <Esp.h>
+
+
+
+#if defined(DEBUG_ESP_CORE)
+#define FSTOOLSDEBUG(_1, ...) { DEBUG_ESP_PORT.printf_P( PSTR(_1),##__VA_ARGS__); }
+#else
+#define FSTOOLSDEBUG(...) {}
+#endif
+
+
+FSTools::FSTools()
+{
+
+}
+
+FSTools::~FSTools()
+{
+    reset();
+}
+
+
+bool FSTools::attemptToMountFS(fs::FS & fs)
+{
+    LittleFSConfig littleFSCfg(false);
+    SPIFFSConfig SPIFFSCfg(false);
+    //  try to apply the "safe" no format config to the FS... doesn't matter which.. just need one to apply correctly..
+    if (!fs.setConfig(littleFSCfg) && ! fs.setConfig(SPIFFSCfg))
+    {
+        return false;
+    }
+    return fs.begin();
+}
+
+
+bool FSTools::mountAlternativeFS(FST::FS_t type, const FST::layout & layout, bool keepMounted)
+{
+    FSConfig * pCfg{nullptr};
+    LittleFSConfig littleFSCfg(false);
+    SPIFFSConfig SPIFFSCfg(false);
+    reset();
+
+    switch (type)
+    {
+    case FST::SPIFFS :
+    {
+        _pFS.reset(new FS(FSImplPtr(new spiffs_impl::SPIFFSImpl(_getStartAddr(layout), _getSize(layout), layout.page, layout.block, 5))));
+        pCfg = &SPIFFSCfg;
+        break;
+    }
+    case FST::LITTLEFS :
+    {
+        _pFS.reset(new FS(FSImplPtr(new littlefs_impl::LittleFSImpl(_getStartAddr(layout), _getSize(layout), layout.page, layout.block, 5))));
+        pCfg = &littleFSCfg;
+        break;
+    }
+    };
+
+    if (_pFS && pCfg && _pFS->setConfig(*pCfg) && _pFS->begin())
+    {
+        if (!keepMounted)
+        {
+            _pFS->end();
+        }
+        _mounted = true;
+        _layout = &layout;
+        return true;
+    }
+
+    if (_pFS)
+    {
+        _pFS.reset();
+    }
+    _mounted = false;
+    return false;
+};
+
+
+bool FSTools::mounted()
+{
+    return _mounted;
+};
+
+
+bool FSTools::moveFS(fs::FS & destinationFS)
+{
+    uint32_t sourceFileCount = 0;
+    uint32_t sourceByteTotal = 0;
+    bool result = false;
+
+    if (!_mounted || !_pFS)
+    {
+        FSTOOLSDEBUG("Source FS not mounted\n");
+        return false;
+    }
+
+    uint32_t startSector = (ESP.getSketchSize() + FLASH_SECTOR_SIZE - 1) & (~(FLASH_SECTOR_SIZE - 1));
+    uint32_t lowestFSStart = 0x40300000;
+
+    if (_layout)
+    {
+        lowestFSStart = _layout->startAddr;
+        FSTOOLSDEBUG("_layout->startADDR = 0x%08x\n", _layout->startAddr);
+    }
+
+    uint32_t endSector = lowestFSStart - 0x40200000;
+    uint32_t tempFSsize = endSector - startSector;
+
+    FSTOOLSDEBUG("TempFS:  start: %u, end: %u, size: %u, sketchSize = %u, _FS_start = %u\n", startSector, endSector, tempFSsize, ESP.getSketchSize(), (uint32_t)&_FS_start);
+
+    fileListIterator(*_pFS, "/", [&sourceFileCount, &sourceByteTotal, this](File & f)
+    {
+        if (f)
+        {
+            sourceFileCount++;
+            sourceByteTotal += f.size();
+
+#ifdef DEBUG_ESP_CORE
+            _dumpFileInfo(f);
+#endif
+
+        }
+    });
+
+    FSTOOLSDEBUG("%u Files Found Total Size = %u\n", sourceFileCount, sourceByteTotal);
+    FSTOOLSDEBUG("Size of dummy FS = %u\n", tempFSsize);
+
+    FS tempFS = FS(FSImplPtr(new littlefs_impl::LittleFSImpl(startSector, tempFSsize, FS_PHYS_PAGE, FS_PHYS_BLOCK, 5)));
+
+    if (tempFS.format() && tempFS.begin())
+    {
+        if (_copyFS(*_pFS, tempFS))
+        {
+            FSTOOLSDEBUG("Files copied to temp File System\n");
+            reset();
+            if (destinationFS.format() && destinationFS.begin()) //  must format then mount the new FS
+            {
+                if (_copyFS(tempFS, destinationFS))
+                {
+                    FSTOOLSDEBUG("Files copied back to new FS\n");
+                    result = true;
+                }
+            }
+            else
+            {
+                FSTOOLSDEBUG("Error Mounting\n");
+            }
+        }
+        else
+        {
+            FSTOOLSDEBUG("Copy Failed\n");
+        }
+        tempFS.end();
+    }
+    else
+    {
+        FSTOOLSDEBUG("Failed to begin() TempFS\n");
+    }
+    return result;
+};
+
+void FSTools::reset()
+{
+    _mounted = false;
+    _layout = nullptr;
+    if (_pFS)
+    {
+        _pFS->end();
+        _pFS.reset();
+    }
+}
+
+void FSTools::fileListIterator(FS & fs, const char * dirName, FST::FileCb Cb)
+{
+    Dir dir = fs.openDir(dirName);
+    while (dir.next())
+    {
+        if (dir.isFile())
+        {
+            File f =  dir.openFile("r");
+            if (Cb)
+            {
+                Cb(f);
+            }
+        }
+        else
+        {
+            fileListIterator(fs, dir.fileName().c_str(), Cb);
+        }
+    }
+}
+
+
+uint32_t FSTools::_getStartAddr(const FST::layout & layout)
+{
+    return (layout.startAddr - 0x40200000);
+}
+
+uint32_t FSTools::_getSize(const FST::layout & layout)
+{
+    return (layout.endAddr - layout.startAddr);
+}
+
+#ifdef DEBUG_ESP_CORE
+void FSTools::_dumpFileInfo(File & f)
+{
+    if (f)
+    {
+        DEBUG_ESP_PORT.printf_P(PSTR(" File: %-30s [%8uB]\n"), f.fullName(), f.size());
+    }
+}
+#endif
+
+bool FSTools::_copyFS(FS & sourceFS, FS & destFS)
+{
+    uint32_t sourceFileCount = 0;
+    uint32_t sourceByteTotal = 0;
+
+    fileListIterator(sourceFS, "/", [&sourceFileCount, &sourceByteTotal](File & f)
+    {
+        if (f)
+        {
+            sourceFileCount++;
+            sourceByteTotal += f.size();
+        }
+    });
+
+    size_t count = 0;
+    fileListIterator(sourceFS, "/", [&count, &destFS](File & sourceFile)
+    {
+        if (sourceFile)
+        {
+            File destFile = destFS.open(sourceFile.fullName(), "w");
+            if (destFile)
+            {
+                destFile.setTimeout(5000); //  this value was chosen empirically as it failed with default timeout. 
+                size_t written = destFile.write(sourceFile);
+                if (written == sourceFile.size())
+                {
+                    count++;
+                }
+            }
+            destFile.close();
+            sourceFile.close();
+            yield();
+        }
+    });
+
+    return (count == sourceFileCount);
+
+}
diff --git a/libraries/FSTools/FSTools.h b/libraries/FSTools/FSTools.h
@@ -0,0 +1,91 @@
+#pragma once
+
+#include <FS.h>
+#include <memory>
+#include <functional>
+/*
+
+    A temporary FS is made between the END of the sketch...  and the start of the partition you try to mount, to maximise the available space for copying the FS.
+    The WORST case this is at 0x40300000 which is for a 3m FS on 4m flash..  leaving 460Kb for copying.
+
+*/
+
+
+
+namespace FST
+{
+
+    struct layout
+    {
+        constexpr layout(uint32_t s, uint32_t e, uint32_t p, uint32_t b) : startAddr(s), endAddr(e), page(p), block(b) {};
+        const uint32_t startAddr{0};
+        const uint32_t endAddr{0};
+        const uint32_t page{0};
+        const uint32_t block{0};
+    };
+
+    enum FS_t : uint8_t
+    {
+        SPIFFS,
+        LITTLEFS
+    };
+
+    static constexpr layout layout_512k32  = { 0x40273000, 0x4027B000, 0x100, 0x1000 };
+    static constexpr layout layout_512k64  = { 0x4026B000, 0x4027B000, 0x100, 0x1000 };
+    static constexpr layout layout_512k128 = { 0x4025B000, 0x4027B000, 0x100, 0x1000 };
+
+    static constexpr layout layout_1m64    = { 0x402EB000, 0x402FB000, 0x100, 0x1000 };
+    static constexpr layout layout_1m128   = { 0x402DB000, 0x402FB000, 0x100, 0x1000 };
+    static constexpr layout layout_1m144   = { 0x402D7000, 0x402FB000, 0x100, 0x1000 };
+    static constexpr layout layout_1m160   = { 0x402D3000, 0x402FB000, 0x100, 0x1000 };
+    static constexpr layout layout_1m192   = { 0x402CB000, 0x402FB000, 0x100, 0x1000 };
+    static constexpr layout layout_1m256   = { 0x402BB000, 0x402FB000, 0x100, 0x1000 };
+    static constexpr layout layout_1m512   = { 0x4027B000, 0x402FB000, 0x100, 0x2000 };
+
+    static constexpr layout layout_2m64    = { 0x403F0000, 0x403FB000, 0x100, 0x1000 };
+    static constexpr layout layout_2m128   = { 0x403E0000, 0x403FB000, 0x100, 0x1000 };
+    static constexpr layout layout_2m256   = { 0x403C0000, 0x403FB000, 0x100, 0x1000 };
+    static constexpr layout layout_2m512   = { 0x40380000, 0x403FA000, 0x100, 0x2000 };
+    static constexpr layout layout_2m1m    = { 0x40300000, 0x403FA000, 0x100, 0x2000 };
+
+    static constexpr layout layout_4m1m    = { 0x40500000, 0x405FA000, 0x100, 0x2000 };
+    static constexpr layout layout_4m2m    = { 0x40400000, 0x405FA000, 0x100, 0x2000 };
+    static constexpr layout layout_4m3m    = { 0x40300000, 0x405FA000, 0x100, 0x2000 };
+
+    static constexpr layout layout_8m6m    = { 0x40400000, 0x409FA000, 0x100, 0x2000 };
+    static constexpr layout layout_8m7m    = { 0x40300000, 0x409FA000, 0x100, 0x2000 };
+
+    static constexpr layout layout_16m14m  = { 0x40400000, 0x411FA000, 0x100, 0x2000 };
+    static constexpr layout layout_16m15m  = { 0x40300000, 0x411FA000, 0x100, 0x2000 };
+
+    typedef std::function<void(File & f)> FileCb;
+
+};
+
+
+class FSTools
+{
+public:
+
+    FSTools();
+    ~FSTools();
+    bool attemptToMountFS(fs::FS & fs);
+    bool mountAlternativeFS(FST::FS_t type, const FST::layout & layout, bool keepMounted = false);
+    bool mounted();
+    bool moveFS(fs::FS & destinationFS);
+    void reset();
+    void fileListIterator(FS & fs, const char * dirName, FST::FileCb Cb);
+
+private:
+    uint32_t _getStartAddr(const FST::layout & layout);
+    uint32_t _getSize(const FST::layout & layout);
+#ifdef DEBUG_ESP_CORE
+    void _dumpFileInfo(File & f);
+#endif
+    bool _copyFS(FS & sourceFS, FS & destFS);
+
+    std::unique_ptr<fs::FS> _pFS;
+    bool _mounted{false};
+    const FST::layout * _layout{nullptr};
+
+};