storage: map_file - read a file straight from memory-mapped flash

extmod/vfs_fat.c

@@ -47,6 +47,9 @@
 #include "extmod/vfs_fat.h"
 #include "shared/timeutils/timeutils.h"
 #include "supervisor/filesystem.h"
+#if CIRCUITPY_STORAGE_MAP_FILE
+#include "shared-bindings/storage/__init__.h"
+#endif
 
 #if FF_MAX_SS == FF_MIN_SS
 #define SECSIZE(fs) (FF_MIN_SS)
@@ -243,6 +246,11 @@ static mp_obj_t fat_vfs_remove_internal(mp_obj_t vfs_in, mp_obj_t path_in, mp_in
 
     // check if path is a file or directory
     if ((fno.fattrib & AM_DIR) == attr) {
+        #if CIRCUITPY_STORAGE_MAP_FILE
+        if (attr == 0) {
+            storage_map_file_check_writable(self, path);
+        }
+        #endif
         res = f_unlink(&self->fatfs, path);
 
         if (res != FR_OK) {

extmod/vfs_fat_file.c

@@ -37,6 +37,9 @@
 #include "lib/oofatfs/ff.h"
 #include "extmod/vfs_fat.h"
 #include "supervisor/filesystem.h"
+#if CIRCUITPY_STORAGE_MAP_FILE
+#include "shared-bindings/storage/__init__.h"
+#endif
 
 // this table converts from FRESULT to POSIX errno
 const byte fresult_to_errno_table[20] = {
@@ -249,7 +252,11 @@ static mp_obj_t fat_vfs_open(mp_obj_t self_in, mp_obj_t path_in, mp_obj_t mode_i
     if ((mode & FA_WRITE) != 0 && !filesystem_is_writable_by_python(self)) {
         mp_raise_OSError(MP_EROFS);
     }
-
+    #if CIRCUITPY_STORAGE_MAP_FILE
+    if ((mode & FA_WRITE) != 0) {
+        storage_map_file_check_writable(self, mp_obj_str_get_str(path_in));
+    }
+    #endif
 
     pyb_file_obj_t *o = mp_obj_malloc_with_finaliser(pyb_file_obj_t, type);
 

main.c

@@ -84,6 +84,10 @@
 #include "shared-module/keypad/__init__.h"
 #endif
 
+#if CIRCUITPY_STORAGE_MAP_FILE
+#include "shared-bindings/storage/__init__.h"
+#endif
+
 #if CIRCUITPY_AUDIOFILEWRITER
 #include "shared-module/audiofilewriter/AudioFileWriter.h"
 #endif
@@ -396,6 +400,10 @@ static void cleanup_after_vm(mp_obj_t exception) {
     keypad_reset();
     #endif
 
+    #if CIRCUITPY_STORAGE_MAP_FILE
+    storage_map_file_reset();
+    #endif
+
     #if CIRCUITPY_AUDIOFILEWRITER
     audiofilewriter_reset();
     #endif

ports/espressif/mpconfigport.mk

@@ -479,3 +479,6 @@ endif
 
 # Usually lots of flash space available
 CIRCUITPY_MESSAGE_COMPRESSION_LEVEL ?= 1
+
+# The CIRCUITPY partition is mapped into the data address space on first use
+CIRCUITPY_STORAGE_MAP_FILE ?= 1

ports/espressif/supervisor/internal_flash.c

@@ -80,6 +80,43 @@ uint32_t supervisor_flash_get_block_count(void) {
 void port_internal_flash_flush(void) {
 }
 
+#if CIRCUITPY_STORAGE_MAP_FILE
+// storage.map_file: each drive partition mapped into the data address space on first use. With
+// extended storage the drive spans two partitions that need not be adjacent in the mapping, so
+// *contiguous stops at the seam and the caller splits a cluster run there.
+static const uint8_t *map_partition(size_t i) {
+    static const uint8_t *base[2];
+    if (base[i] == NULL) {
+        const void *p;
+        esp_partition_mmap_handle_t handle;                  // stays mapped for the session
+        if (esp_partition_mmap(_partition[i], 0, _partition[i]->size, ESP_PARTITION_MMAP_DATA,
+            &p, &handle) != ESP_OK) {
+            return NULL;
+        }
+        base[i] = p;
+    }
+    return base[i];
+}
+
+const uint8_t *port_internal_flash_xip_address(uint32_t block, uint32_t *contiguous) {
+    size_t i = 0;
+    uint32_t blocks = _partition[0]->size / FILESYSTEM_BLOCK_SIZE;
+    #if CIRCUITPY_STORAGE_EXTEND
+    if (storage_extended && block >= blocks) {
+        block -= blocks;
+        blocks = _partition[1]->size / FILESYSTEM_BLOCK_SIZE;
+        i = 1;
+    }
+    #endif
+    const uint8_t *base = map_partition(i);
+    if (base == NULL) {
+        return NULL;
+    }
+    *contiguous = blocks - block;
+    return base + block * FILESYSTEM_BLOCK_SIZE;
+}
+#endif
+
 static void single_partition_rw(const esp_partition_t *partition, uint8_t *data,
     const uint32_t offset, const uint32_t size_total, const bool op) {
     if (op == OP_READ) {

ports/raspberrypi/mpconfigport.mk

@@ -26,6 +26,8 @@ CIRCUITPY_PWMIO ?= 1
 CIRCUITPY_RGBMATRIX ?= $(CIRCUITPY_DISPLAYIO)
 CIRCUITPY_ROTARYIO ?= 1
 CIRCUITPY_ROTARYIO_SOFTENCODER = 1
+# The CIRCUITPY drive is execute-in-place on every RP2 board
+CIRCUITPY_STORAGE_MAP_FILE ?= 1
 CIRCUITPY_SYNTHIO_MAX_CHANNELS = 24
 CIRCUITPY_USB_HOST ?= 1
 CIRCUITPY_USB_VIDEO ?= 1

ports/raspberrypi/supervisor/internal_flash.c

@@ -149,3 +149,11 @@ mp_uint_t supervisor_flash_write_blocks(const uint8_t *src, uint32_t lba, uint32
 
 void supervisor_flash_release_cache(void) {
 }
+
+#if CIRCUITPY_STORAGE_MAP_FILE
+// storage.map_file: the CIRCUITPY drive is execute-in-place on every RP2 board.
+const uint8_t *port_internal_flash_xip_address(uint32_t block, uint32_t *contiguous) {
+    *contiguous = supervisor_flash_get_block_count() - block;
+    return (const uint8_t *)(XIP_BASE + CIRCUITPY_CIRCUITPY_DRIVE_START_ADDR + block * FILESYSTEM_BLOCK_SIZE);
+}
+#endif

py/circuitpy_mpconfig.mk

@@ -636,6 +636,12 @@ CFLAGS += -DCIRCUITPY_STORAGE=$(CIRCUITPY_STORAGE)
 CIRCUITPY_STORAGE_EXTEND ?= $(CIRCUITPY_DUALBANK)
 CFLAGS += -DCIRCUITPY_STORAGE_EXTEND=$(CIRCUITPY_STORAGE_EXTEND)
 
+# storage.map_file(): read a file straight out of memory-mapped flash. A port whose CIRCUITPY drive
+# is memory-mapped implements port_internal_flash_xip_address() and turns this on; turning it on
+# without that function is a link error, not a silent no-op.
+CIRCUITPY_STORAGE_MAP_FILE ?= 0
+CFLAGS += -DCIRCUITPY_STORAGE_MAP_FILE=$(CIRCUITPY_STORAGE_MAP_FILE)
+
 CIRCUITPY_STRUCT ?= 1
 CFLAGS += -DCIRCUITPY_STRUCT=$(CIRCUITPY_STRUCT)
 

shared-bindings/storage/__init__.c

@@ -300,13 +300,35 @@ static mp_obj_t storage_enable_usb_drive(void) {
 }
 MP_DEFINE_CONST_FUN_OBJ_0(storage_enable_usb_drive_obj, storage_enable_usb_drive);
 
+
+//| def map_file(file: typing.BinaryIO) -> Tuple[memoryview, ...]:
+//|     """Map a file on the CIRCUITPY drive straight from flash, without copying it into RAM.
+//|     Returns one read-only `memoryview` per contiguous run of the file's clusters, in file
+//|     order. The views stay valid after the file is closed.
+//|
+//|     Until the next reload, opening the file for writing or removing it raises ``OSError``
+//|     ``EACCES``. A USB host can still rewrite it.
+//|
+//|     :param typing.BinaryIO file: A file on the CIRCUITPY drive open with ``"rb"``
+//|     :raises OSError: ``EINVAL`` if the file is closed or not open for reading only,
+//|       ``EOPNOTSUPP`` if it is not on a FAT CIRCUITPY drive or the drive is not memory-mapped,
+//|       ``EIO`` if its cluster chain is corrupt
+//|     :raises ~builtins.MemoryError: if the file's cluster map could not be allocated
+//|     :raises NotImplementedError: on a port whose drive is not memory-mapped"""
+//|     ...
+static mp_obj_t storage_map_file(mp_obj_t file_in) {
+    return common_hal_storage_map_file(file_in);
+}
+MP_DEFINE_CONST_FUN_OBJ_1(storage_map_file_obj, storage_map_file);
+
 static const mp_rom_map_elem_t storage_module_globals_table[] = {
     { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_storage) },
 
     { MP_ROM_QSTR(MP_QSTR_mount),                    MP_ROM_PTR(&storage_mount_obj) },
     { MP_ROM_QSTR(MP_QSTR_umount),                   MP_ROM_PTR(&storage_umount_obj) },
     { MP_ROM_QSTR(MP_QSTR_remount),                  MP_ROM_PTR(&storage_remount_obj) },
     { MP_ROM_QSTR(MP_QSTR_getmount),                 MP_ROM_PTR(&storage_getmount_obj) },
+    { MP_ROM_QSTR(MP_QSTR_map_file),                 MP_ROM_PTR(&storage_map_file_obj) },
     { MP_ROM_QSTR(MP_QSTR_erase_filesystem),         MP_ROM_PTR(&storage_erase_filesystem_obj) },
     { MP_ROM_QSTR(MP_QSTR_disable_usb_drive),        MP_ROM_PTR(&storage_disable_usb_drive_obj) },
     { MP_ROM_QSTR(MP_QSTR_enable_usb_drive),         MP_ROM_PTR(&storage_enable_usb_drive_obj) },

shared-bindings/storage/__init__.h

@@ -21,3 +21,11 @@ MP_NORETURN void common_hal_storage_erase_filesystem(bool extended);
 bool common_hal_storage_disable_usb_drive(void);
 bool common_hal_storage_unsafe_disable_usb_drive(void);
 bool common_hal_storage_enable_usb_drive(void);
+
+mp_obj_t common_hal_storage_map_file(mp_obj_t file);
+#if CIRCUITPY_STORAGE_MAP_FILE
+struct _fs_user_mount_t;
+// Raises OSError if path on vfs is a file mapped this run: a write would change bytes in use.
+void storage_map_file_check_writable(struct _fs_user_mount_t *vfs, const char *path);
+void storage_map_file_reset(void);
+#endif

shared-module/storage/__init__.c

@@ -18,6 +18,12 @@
 #include "shared-bindings/storage/__init__.h"
 #include "supervisor/filesystem.h"
 #include "supervisor/flash.h"
+#if CIRCUITPY_STORAGE_MAP_FILE
+#include "extmod/vfs_fat.h"
+#include "lib/oofatfs/ff.h"
+#include "py/objarray.h"
+#include "py/objlist.h"
+#endif
 
 #if CIRCUITPY_USB_DEVICE
 #include "supervisor/usb.h"
@@ -233,3 +239,109 @@ void common_hal_storage_erase_filesystem(bool extended) {
     common_hal_mcu_reset();
     // We won't actually get here, since we're resetting.
 }
+
+#if CIRCUITPY_STORAGE_MAP_FILE
+// Start clusters of the files mapped this run; a write to one of them is refused until reload.
+MP_REGISTER_ROOT_POINTER(mp_obj_t storage_mapped_files);
+
+void storage_map_file_reset(void) {
+    MP_STATE_VM(storage_mapped_files) = MP_OBJ_NULL;
+}
+
+static bool mapped_files_contain(DWORD sclust) {
+    mp_obj_list_t *mapped = MP_OBJ_TO_PTR(MP_STATE_VM(storage_mapped_files));
+    for (size_t i = 0; i < mapped->len; i++) {
+        if (MP_OBJ_SMALL_INT_VALUE(mapped->items[i]) == (mp_int_t)sclust) {
+            return true;
+        }
+    }
+    return false;
+}
+
+static void mapped_files_add(DWORD sclust) {
+    if (MP_STATE_VM(storage_mapped_files) == MP_OBJ_NULL) {
+        MP_STATE_VM(storage_mapped_files) = mp_obj_new_list(0, NULL);
+    }
+    if (!mapped_files_contain(sclust)) {
+        mp_obj_list_append(MP_STATE_VM(storage_mapped_files), MP_OBJ_NEW_SMALL_INT(sclust));
+    }
+}
+
+void storage_map_file_check_writable(fs_user_mount_t *vfs, const char *path) {
+    if (MP_STATE_VM(storage_mapped_files) == MP_OBJ_NULL || vfs != filesystem_circuitpy()) {
+        return;
+    }
+    FIL fp;
+    if (f_open(&vfs->fatfs, &fp, path, FA_READ) != FR_OK) {
+        return;                                 // no such file yet
+    }
+    DWORD sclust = fp.obj.sclust;
+    f_close(&fp);
+    if (mapped_files_contain(sclust)) {
+        mp_raise_OSError(MP_EACCES);            // mapped: its flash bytes are in use
+    }
+}
+#endif
+
+mp_obj_t common_hal_storage_map_file(mp_obj_t file_in) {
+    #if CIRCUITPY_STORAGE_MAP_FILE
+    if (!mp_obj_is_type(file_in, &mp_type_vfs_fat_fileio)) {
+        mp_raise_OSError(MP_EOPNOTSUPP);        // only a FAT volume stores a file as flash bytes
+    }
+    pyb_file_obj_t *file = MP_OBJ_TO_PTR(file_in);
+    FATFS *fatfs = file->fp.obj.fs;
+    if (fatfs == NULL || (file->fp.flag & FA_WRITE)) {
+        mp_raise_OSError(MP_EINVAL);            // closed, or not open for reading only
+    }
+    fs_user_mount_t *drive = filesystem_circuitpy();
+    if (drive == NULL || fatfs != &drive->fatfs) {
+        mp_raise_OSError(MP_EOPNOTSUPP);        // another mount (an SD card): not memory-mapped
+    }
+    DWORD *tbl = file->fp.cltbl;
+    if (tbl == NULL) {
+        mp_raise_type(&mp_type_MemoryError);    // open() could not allocate the cluster map
+    }
+    FSIZE_t size = f_size(&file->fp);
+    if (size == 0) {
+        return mp_const_empty_tuple;
+    }
+    supervisor_flash_flush();                   // write back any RAM sector cache
+    // Walk the cluster runs of the link map open() built and split a run wherever the port's
+    // mapping is not contiguous, e.g. at the seam of a drive that spans two flash partitions.
+    mp_obj_t views = mp_obj_new_list(0, NULL);
+    size_t nruns = (size_t)((tbl[0] - 2) / 2);
+    FSIZE_t left = size;
+    for (size_t i = 0; i < nruns && left > 0; i++) {
+        DWORD sector = fatfs->database + (tbl[2 + 2 * i] - 2) * fatfs->csize;
+        FSIZE_t span = (FSIZE_t)tbl[1 + 2 * i] * fatfs->csize * FF_MIN_SS;
+        if (span > left) {
+            span = left;                        // the last run is clipped to the dir-entry size
+        }
+        while (span > 0) {
+            uint32_t contiguous;
+            const uint8_t *addr = supervisor_flash_xip_address(sector, &contiguous);
+            if (addr == NULL) {
+                mp_raise_OSError(MP_EOPNOTSUPP);    // this build cannot map the drive here
+            }
+            FSIZE_t piece = (FSIZE_t)contiguous * FF_MIN_SS;
+            if (piece > span) {
+                piece = span;
+            }
+            // read-only: a stray write raises instead of silently hitting the flash window
+            mp_obj_list_append(views, mp_obj_new_memoryview('B', (size_t)piece, (void *)addr));
+            span -= piece;
+            left -= piece;
+            sector += contiguous;
+        }
+    }
+    if (left != 0) {
+        mp_raise_OSError(MP_EIO);               // chain shorter than the dir-entry size: corrupt
+    }
+    mapped_files_add(file->fp.obj.sclust);
+    mp_obj_list_t *list = MP_OBJ_TO_PTR(views);
+    return mp_obj_new_tuple(list->len, list->items);
+    #else
+    (void)file_in;
+    mp_raise_type(&mp_type_NotImplementedError);    // this port does not map its drive
+    #endif
+}

supervisor/flash.h

@@ -29,6 +29,12 @@ void supervisor_flash_init_vfs(struct _fs_user_mount_t *vfs);
 void supervisor_flash_flush(void);
 void supervisor_flash_release_cache(void);
 
+#if CIRCUITPY_STORAGE_MAP_FILE
+// storage.map_file: flash address of a CIRCUITPY FatFs sector, NULL if the port cannot map the
+// drive there; *contiguous receives how many sectors from it are contiguous in the mapping.
+const uint8_t *supervisor_flash_xip_address(uint32_t fatfs_sector, uint32_t *contiguous);
+#endif
+
 void supervisor_flash_set_extended(bool extended);
 bool supervisor_flash_get_extended(void);
 void supervisor_flash_update_extended(void);

supervisor/shared/flash.c

@@ -149,6 +149,18 @@ static mp_uint_t flash_read_blocks(mp_obj_t self_in, uint8_t *dest, uint32_t blo
     return supervisor_flash_read_blocks(dest, block_num, num_blocks);
 }
 
+#if CIRCUITPY_STORAGE_MAP_FILE
+// The sector -> block translation of flash_read_blocks(); FatFs has already validated the
+// cluster chain, so every sector it hands over lies inside the volume.
+const uint8_t *supervisor_flash_xip_address(uint32_t fatfs_sector, uint32_t *contiguous) {
+    uint32_t block = fatfs_sector - PART1_START_BLOCK;
+    #if CIRCUITPY_SAVES_PARTITION_SIZE > 0
+    block += CIRCUITPY_SAVES_PARTITION_SIZE / FILESYSTEM_BLOCK_SIZE;
+    #endif
+    return port_internal_flash_xip_address(block, contiguous);
+}
+#endif
+
 static volatile bool filesystem_dirty = false;
 
 static mp_uint_t flash_write_blocks(mp_obj_t self_in, const uint8_t *src, uint32_t block_num, uint32_t num_blocks) {

supervisor/shared/internal_flash.h

@@ -8,3 +8,10 @@
 #include "supervisor/internal_flash.h" // This is per-port.
 
 void port_internal_flash_flush(void);
+
+#if CIRCUITPY_STORAGE_MAP_FILE
+// The memory-mapped address of a drive block, NULL if the port cannot map it; *contiguous
+// receives how many blocks from it are contiguous in the mapping. A port that sets
+// CIRCUITPY_STORAGE_MAP_FILE implements this.
+const uint8_t *port_internal_flash_xip_address(uint32_t block, uint32_t *contiguous);
+#endif