From 33fd2e4026d4d46449246f3a28f05a9f928fdecd Mon Sep 17 00:00:00 2001 From: andresovela Date: Wed, 17 Jul 2024 17:25:15 +0200 Subject: [PATCH 1/5] Split tensor arena into persistent and non-persistent arenas --- tflite_micro_compiler/src/Api.cc | 13 ++++- tflite_micro_compiler/src/Api.h | 6 ++ tflite_micro_compiler/src/Compiler.cc | 79 +++++++++++++++++++++------ tflite_micro_compiler/src/Compiler.h | 11 +++- 4 files changed, 90 insertions(+), 19 deletions(-) diff --git a/tflite_micro_compiler/src/Api.cc b/tflite_micro_compiler/src/Api.cc index 7502f2b6..3fa42901 100644 --- a/tflite_micro_compiler/src/Api.cc +++ b/tflite_micro_compiler/src/Api.cc @@ -25,8 +25,19 @@ std::string TFLMC_Compiler::getTensorName(int tensorIndex, int sg) const { return compiler_->getTensorName(tensorIndex, sg); } -// Returns tensor arena size +// Returns tensor arena size (persistent + non-persistent) size_t TFLMC_Compiler::getTensorArenaSize() const { return compiler_->getTensorArenaSize(); } + +// Returns non-persistent tensor arena size +size_t TFLMC_Compiler::getNonPersistentTensorArenaSize() const { + return compiler_->getNonPersistentTensorArenaSize(); +} + +// Returns persistent tensor arena size +size_t TFLMC_Compiler::getPersistentTensorArenaSize() const { + return compiler_->getPersistentTensorArenaSize(); +} + } // namespace tflmc \ No newline at end of file diff --git a/tflite_micro_compiler/src/Api.h b/tflite_micro_compiler/src/Api.h index c7de6d20..92836b1e 100644 --- a/tflite_micro_compiler/src/Api.h +++ b/tflite_micro_compiler/src/Api.h @@ -27,6 +27,12 @@ class TFLMC_Compiler { // Returns tensor arena size size_t getTensorArenaSize() const; + // Returns non-persistent tensor arena size + size_t TFLMC_Compiler::getNonPersistentTensorArenaSize() const; + + // Returns persistent tensor arena size + size_t TFLMC_Compiler::getPersistentTensorArenaSize() const; + private: Compiler *compiler_; }; diff --git a/tflite_micro_compiler/src/Compiler.cc b/tflite_micro_compiler/src/Compiler.cc index 7c05301d..5d9e188b 100644 --- a/tflite_micro_compiler/src/Compiler.cc +++ b/tflite_micro_compiler/src/Compiler.cc @@ -180,6 +180,7 @@ bool tflmc::Compiler::init(const void *modelData) { arena_buf_.resize(SUFFICIENT_ARENA_SIZE); g_arena_size = SUFFICIENT_ARENA_SIZE; + // TODO: This looks unused? std::vector arena_buf(g_arena_size); g_arenaPtr = arena_buf_.data(); @@ -348,13 +349,15 @@ bool tflmc::Compiler::init(const void *modelData) { } } - // This includes: + // Non-persistent arena consists of: // - Tensors // - Scratch buffers + nonPersistentArenaSize_ = ramTensorBufferSize; + + // Persistent arena consists of: // - Persistent buffers // - Variable tensors (which are allocated as persistent buffers separately) - arenaBufferSize_ = - ramTensorBufferSize + totalRuntimeAllocSize + varTensorsSize; + persistentArenaSize_ = totalRuntimeAllocSize + varTensorsSize; if (debugPrint_) { interpreter_->allocator_.memory_planner()->PrintMemoryPlan(); @@ -545,14 +548,31 @@ namespace xcore { wr << R"( constexpr int kTensorArenaSize = )" - << arenaBufferSize_ << R"(; -#ifndef SHARED_TENSOR_ARENA + << persistentArenaSize_ + nonPersistentArenaSize_ << R"(; +constexpr int kPersistentArenaBufferSize = )" + << persistentArenaSize_ << R"(; +constexpr int kNonPersistentArenaBufferSize = )" + << nonPersistentArenaSize_ << R"(; + +#ifdef EXTERN_TENSOR_ARENA + +#ifdef SPLIT_PERSISTENT_TENSOR_ARENA +extern uint8_t persistent_tensor_arena[]; +#endif // SPLIT_PERSISTENT_TENSOR_ARENA + +extern uint8_t tensor_arena[]; + +#else + +#ifdef SPLIT_PERSISTENT_TENSOR_ARENA +#error "Split persistent/non-persistent tensor arenas require externally defined tensor arenas (EXTERN_TENSOR_ARENA must be defined) " +#endif + namespace { uint8_t tensor_arena[kTensorArenaSize] ALIGN(8); } -#else -extern uint8_t tensor_arena[]; -#endif + +#endif // EXTERN_TENSOR_ARENA namespace { template struct TfArray { @@ -899,6 +919,7 @@ static TfLiteStatus RequestScratchBufferInArena(struct TfLiteContext *context, s static void *GetScratchBuffer(struct TfLiteContext *context, int buffer_idx) { + // Scratch buffers are allocated in the non-persistent tensor arena return tensor_arena + scratch_buffer_offsets[buffer_idx]; } @@ -1044,7 +1065,12 @@ TfLiteStatus )" << prefix_ << R"(init(void *flash_data) { // Clear and initialize scratch_buffer_idx = 0; + +#ifdef SPLIT_PERSISTENT_TENSOR_ARENA + persistentBufferPtr = persistent_tensor_arena + kPersistentTensorArenaSize; +#else persistentBufferPtr = tensor_arena + kTensorArenaSize; +#endif // Set flash data in xcore context config xc_config.flash_data = flash_data; @@ -1348,14 +1374,35 @@ void tflmc::Compiler::writeHeader(std::ostream &out) { #include "tensorflow/lite/c/common.h" -#ifdef SHARED_TENSOR_ARENA - #ifndef LARGEST_TENSOR_ARENA_SIZE - #define LARGEST_TENSOR_ARENA_SIZE )" + - std::to_string(arenaBufferSize_) + R"( - #elif LARGEST_TENSOR_ARENA_SIZE < )" + - std::to_string(arenaBufferSize_) + R"( - #define LARGEST_TENSOR_ARENA_SIZE )" + - std::to_string(arenaBufferSize_) + R"( +#ifdef EXTERN_TENSOR_ARENA + #ifdef SPLIT_PERSISTENT_TENSOR_ARENA + #ifndef LARGEST_TENSOR_ARENA_SIZE + #define LARGEST_TENSOR_ARENA_SIZE )" + + std::to_string(nonPersistentArenaSize_) + R"( + #elif LARGEST_TENSOR_ARENA_SIZE < )" + + std::to_string(nonPersistentArenaSize_) + R"( + #define LARGEST_TENSOR_ARENA_SIZE )" + + std::to_string(nonPersistentArenaSize_) + R"( + #endif + + #ifndef LARGEST_PERSISTENT_TENSOR_ARENA_SIZE + #define LARGEST_PERSISTENT_TENSOR_ARENA_SIZE )" + + std::to_string(persistentArenaSize_) + R"( + #elif LARGEST_PERSISTENT_TENSOR_ARENA_SIZE < )" + + std::to_string(persistentArenaSize_) + R"( + #define LARGEST_PERSISTENT_TENSOR_ARENA_SIZE )" + + std::to_string(persistentArenaSize_) + R"( + #endif + + #else + #ifndef LARGEST_TENSOR_ARENA_SIZE + #define LARGEST_TENSOR_ARENA_SIZE )" + + std::to_string(persistentArenaSize_ + nonPersistentArenaSize_) + R"( + #elif LARGEST_TENSOR_ARENA_SIZE < )" + + std::to_string(persistentArenaSize_ + nonPersistentArenaSize_) + R"( + #define LARGEST_TENSOR_ARENA_SIZE )" + + std::to_string(persistentArenaSize_ + nonPersistentArenaSize_) + R"( + #endif #endif #endif diff --git a/tflite_micro_compiler/src/Compiler.h b/tflite_micro_compiler/src/Compiler.h index 56b82357..d44664d3 100644 --- a/tflite_micro_compiler/src/Compiler.h +++ b/tflite_micro_compiler/src/Compiler.h @@ -49,7 +49,13 @@ class Compiler { std::string getTensorName(int tensorIndex, int sg) const; // Returns tensor arena size - size_t getTensorArenaSize() const { return arenaBufferSize_; } + size_t getTensorArenaSize() const { return persistentArenaSize_ + nonPersistentArenaSize_; } + + // Returns persistent arena size + size_t getPersistentArenaSize() const { return persistentArenaSize_; } + + // Returns non-persistent arena size + size_t getNonPersistentArenaSize() const { return nonPersistentArenaSize_; } private: bool init(const void *modelData); @@ -105,7 +111,8 @@ class Compiler { std::unique_ptr interpreter_; MemMap memMap_; - size_t arenaBufferSize_ = 0; + size_t persistentArenaSize_ = 0; + size_t nonPersistentArenaSize_ = 0; size_t varTensors_count = 0; // Vector of vector is for subgraphs std::vector> tensors_; From 72577384344fff6b8231bbf0ff401661ec152ff4 Mon Sep 17 00:00:00 2001 From: andresovela Date: Thu, 18 Jul 2024 16:27:16 +0200 Subject: [PATCH 2/5] Fix compiler errors --- tflite_micro_compiler/src/Api.cc | 12 ++++++------ tflite_micro_compiler/src/Api.h | 8 ++++---- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tflite_micro_compiler/src/Api.cc b/tflite_micro_compiler/src/Api.cc index 3fa42901..c4c0f507 100644 --- a/tflite_micro_compiler/src/Api.cc +++ b/tflite_micro_compiler/src/Api.cc @@ -30,14 +30,14 @@ size_t TFLMC_Compiler::getTensorArenaSize() const { return compiler_->getTensorArenaSize(); } -// Returns non-persistent tensor arena size -size_t TFLMC_Compiler::getNonPersistentTensorArenaSize() const { - return compiler_->getNonPersistentTensorArenaSize(); +// Returns non-persistent arena size +size_t TFLMC_Compiler::getNonPersistentArenaSize() const { + return compiler_->getNonPersistentArenaSize(); } -// Returns persistent tensor arena size -size_t TFLMC_Compiler::getPersistentTensorArenaSize() const { - return compiler_->getPersistentTensorArenaSize(); +// Returns persistent arena size +size_t TFLMC_Compiler::getPersistentArenaSize() const { + return compiler_->getPersistentArenaSize(); } } // namespace tflmc \ No newline at end of file diff --git a/tflite_micro_compiler/src/Api.h b/tflite_micro_compiler/src/Api.h index 92836b1e..de1f2ee8 100644 --- a/tflite_micro_compiler/src/Api.h +++ b/tflite_micro_compiler/src/Api.h @@ -27,11 +27,11 @@ class TFLMC_Compiler { // Returns tensor arena size size_t getTensorArenaSize() const; - // Returns non-persistent tensor arena size - size_t TFLMC_Compiler::getNonPersistentTensorArenaSize() const; + // Returns non-persistent arena size + size_t getNonPersistentArenaSize() const; - // Returns persistent tensor arena size - size_t TFLMC_Compiler::getPersistentTensorArenaSize() const; + // Returns persistent arena size + size_t getPersistentArenaSize() const; private: Compiler *compiler_; From 2925a8bbf803b13cb118e7a64874f1a6ceb9c3b0 Mon Sep 17 00:00:00 2001 From: andresovela Date: Thu, 18 Jul 2024 17:28:43 +0200 Subject: [PATCH 3/5] Rename misnamed variables --- tflite_micro_compiler/src/Compiler.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tflite_micro_compiler/src/Compiler.cc b/tflite_micro_compiler/src/Compiler.cc index 5d9e188b..e97331be 100644 --- a/tflite_micro_compiler/src/Compiler.cc +++ b/tflite_micro_compiler/src/Compiler.cc @@ -549,9 +549,9 @@ namespace xcore { constexpr int kTensorArenaSize = )" << persistentArenaSize_ + nonPersistentArenaSize_ << R"(; -constexpr int kPersistentArenaBufferSize = )" +constexpr int kPersistentTensorArenaSize = )" << persistentArenaSize_ << R"(; -constexpr int kNonPersistentArenaBufferSize = )" +constexpr int kNonPersistentTensorArenaSize = )" << nonPersistentArenaSize_ << R"(; #ifdef EXTERN_TENSOR_ARENA From 63f24c3587584afe807a6f694f350887d7ac423e Mon Sep 17 00:00:00 2001 From: andresovela Date: Thu, 18 Jul 2024 17:30:08 +0200 Subject: [PATCH 4/5] Remove unused variable --- tflite_micro_compiler/src/Compiler.cc | 2 -- 1 file changed, 2 deletions(-) diff --git a/tflite_micro_compiler/src/Compiler.cc b/tflite_micro_compiler/src/Compiler.cc index e97331be..c3c7e316 100644 --- a/tflite_micro_compiler/src/Compiler.cc +++ b/tflite_micro_compiler/src/Compiler.cc @@ -551,8 +551,6 @@ constexpr int kTensorArenaSize = )" << persistentArenaSize_ + nonPersistentArenaSize_ << R"(; constexpr int kPersistentTensorArenaSize = )" << persistentArenaSize_ << R"(; -constexpr int kNonPersistentTensorArenaSize = )" - << nonPersistentArenaSize_ << R"(; #ifdef EXTERN_TENSOR_ARENA From 198ef5f27cecb2ac32e2f304d5534e14a3796e41 Mon Sep 17 00:00:00 2001 From: andresovela Date: Thu, 18 Jul 2024 17:30:38 +0200 Subject: [PATCH 5/5] Declar kPersistentTensorArenaSize only if SPLIT_PERSISTENT_TENSOR_ARENA is defined --- tflite_micro_compiler/src/Compiler.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tflite_micro_compiler/src/Compiler.cc b/tflite_micro_compiler/src/Compiler.cc index c3c7e316..2064a1b5 100644 --- a/tflite_micro_compiler/src/Compiler.cc +++ b/tflite_micro_compiler/src/Compiler.cc @@ -549,12 +549,12 @@ namespace xcore { constexpr int kTensorArenaSize = )" << persistentArenaSize_ + nonPersistentArenaSize_ << R"(; -constexpr int kPersistentTensorArenaSize = )" - << persistentArenaSize_ << R"(; #ifdef EXTERN_TENSOR_ARENA #ifdef SPLIT_PERSISTENT_TENSOR_ARENA +constexpr int kPersistentTensorArenaSize = )" + << persistentArenaSize_ << R"(; extern uint8_t persistent_tensor_arena[]; #endif // SPLIT_PERSISTENT_TENSOR_ARENA