Update chaindb to use a slab allocator for small size buffers.

libraries/chain/controller.cpp

@@ -1983,8 +1983,12 @@ struct controller_impl {
                }
             }
          });
+
          auto snapshot_load_time = (fc::time_point::now() - snapshot_load_start_time).to_seconds();
-         ilog( "Finished initialization from snapshot (snapshot load time was ${t}s)", ("t", snapshot_load_time) );
+         auto db_size = db.get_segment_manager()->get_size();
+         auto free_size = db.get_segment_manager()->get_free_memory();
+
+         ilog( "Finished initialization from snapshot (snapshot load time was ${t}s, db size used is ${s} bytes)", ("t", snapshot_load_time)("s", db_size - free_size) );
       } catch (boost::interprocess::bad_alloc& e) {
          elog( "Failed initialization from snapshot - db storage not configured to have enough storage for the provided snapshot, please increase and retry snapshot" );
          shutdown();

libraries/chain/include/eosio/chain/database_utils.hpp

@@ -61,6 +61,10 @@ namespace eosio::chain {
          static void create( chainbase::database& db, F cons ) {
             db.create<typename index_t::value_type>(cons);
          }
+
+         static void preallocate( chainbase::database& db, size_t num ) {
+            db.preallocate<typename index_t::value_type>(num);
+         }
    };
 
    template<typename Index>

libraries/chain/include/eosio/chain/protocol_state_object.hpp

@@ -17,10 +17,11 @@ namespace eosio { namespace chain {
    class protocol_state_object : public chainbase::object<protocol_state_object_type, protocol_state_object>
    {
    public:
-      template<typename Constructor>
-      protocol_state_object(Constructor&& c, chainbase::constructor_tag) :
-         id(0),
-         whitelisted_intrinsics(*activated_protocol_features.get_allocator(this)) {
+      template <typename Constructor>
+      protocol_state_object(Constructor&& c, chainbase::constructor_tag)
+         : id(0)
+         , whitelisted_intrinsics(
+              chainbase::make_allocator<whitelisted_intrinsics_type::value_type>(&whitelisted_intrinsics)) {
          c(*this);
       }
 

libraries/chaindb/include/chainbase/chainbase.hpp

@@ -509,6 +509,16 @@ namespace chainbase {
              return get_mutable_index<index_type>().remove( obj );
          }
 
+         template<typename ObjectType>
+         void preallocate( size_t num )
+         {
+             if ( _read_only_mode ) {
+                BOOST_THROW_EXCEPTION( std::logic_error( "attempting to preallocate in read-only mode" ) );
+             }
+             typedef typename get_index_type<ObjectType>::type index_type;
+             get_mutable_index<index_type>().preallocate( num );
+         }
+
          template<typename ObjectType, typename Constructor>
          const ObjectType& create( Constructor&& con )
          {

libraries/chaindb/include/chainbase/chainbase_node_allocator.hpp

@@ -14,57 +14,82 @@ namespace chainbase {
     public:
       using value_type = T;
       using pointer = bip::offset_ptr<T>;
-      chainbase_node_allocator(segment_manager* manager) : _manager{manager} {}
-      chainbase_node_allocator(const chainbase_node_allocator& other) : _manager(other._manager) {}
+
+      chainbase_node_allocator(segment_manager* manager) : _manager{manager} {
+         _ss_alloc = pinnable_mapped_file::get_small_size_allocator((std::byte*)manager);
+      }
+
+      chainbase_node_allocator(const chainbase_node_allocator& other) : chainbase_node_allocator(&*other._manager) {}
+
       template<typename U>
-      chainbase_node_allocator(const chainbase_node_allocator<U, S>& other) : _manager(other._manager) {}
+      chainbase_node_allocator(const chainbase_node_allocator<U, S>& other) : chainbase_node_allocator(&*other._manager) {}
+
       pointer allocate(std::size_t num) {
          if (num == 1) {
-            if (_freelist == nullptr) {
-               get_some();
+            if (_block_start == _block_end && _freelist == nullptr) {
+               get_some(_allocation_batch_size);
             }
+            if (_block_start < _block_end) {
+               pointer result =  pointer{static_cast<T*>(static_cast<void*>(_block_start.get()))};
+               _block_start += sizeof(T);
+               return result;
+            }
+            assert(_freelist != nullptr);
             list_item* result = &*_freelist;
             _freelist = _freelist->_next;
             result->~list_item();
             --_freelist_size;
             return pointer{(T*)result};
          } else {
-            return pointer{(T*)_manager->allocate(num*sizeof(T))};
+            return pointer{(T*)&*_ss_alloc->allocate(num*sizeof(T))};
          }
       }
+
       void deallocate(const pointer& p, std::size_t num) {
          if (num == 1) {
             _freelist = new (&*p) list_item{_freelist};
             ++_freelist_size;
          } else {
-            _manager->deallocate(&*p);
+            _ss_alloc->deallocate(ss_allocator_t::pointer((char*)&*p), num*sizeof(T));
          }
       }
+
+      void preallocate(std::size_t num) {
+         if (num >= 2 * _allocation_batch_size)
+            get_some((num + 7) & ~7);
+      }
+
       bool operator==(const chainbase_node_allocator& other) const { return this == &other; }
       bool operator!=(const chainbase_node_allocator& other) const { return this != &other; }
       segment_manager* get_segment_manager() const { return _manager.get(); }
-      size_t freelist_memory_usage() const { return _freelist_size * sizeof(T); }
+      size_t freelist_memory_usage() const { return _freelist_size * sizeof(T) + (_block_end - _block_start); }
+
     private:
       template<typename T2, typename S2>
       friend class chainbase_node_allocator;
-      void get_some() {
+
+      void get_some(size_t num_to_alloc) {
          static_assert(sizeof(T) >= sizeof(list_item), "Too small for free list");
          static_assert(sizeof(T) % alignof(list_item) == 0, "Bad alignment for free list");
-         const unsigned allocation_batch_size = 64;
-         char* result = (char*)_manager->allocate(sizeof(T) * allocation_batch_size);
-         _freelist_size += allocation_batch_size;
-         _freelist = bip::offset_ptr<list_item>{(list_item*)result};
-         for(unsigned i = 0; i < allocation_batch_size-1; ++i) {
-            char* next = result + sizeof(T);
-            new(result) list_item{bip::offset_ptr<list_item>{(list_item*)next}};
-            result = next;
-         }
-         new(result) list_item{nullptr};
+
+         _block_start = static_cast<char*>(_manager->allocate(sizeof(T) * num_to_alloc));
+         _block_end   = _block_start + sizeof(T) * num_to_alloc;
+
+         if (_allocation_batch_size < max_allocation_batch_size)
+            _allocation_batch_size *= 2;
       }
+
       struct list_item { bip::offset_ptr<list_item> _next; };
+
+      static constexpr size_t max_allocation_batch_size = 512;
+
+      bip::offset_ptr<char>            _block_start;
+      bip::offset_ptr<char>            _block_end;
+      bip::offset_ptr<list_item>       _freelist{};
+      bip::offset_ptr<ss_allocator_t>  _ss_alloc;
       bip::offset_ptr<segment_manager> _manager;
-      bip::offset_ptr<list_item> _freelist{};
-      size_t _freelist_size = 0;
+      size_t                           _allocation_batch_size = 32;
+      size_t                           _freelist_size = 0;
    };
 
 }  // namepsace chainbase

libraries/chaindb/include/chainbase/environment.hpp

@@ -5,9 +5,13 @@
 namespace chainbase {
 
 constexpr size_t header_size = 1024;
+
 // `CHAINB01` reflects changes since `EOSIODB3`.
+// `CHAINB02` adds the small size allocator
 // Spring 1.0 is compatible with `CHAINB01`.
-constexpr uint64_t header_id = 0x3130424e49414843ULL; //"CHAINB01" little endian
+// Spring 2.0 is compatible with `CHAINB02`.
+// ---------------------------------------------
+constexpr uint64_t header_id = 0x3230424e49414843ULL; //"CHAINB02" little endian
 
 struct environment  {
    environment() {
@@ -67,10 +71,11 @@ struct environment  {
 } __attribute__ ((packed));
 
 struct db_header  {
-   uint64_t id = header_id;
-   bool dirty = false;
-   environment dbenviron;
-} __attribute__ ((packed));
+   uint64_t               id    = header_id;
+   bool                   dirty = false;
+   bip::offset_ptr<char>  small_size_allocator;
+   environment            dbenviron;
+};
 
 constexpr size_t header_dirty_bit_offset = offsetof(db_header, dirty);
 

libraries/chaindb/include/chainbase/pinnable_mapped_file.hpp

@@ -5,6 +5,7 @@
 #include <boost/interprocess/sync/file_lock.hpp>
 #include <boost/asio/io_context.hpp>
 #include <boost/container/flat_map.hpp>
+#include <chainbase/small_size_allocator.hpp>
 #include <filesystem>
 #include <vector>
 #include <optional>
@@ -45,7 +46,24 @@ class chainbase_error_category : public std::error_category {
 using segment_manager = bip::managed_mapped_file::segment_manager;
 
 template<typename T>
-using allocator = bip::allocator<T, segment_manager>;
+using segment_allocator_t = bip::allocator<T, segment_manager>;
+
+using byte_segment_allocator_t = segment_allocator_t<char>;
+
+using ss_allocator_t = small_size_allocator<byte_segment_allocator_t>;
+
+// An allocator for objects of type T within the segment_manager
+// -------------------------------------------------------------
+// - If the allocation size (num_objects * sizeof(T)) is less than 512 bytes, it will be routed
+//   through the small size allocator which allocates in batch from the `segment_manager`.
+// - If the allocation size (num_objects * sizeof(T)) is greater than 512 bytes, the allocator
+//   will allocate directly from the segment manager.
+// - the 512 bytes limit is derived from the template parameters of `small_size_allocator`
+//   (size_t num_allocators = 64, size_t size_increment = 8)
+// - emulates the API of `bip::allocator<T, segment_manager>`
+// ---------------------------------------------------------------------------------------------
+template<typename T>
+using allocator = object_allocator<T, ss_allocator_t>; 
 
 class pinnable_mapped_file {
    public:
@@ -66,19 +84,22 @@ class pinnable_mapped_file {
       segment_manager* get_segment_manager() const { return _segment_manager;}
       size_t           check_memory_and_flush_if_needed();
 
+      static ss_allocator_t* get_small_size_allocator(std::byte* seg_mgr);
+
       template<typename T>
       static std::optional<allocator<T>> get_allocator(void *object) {
          if (!_segment_manager_map.empty()) {
             auto it = _segment_manager_map.upper_bound(object);
             if(it == _segment_manager_map.begin())
                return {};
-            auto [seg_start, seg_end] = *(--it);
+            auto& [seg_start, seg_info] = *(--it);
             // important: we need to check whether the pointer is really within the segment, as shared objects'
             // can also be created on the stack (in which case the data is actually allocated on the heap using
             // std::allocator). This happens for example when `shared_cow_string`s are inserted into a bip::multimap,
             // and temporary pairs are created on the stack by the bip::multimap code.
-            if (object < seg_end)
-               return allocator<T>(reinterpret_cast<segment_manager *>(seg_start));
+            if (object < seg_info.seg_end) {
+               return std::optional<allocator<T>>{allocator<T>(get_small_size_allocator(static_cast<std::byte*>(seg_start)))};
+            }
          }
          return {};
       }
@@ -114,13 +135,32 @@ class pinnable_mapped_file {
 
       static std::vector<pinnable_mapped_file*>     _instance_tracker;
 
-      using segment_manager_map_t = boost::container::flat_map<void*, void *>;
+      struct seg_info_t { void* seg_end; };
+      using segment_manager_map_t = boost::container::flat_map<void*, seg_info_t>;
       static segment_manager_map_t                  _segment_manager_map;
 
       constexpr static unsigned                     _db_size_multiple_requirement = 1024*1024; //1MB
       constexpr static size_t                       _db_size_copy_increment       = 1024*1024*1024; //1GB
 };
 
+// There can be at most one `small_size_allocator` per `segment_manager` (hence the `assert` below).
+// There is none created if the pinnable_mapped_file is read-only.
+// ----------------------------------------------------------------------------------------------------
+template <class backing_allocator>
+auto make_small_size_allocator(segment_manager* seg_mgr) {
+   assert(pinnable_mapped_file::get_small_size_allocator((std::byte*)seg_mgr) == nullptr);
+   byte_segment_allocator_t byte_allocator(seg_mgr);
+   return new (seg_mgr->allocate(sizeof(ss_allocator_t))) ss_allocator_t(byte_allocator);
+}
+
+// Create an allocator for a specific object type. 
+// pointer can be to the segment manager, or any object contained within.
+// ---------------------------------------------------------------------
+template <class T>
+auto make_allocator(void* seg_mgr) {
+   return *pinnable_mapped_file::get_allocator<T>(seg_mgr);
+}
+
 std::istream& operator>>(std::istream& in, pinnable_mapped_file::map_mode& runtime);
 std::ostream& operator<<(std::ostream& osm, pinnable_mapped_file::map_mode m);
 

libraries/chaindb/include/chainbase/shared_cow_string.hpp

@@ -26,7 +26,7 @@ namespace chainbase {
       };
 
     public:
-      using allocator_type = bip::allocator<char, segment_manager>;
+      using allocator_type = allocator<char>;
       using iterator       = const char*;
       using const_iterator = const char*;
 

libraries/chaindb/include/chainbase/shared_cow_vector.hpp

@@ -23,7 +23,7 @@ namespace chainbase {
       };
 
    public:
-      using allocator_type = bip::allocator<char, segment_manager>;
+      using allocator_type = allocator<char>;
       using iterator       = const T*;      // const because of copy-on-write
       using const_iterator = const T*;
       using value_type     = T;