Libosmium: include/osmium/storage/item_stash.hpp Source File

Libosmium Fast and flexible C++ library for working with OpenStreetMap data

Loading...
Searching...
No Matches
item_stash.hpp
Go to the documentation of this file.
1#ifndef OSMIUM_STORAGE_ITEM_STASH_HPP
2#define OSMIUM_STORAGE_ITEM_STASH_HPP
3
4/*
5
6This file is part of Osmium (https://osmcode.org/libosmium).
7
8Copyright 2013-2026 Jochen Topf <jochen@topf.org> and others (see README).
9
10Boost Software License - Version 1.0 - August 17th, 2003
11
12Permission is hereby granted, free of charge, to any person or organization
13obtaining a copy of the software and accompanying documentation covered by
14this license (the "Software") to use, reproduce, display, distribute,
15execute, and transmit the Software, and to prepare derivative works of the
16Software, and to permit third-parties to whom the Software is furnished to
17do so, all subject to the following:
18
19The copyright notices in the Software and this entire statement, including
20the above license grant, this restriction and the following disclaimer,
21must be included in all copies of the Software, in whole or in part, and
22all derivative works of the Software, unless such copies or derivative
23works are solely in the form of machine-executable object code generated by
24a source language processor.
25
26THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
29SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
30FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
31ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
32DEALINGS IN THE SOFTWARE.
33
34*/
35
36#include <osmium/memory/buffer.hpp>
37#include <osmium/memory/item.hpp>
38
39#include <cassert>
40#include <cstdlib>
41#include <limits>
42#include <ostream>
43#include <vector>
44
45#ifdef OSMIUM_ITEM_STORAGE_GC_DEBUG
46# include <iostream>
47# include <chrono>
48#endif
49
50namespace osmium {
51
57 class ItemStash {
58
59 public:
60
71 class handle_type {
72
73 friend class ItemStash;
74
75 std::size_t value; // NOLINT(modernize-use-default-member-init)
76 // Some compilers don't like the default member
77 // init: "error: defaulted default constructor
78 // of 'handle_type' cannot be used by non-static
79 // data member initializer which appears before
80 // end of class definition"
81
82 explicit handle_type(std::size_t new_value) noexcept :
83 value(new_value) {
84 assert(new_value > 0);
85 }
86
87 public:
88
90 handle_type() noexcept :
91 value(0) {
92 }
93
95 bool valid() const noexcept {
96 return value != 0;
97 }
98
104 template <typename TChar, typename TTraits>
105 friend inline std::basic_ostream<TChar, TTraits>& operator<<(std::basic_ostream<TChar, TTraits>& out, const ItemStash::handle_type& handle) { // NOLINT(readability-redundant-inline-specifier)
106 if (handle.valid()) {
107 out << handle.value;
108 } else {
109 out << '-';
110 }
111 return out;
112 }
113
114 }; // class handle_type
115
116 private:
117
118 enum {
119 initial_buffer_size = 1024UL * 1024UL
120 };
121
122 enum {
123 removed_item_offset = std::numeric_limits<std::size_t>::max()
124 };
125
126 osmium::memory::Buffer m_buffer;
127 std::vector<std::size_t> m_index;
128 std::size_t m_count_items = 0;
129 std::size_t m_count_removed = 0;
130#ifdef OSMIUM_ITEM_STORAGE_GC_DEBUG
131 int64_t m_gc_time = 0;
132#endif
133
134 class cleanup_helper {
135
136 std::vector<std::size_t>& m_index;
137 std::size_t m_pos = 0;
138
139 public:
140
141 explicit cleanup_helper(std::vector<std::size_t>& index) :
142 m_index(index) {
143 }
144
145 void moving_in_buffer(std::size_t old_offset, std::size_t new_offset) {
146 while (m_index[m_pos] != old_offset) {
147 ++m_pos;
148 assert(m_pos < m_index.size());
149 }
150 m_index[m_pos] = new_offset;
151 ++m_pos;
152 }
153
154 }; // cleanup_helper
155
156 std::size_t& get_item_offset_ref(handle_type handle) noexcept {
157 assert(handle.valid() && "handle must be valid");
158 assert(handle.value <= m_index.size());
159 auto& offset = m_index[handle.value - 1];
160 assert(offset != removed_item_offset);
161 assert(offset < m_buffer.committed());
162 return offset;
163 }
164
165 std::size_t get_item_offset(handle_type handle) const noexcept {
166 assert(handle.valid() && "handle must be valid");
167 assert(handle.value <= m_index.size());
168 const auto& offset = m_index[handle.value - 1];
169 assert(offset != removed_item_offset);
170 assert(offset < m_buffer.committed());
171 return offset;
172 }
173
174 // This function decides whether it makes sense to garbage collect the
175 // database. The values here are the result of some experimentation
176 // with real data. We need to balance the memory use with the time
177 // spent on garbage collecting. We don't need to garbage collect if
178 // there is enough space in the buffer anyway (*4). On the other hand,
179 // if there aren't enough removed objects we would just call the
180 // garbage collection again and again, then it is better to let the
181 // buffer grow (*3). The checks (*1) and (*2) make sure there is
182 // minimum and maximum for the number of removed objects.
183 bool should_gc() const noexcept {
184 if (m_count_removed < 10UL * 1000UL) { // *1
185 return false;
186 }
187 if (m_count_removed > 5UL * 1000UL * 1000UL) { // *2
188 return true;
189 }
190 if (m_count_removed * 5UL < m_count_items) { // *3
191 return false;
192 }
193 return m_buffer.capacity() - m_buffer.committed() < 10UL * 1024UL; // *4
194 }
195
196 public:
197
198 ItemStash() :
199 m_buffer(initial_buffer_size, osmium::memory::Buffer::auto_grow::yes) {
200 }
201
208 std::size_t used_memory() const noexcept {
209 return sizeof(ItemStash) +
210 m_buffer.capacity() +
211 (m_index.capacity() * sizeof(std::size_t));
212 }
213
220 std::size_t size() const noexcept {
221 return m_count_items;
222 }
223
230 std::size_t count_removed() const noexcept {
231 return m_count_removed;
232 }
233
238 void clear() {
239 m_buffer.clear();
240 m_index.clear();
241 m_count_items = 0;
242 m_count_removed = 0;
243 }
244
251 handle_type add_item(const osmium::memory::Item& item) {
252 if (should_gc()) {
253 garbage_collect();
254 }
255 ++m_count_items;
256 const auto offset = m_buffer.committed();
257 m_buffer.add_item(item);
258 m_buffer.commit();
259 m_index.push_back(offset);
260 return handle_type{m_index.size()};
261 }
262
274 osmium::memory::Item& get_item(handle_type handle) const {
275 return m_buffer.get<osmium::memory::Item>(get_item_offset(handle));
276 }
277
293 template <typename T>
294 T& get(handle_type handle) const {
295 return static_cast<T&>(get_item(handle));
296 }
297
306 void garbage_collect() {
307#ifdef OSMIUM_ITEM_STORAGE_GC_DEBUG
308 std::cerr << "GC items=" << m_count_items << " removed=" << m_count_removed << " buffer.committed=" << m_buffer.committed() << " buffer.capacity=" << m_buffer.capacity() << "\n";
309 using clock = std::chrono::high_resolution_clock;
310 std::chrono::time_point<clock> start = clock::now();
311#endif
312
313 m_count_removed = 0;
314 cleanup_helper helper{m_index};
315 m_buffer.purge_removed(&helper);
316
317#ifdef OSMIUM_ITEM_STORAGE_GC_DEBUG
318 std::chrono::time_point<clock> stop = clock::now();
319 const int64_t time = std::chrono::duration_cast<std::chrono::microseconds>(stop - start).count();
320 m_gc_time += time;
321 std::cerr << " time=" << time
322 << "us total=" << m_gc_time << "us\n";
323#endif
324 }
325
338 void remove_item(handle_type handle) {
339 auto& offset = get_item_offset_ref(handle);
340 auto& item = m_buffer.get<osmium::memory::Item>(offset);
341 assert(!item.removed() && "can not call remove_item() on already removed item");
342 item.set_removed(true);
343 offset = removed_item_offset;
344 --m_count_items;
345 ++m_count_removed;
346 }
347
348 }; // class ItemStash
349
350} // namespace osmium
351
352#endif // OSMIUM_STORAGE_ITEM_STASH_HPP
buffer.hpp
osmium::ItemStash::cleanup_helper
Definition item_stash.hpp:134
osmium::ItemStash::cleanup_helper::m_index
std::vector< std::size_t > & m_index
Definition item_stash.hpp:136
osmium::ItemStash::cleanup_helper::cleanup_helper
cleanup_helper(std::vector< std::size_t > &index)
Definition item_stash.hpp:141
osmium::ItemStash::cleanup_helper::moving_in_buffer
void moving_in_buffer(std::size_t old_offset, std::size_t new_offset)
Definition item_stash.hpp:145
osmium::ItemStash::cleanup_helper::m_pos
std::size_t m_pos
Definition item_stash.hpp:137
osmium::ItemStash::handle_type
Definition item_stash.hpp:71
osmium::ItemStash::handle_type::valid
bool valid() const noexcept
Is this a valid handle?
Definition item_stash.hpp:95
osmium::ItemStash::handle_type::ItemStash
friend class ItemStash
Definition item_stash.hpp:73
osmium::ItemStash::handle_type::handle_type
handle_type(std::size_t new_value) noexcept
Definition item_stash.hpp:82
osmium::ItemStash::handle_type::operator<<
friend std::basic_ostream< TChar, TTraits > & operator<<(std::basic_ostream< TChar, TTraits > &out, const ItemStash::handle_type &handle)
Definition item_stash.hpp:105
osmium::ItemStash::handle_type::handle_type
handle_type() noexcept
The default constructor creates an invalid handle.
Definition item_stash.hpp:90
osmium::ItemStash::handle_type::value
std::size_t value
Definition item_stash.hpp:75
osmium::ItemStash::removed_item_offset
@ removed_item_offset
Definition item_stash.hpp:123
osmium::ItemStash::get
T & get(handle_type handle) const
Definition item_stash.hpp:294
osmium::ItemStash::remove_item
void remove_item(handle_type handle)
Definition item_stash.hpp:338
osmium::ItemStash::get_item_offset
std::size_t get_item_offset(handle_type handle) const noexcept
Definition item_stash.hpp:165
osmium::ItemStash::get_item_offset_ref
std::size_t & get_item_offset_ref(handle_type handle) noexcept
Definition item_stash.hpp:156
osmium::ItemStash::get_item
osmium::memory::Item & get_item(handle_type handle) const
Definition item_stash.hpp:274
osmium::ItemStash::m_index
std::vector< std::size_t > m_index
Definition item_stash.hpp:127
osmium::ItemStash::initial_buffer_size
@ initial_buffer_size
Definition item_stash.hpp:119
osmium::ItemStash::add_item
handle_type add_item(const osmium::memory::Item &item)
Definition item_stash.hpp:251
osmium::ItemStash::clear
void clear()
Definition item_stash.hpp:238
osmium::ItemStash::m_count_items
std::size_t m_count_items
Definition item_stash.hpp:128
osmium::ItemStash::size
std::size_t size() const noexcept
Definition item_stash.hpp:220
osmium::ItemStash::count_removed
std::size_t count_removed() const noexcept
Definition item_stash.hpp:230
osmium::ItemStash::garbage_collect
void garbage_collect()
Definition item_stash.hpp:306
osmium::ItemStash::used_memory
std::size_t used_memory() const noexcept
Definition item_stash.hpp:208
osmium::ItemStash::ItemStash
ItemStash()
Definition item_stash.hpp:198
osmium::ItemStash::m_buffer
osmium::memory::Buffer m_buffer
Definition item_stash.hpp:126
osmium::ItemStash::should_gc
bool should_gc() const noexcept
Definition item_stash.hpp:183
osmium::ItemStash::m_count_removed
std::size_t m_count_removed
Definition item_stash.hpp:129
osmium::memory::Item
Definition item.hpp:105
item.hpp
osmium::index
Indexing of OSM data, Locations, etc.
Definition id_set.hpp:51
osmium::memory
Definition osm_object_builder.hpp:66
osmium
Namespace for everything in the Osmium library.
Definition assembler.hpp:53