Libosmium  2.11.3
Fast and flexible C++ library for working with OpenStreetMap data
buffer.hpp
Go to the documentation of this file.
1 #ifndef OSMIUM_MEMORY_BUFFER_HPP
2 #define OSMIUM_MEMORY_BUFFER_HPP
3 
4 /*
5 
6 This file is part of Osmium (http://osmcode.org/libosmium).
7 
8 Copyright 2013-2017 Jochen Topf <jochen@topf.org> and others (see README).
9 
10 Boost Software License - Version 1.0 - August 17th, 2003
11 
12 Permission is hereby granted, free of charge, to any person or organization
13 obtaining a copy of the software and accompanying documentation covered by
14 this license (the "Software") to use, reproduce, display, distribute,
15 execute, and transmit the Software, and to prepare derivative works of the
16 Software, and to permit third-parties to whom the Software is furnished to
17 do so, all subject to the following:
18 
19 The copyright notices in the Software and this entire statement, including
20 the above license grant, this restriction and the following disclaimer,
21 must be included in all copies of the Software, in whole or in part, and
22 all derivative works of the Software, unless such copies or derivative
23 works are solely in the form of machine-executable object code generated by
24 a source language processor.
25 
26 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
29 SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
30 FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
31 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
32 DEALINGS IN THE SOFTWARE.
33 
34 */
35 
36 #include <algorithm>
37 #include <cassert>
38 #include <cstddef>
39 #include <cstring>
40 #include <functional>
41 #include <iterator>
42 #include <memory>
43 #include <stdexcept>
44 #include <utility>
45 
46 #include <osmium/memory/item.hpp>
48 #include <osmium/osm/entity.hpp>
50 
51 namespace osmium {
52 
58  struct buffer_is_full : public std::runtime_error {
59 
61  std::runtime_error("Osmium buffer is full") {
62  }
63 
64  }; // struct buffer_is_full
65 
69  namespace memory {
70 
97  class Buffer {
98 
99  public:
100 
101  // This is needed so we can call std::back_inserter() on a Buffer.
102  using value_type = Item;
103 
104  enum class auto_grow : bool {
105  yes = true,
106  no = false
107  }; // enum class auto_grow
108 
109  private:
110 
111  std::unique_ptr<unsigned char[]> m_memory;
112  unsigned char* m_data;
113  size_t m_capacity;
114  size_t m_written;
115  size_t m_committed;
116 #ifndef NDEBUG
117  uint8_t m_builder_count{0};
118 #endif
119  auto_grow m_auto_grow{auto_grow::no};
120  std::function<void(Buffer&)> m_full;
121 
122  static size_t calculate_capacity(size_t capacity) noexcept {
123  // The majority of all Nodes will fit into this size.
124  constexpr static const size_t min_capacity = 64;
125  if (capacity < min_capacity) {
126  return min_capacity;
127  }
128  return capacity;
129  }
130 
131  public:
132 
141  Buffer() noexcept :
142  m_memory(),
143  m_data(nullptr),
144  m_capacity(0),
145  m_written(0),
146  m_committed(0) {
147  }
148 
159  explicit Buffer(unsigned char* data, size_t size) :
160  m_memory(),
161  m_data(data),
162  m_capacity(size),
163  m_written(size),
164  m_committed(size) {
165  if (size % align_bytes != 0) {
166  throw std::invalid_argument("buffer size needs to be multiple of alignment");
167  }
168  }
169 
181  explicit Buffer(unsigned char* data, size_t capacity, size_t committed) :
182  m_memory(),
183  m_data(data),
184  m_capacity(capacity),
185  m_written(committed),
186  m_committed(committed) {
187  if (capacity % align_bytes != 0) {
188  throw std::invalid_argument("buffer capacity needs to be multiple of alignment");
189  }
190  if (committed % align_bytes != 0) {
191  throw std::invalid_argument("buffer parameter 'committed' needs to be multiple of alignment");
192  }
193  }
194 
209  explicit Buffer(size_t capacity, auto_grow auto_grow = auto_grow::yes) :
210  m_memory(new unsigned char[calculate_capacity(capacity)]),
211  m_data(m_memory.get()),
212  m_capacity(calculate_capacity(capacity)),
213  m_written(0),
214  m_committed(0),
215  m_auto_grow(auto_grow) {
216  if (m_capacity % align_bytes != 0) {
217  throw std::invalid_argument("buffer capacity needs to be multiple of alignment");
218  }
219  }
220 
221  // buffers can not be copied
222  Buffer(const Buffer&) = delete;
223  Buffer& operator=(const Buffer&) = delete;
224 
225  // buffers can be moved
226  Buffer(Buffer&&) = default;
227  Buffer& operator=(Buffer&&) = default;
228 
229  ~Buffer() = default;
230 
231 #ifndef NDEBUG
232  void increment_builder_count() noexcept {
233  ++m_builder_count;
234  }
235 
236  void decrement_builder_count() noexcept {
237  assert(m_builder_count > 0);
238  --m_builder_count;
239  }
240 
241  uint8_t builder_count() const noexcept {
242  return m_builder_count;
243  }
244 #endif
245 
251  unsigned char* data() const noexcept {
252  assert(m_data && "This must be a valid buffer");
253  return m_data;
254  }
255 
260  size_t capacity() const noexcept {
261  return m_capacity;
262  }
263 
268  size_t committed() const noexcept {
269  return m_committed;
270  }
271 
277  size_t written() const noexcept {
278  return m_written;
279  }
280 
287  bool is_aligned() const noexcept {
288  assert(m_data && "This must be a valid buffer");
289  return (m_written % align_bytes == 0) && (m_committed % align_bytes == 0);
290  }
291 
312  OSMIUM_DEPRECATED void set_full_callback(std::function<void(Buffer&)> full) {
313  assert(m_data && "This must be a valid buffer");
314  m_full = full;
315  }
316 
333  void grow(size_t size) {
334  assert(m_data && "This must be a valid buffer");
335  if (!m_memory) {
336  throw std::logic_error("Can't grow Buffer if it doesn't use internal memory management.");
337  }
338  if (m_capacity < size) {
339  if (size % align_bytes != 0) {
340  throw std::invalid_argument("buffer capacity needs to be multiple of alignment");
341  }
342  std::unique_ptr<unsigned char[]> memory(new unsigned char[size]);
343  std::copy_n(m_memory.get(), m_capacity, memory.get());
344  using std::swap;
345  swap(m_memory, memory);
346  m_data = m_memory.get();
347  m_capacity = size;
348  }
349  }
350 
363  size_t commit() {
364  assert(m_data && "This must be a valid buffer");
365  assert(m_builder_count == 0 && "Make sure there are no Builder objects still in scope");
366  assert(is_aligned());
367 
368  const size_t offset = m_committed;
369  m_committed = m_written;
370  return offset;
371  }
372 
379  void rollback() {
380  assert(m_data && "This must be a valid buffer");
381  assert(m_builder_count == 0 && "Make sure there are no Builder objects still in scope");
382  m_written = m_committed;
383  }
384 
394  size_t clear() {
395  assert(m_builder_count == 0 && "Make sure there are no Builder objects still in scope");
396  const size_t committed = m_committed;
397  m_written = 0;
398  m_committed = 0;
399  return committed;
400  }
401 
412  template <typename T>
413  T& get(const size_t offset) const {
414  assert(m_data && "This must be a valid buffer");
415  return *reinterpret_cast<T*>(&m_data[offset]);
416  }
417 
451  unsigned char* reserve_space(const size_t size) {
452  assert(m_data && "This must be a valid buffer");
453  // try to flush the buffer empty first.
454  if (m_written + size > m_capacity && m_full) {
455  m_full(*this);
456  }
457  // if there's still not enough space, then try growing the buffer.
458  if (m_written + size > m_capacity) {
459  if (m_memory && (m_auto_grow == auto_grow::yes)) {
460  // double buffer size until there is enough space
461  size_t new_capacity = m_capacity * 2;
462  while (m_written + size > new_capacity) {
463  new_capacity *= 2;
464  }
465  grow(new_capacity);
466  } else {
467  throw osmium::buffer_is_full();
468  }
469  }
470  unsigned char* data = &m_data[m_written];
471  m_written += size;
472  return data;
473  }
474 
490  template <typename T>
491  T& add_item(const T& item) {
492  assert(m_data && "This must be a valid buffer");
493  unsigned char* target = reserve_space(item.padded_size());
494  std::copy_n(reinterpret_cast<const unsigned char*>(&item), item.padded_size(), target);
495  return *reinterpret_cast<T*>(target);
496  }
497 
509  void add_buffer(const Buffer& buffer) {
510  assert(m_data && "This must be a valid buffer");
511  assert(buffer && "Buffer parameter must be a valid buffer");
512  assert(m_builder_count == 0 && "Make sure there are no Builder objects still in scope");
513  unsigned char* target = reserve_space(buffer.committed());
514  std::copy_n(buffer.data(), buffer.committed(), target);
515  }
516 
526  void push_back(const osmium::memory::Item& item) {
527  assert(m_data && "This must be a valid buffer");
528  assert(m_builder_count == 0 && "Make sure there are no Builder objects still in scope");
529  add_item(item);
530  commit();
531  }
532 
537  template <typename T>
539 
544  template <typename T>
546 
552 
558 
559  template <typename T>
561  return ItemIteratorRange<T>{m_data, m_data + m_committed};
562  }
563 
564  template <typename T>
566  return ItemIteratorRange<const T>{m_data, m_data + m_committed};
567  }
568 
577  template <typename T>
579  assert(m_data && "This must be a valid buffer");
580  return t_iterator<T>(m_data, m_data + m_committed);
581  }
582 
592  assert(m_data && "This must be a valid buffer");
593  return iterator(m_data, m_data + m_committed);
594  }
595 
605  template <typename T>
606  t_iterator<T> get_iterator(size_t offset) {
607  assert(m_data && "This must be a valid buffer");
608  return t_iterator<T>(m_data + offset, m_data + m_committed);
609  }
610 
620  iterator get_iterator(size_t offset) {
621  assert(m_data && "This must be a valid buffer");
622  return iterator(m_data + offset, m_data + m_committed);
623  }
624 
633  template <typename T>
635  assert(m_data && "This must be a valid buffer");
636  return t_iterator<T>(m_data + m_committed, m_data + m_committed);
637  }
638 
648  assert(m_data && "This must be a valid buffer");
649  return iterator(m_data + m_committed, m_data + m_committed);
650  }
651 
652  template <typename T>
654  assert(m_data && "This must be a valid buffer");
655  return t_const_iterator<T>(m_data, m_data + m_committed);
656  }
657 
659  assert(m_data && "This must be a valid buffer");
660  return const_iterator(m_data, m_data + m_committed);
661  }
662 
663  template <typename T>
664  t_const_iterator<T> get_iterator(size_t offset) const {
665  assert(m_data && "This must be a valid buffer");
666  return t_const_iterator<T>(m_data + offset, m_data + m_committed);
667  }
668 
669  const_iterator get_iterator(size_t offset) const {
670  assert(m_data && "This must be a valid buffer");
671  return const_iterator(m_data + offset, m_data + m_committed);
672  }
673 
674  template <typename T>
676  assert(m_data && "This must be a valid buffer");
677  return t_const_iterator<T>(m_data + m_committed, m_data + m_committed);
678  }
679 
681  assert(m_data && "This must be a valid buffer");
682  return const_iterator(m_data + m_committed, m_data + m_committed);
683  }
684 
685  template <typename T>
687  return cbegin<T>();
688  }
689 
691  return cbegin();
692  }
693 
694  template <typename T>
696  return cend<T>();
697  }
698 
699  const_iterator end() const {
700  return cend();
701  }
702 
706  explicit operator bool() const noexcept {
707  return m_data != nullptr;
708  }
709 
710  void swap(Buffer& other) {
711  using std::swap;
712 
713  swap(m_memory, other.m_memory);
714  swap(m_data, other.m_data);
715  swap(m_capacity, other.m_capacity);
716  swap(m_written, other.m_written);
717  swap(m_committed, other.m_committed);
718  swap(m_auto_grow, other.m_auto_grow);
719  swap(m_full, other.m_full);
720  }
721 
738  template <typename TCallbackClass>
739  void purge_removed(TCallbackClass* callback) {
740  assert(m_data && "This must be a valid buffer");
741  if (begin() == end()) {
742  return;
743  }
744 
745  iterator it_write = begin();
746 
747  iterator next;
748  for (iterator it_read = begin(); it_read != end(); it_read = next) {
749  next = std::next(it_read);
750  if (!it_read->removed()) {
751  if (it_read != it_write) {
752  assert(it_read.data() >= data());
753  assert(it_write.data() >= data());
754  size_t old_offset = static_cast<size_t>(it_read.data() - data());
755  size_t new_offset = static_cast<size_t>(it_write.data() - data());
756  callback->moving_in_buffer(old_offset, new_offset);
757  std::memmove(it_write.data(), it_read.data(), it_read->padded_size());
758  }
759  it_write.advance_once();
760  }
761  }
762 
763  assert(it_write.data() >= data());
764  m_written = static_cast<size_t>(it_write.data() - data());
765  m_committed = m_written;
766  }
767 
768  }; // class Buffer
769 
770  inline void swap(Buffer& lhs, Buffer& rhs) {
771  lhs.swap(rhs);
772  }
773 
781  inline bool operator==(const Buffer& lhs, const Buffer& rhs) noexcept {
782  if (!lhs || !rhs) {
783  return !lhs && !rhs;
784  }
785  return lhs.data() == rhs.data() && lhs.capacity() == rhs.capacity() && lhs.committed() == rhs.committed();
786  }
787 
788  inline bool operator!=(const Buffer& lhs, const Buffer& rhs) noexcept {
789  return ! (lhs == rhs);
790  }
791 
792  } // namespace memory
793 
794 } // namespace osmium
795 
796 #endif // OSMIUM_MEMORY_BUFFER_HPP
size_t m_written
Definition: buffer.hpp:114
void swap(Buffer &other)
Definition: buffer.hpp:710
t_const_iterator< T > begin() const
Definition: buffer.hpp:686
size_t clear()
Definition: buffer.hpp:394
#define OSMIUM_DEPRECATED
Definition: compatibility.hpp:50
bool is_aligned() const noexcept
Definition: buffer.hpp:287
t_const_iterator< T > get_iterator(size_t offset) const
Definition: buffer.hpp:664
size_t written() const noexcept
Definition: buffer.hpp:277
OSMIUM_DEPRECATED void set_full_callback(std::function< void(Buffer &)> full)
Definition: buffer.hpp:312
Definition: item_iterator.hpp:175
iterator get_iterator(size_t offset)
Definition: buffer.hpp:620
void grow(size_t size)
Definition: buffer.hpp:333
t_const_iterator< T > cend() const
Definition: buffer.hpp:675
const_iterator cend() const
Definition: buffer.hpp:680
Definition: item_iterator.hpp:59
constexpr bool operator==(const Box &lhs, const Box &rhs) noexcept
Definition: box.hpp:221
unsigned char * m_data
Definition: buffer.hpp:112
const_iterator get_iterator(size_t offset) const
Definition: buffer.hpp:669
void increment_builder_count() noexcept
Definition: buffer.hpp:232
Buffer(unsigned char *data, size_t capacity, size_t committed)
Definition: buffer.hpp:181
Definition: reader_iterator.hpp:39
ItemIteratorRange< const T > select() const
Definition: buffer.hpp:565
void swap(Buffer &lhs, Buffer &rhs)
Definition: buffer.hpp:770
Buffer(size_t capacity, auto_grow auto_grow=auto_grow::yes)
Definition: buffer.hpp:209
const_iterator begin() const
Definition: buffer.hpp:690
t_iterator< T > end()
Definition: buffer.hpp:634
Definition: item.hpp:105
Namespace for everything in the Osmium library.
Definition: assembler.hpp:73
T & add_item(const T &item)
Definition: buffer.hpp:491
void purge_removed(TCallbackClass *callback)
Definition: buffer.hpp:739
t_iterator< T > begin()
Definition: buffer.hpp:578
void add_buffer(const Buffer &buffer)
Definition: buffer.hpp:509
uint8_t builder_count() const noexcept
Definition: buffer.hpp:241
size_t m_committed
Definition: buffer.hpp:115
ItemIterator< TMember > & advance_once() noexcept
Definition: item_iterator.hpp:114
unsigned char * data() const noexcept
Definition: buffer.hpp:251
Buffer() noexcept
Definition: buffer.hpp:141
size_t capacity() const noexcept
Definition: buffer.hpp:260
osmium::io::InputIterator< osmium::io::Reader > end(osmium::io::Reader &)
Definition: reader_iterator.hpp:45
unsigned char * reserve_space(const size_t size)
Definition: buffer.hpp:451
auto_grow m_auto_grow
Definition: buffer.hpp:119
void push_back(const osmium::memory::Item &item)
Definition: buffer.hpp:526
constexpr const item_size_type align_bytes
Definition: item.hpp:62
iterator end()
Definition: buffer.hpp:647
iterator begin()
Definition: buffer.hpp:591
size_t m_capacity
Definition: buffer.hpp:113
size_t committed() const noexcept
Definition: buffer.hpp:268
Buffer(unsigned char *data, size_t size)
Definition: buffer.hpp:159
static size_t calculate_capacity(size_t capacity) noexcept
Definition: buffer.hpp:122
Definition: buffer.hpp:97
t_const_iterator< T > cbegin() const
Definition: buffer.hpp:653
Definition: buffer.hpp:58
void decrement_builder_count() noexcept
Definition: buffer.hpp:236
const_iterator end() const
Definition: buffer.hpp:699
auto_grow
Definition: buffer.hpp:104
t_const_iterator< T > end() const
Definition: buffer.hpp:695
data_type data() noexcept
Definition: item_iterator.hpp:135
buffer_is_full()
Definition: buffer.hpp:60
ItemIteratorRange< T > select()
Definition: buffer.hpp:560
osmium::io::InputIterator< osmium::io::Reader > begin(osmium::io::Reader &reader)
Definition: reader_iterator.hpp:41
void rollback()
Definition: buffer.hpp:379
std::unique_ptr< unsigned char[]> m_memory
Definition: buffer.hpp:111
bool operator!=(const Changeset &lhs, const Changeset &rhs)
Definition: changeset.hpp:440
t_iterator< T > get_iterator(size_t offset)
Definition: buffer.hpp:606
const_iterator cbegin() const
Definition: buffer.hpp:658
std::function< void(Buffer &)> m_full
Definition: buffer.hpp:120
size_t commit()
Definition: buffer.hpp:363