36 #ifndef VIGRA_MULTI_ARRAY_CHUNKED_HDF5_HXX
37 #define VIGRA_MULTI_ARRAY_CHUNKED_HDF5_HXX
41 #include "multi_array_chunked.hxx"
42 #include "hdf5impex.hxx"
45 #ifdef VIGRA_CHECK_BOUNDS
46 #define VIGRA_ASSERT_INSIDE(diff) \
47 vigra_precondition(this->isInside(diff), "Index out of bounds")
49 #define VIGRA_ASSERT_INSIDE(diff)
67 template <
unsigned int N,
class T,
class Alloc = std::allocator<T> >
83 :
public ChunkBase<N, T>
88 typedef value_type * pointer;
89 typedef value_type & reference;
93 : ChunkBase<N, T>(detail::defaultStride(shape))
105 std::size_t
size()
const
110 void write(
bool deallocate =
true)
112 if(this->pointer_ != 0)
114 if(!array_->file_.isReadOnly())
116 herr_t status = array_->file_.writeBlock(array_->dataset_, start_,
118 vigra_postcondition(status >= 0,
119 "ChunkedArrayHDF5: write to dataset failed.");
123 alloc_.deallocate(this->pointer_, this->
size());
131 if(this->pointer_ == 0)
133 this->pointer_ = alloc_.allocate(this->
size());
134 herr_t status = array_->file_.readBlock(array_->dataset_, start_, shape_,
136 vigra_postcondition(status >= 0,
137 "ChunkedArrayHDF5: read from dataset failed.");
139 return this->pointer_;
147 Chunk & operator=(Chunk
const &);
153 typedef T value_type;
154 typedef value_type * pointer;
155 typedef value_type & reference;
188 Alloc
const & alloc = Alloc())
191 dataset_name_(dataset),
193 compression_(options.compression_method),
224 Alloc
const & alloc = Alloc())
227 dataset_name_(dataset),
229 compression_(options.compression_method),
239 if(mode == HDF5File::Replace)
241 mode = HDF5File::New;
243 else if(mode == HDF5File::Default)
246 mode = HDF5File::ReadOnly;
248 mode = HDF5File::New;
251 if(mode == HDF5File::ReadOnly)
254 vigra_precondition(!file_.isReadOnly(),
255 "ChunkedArrayHDF5(): 'mode' is incompatible with read-only file.");
257 vigra_precondition(exists || !file_.isReadOnly(),
258 "ChunkedArrayHDF5(): dataset does not exist, but file is read-only.");
260 if(!exists || mode == HDF5File::New)
279 if(compression_ == DEFAULT_COMPRESSION)
280 compression_ = ZLIB_FAST;
281 vigra_precondition(compression_ != LZ4,
282 "ChunkedArrayHDF5(): HDF5 does not support LZ4 compression.");
284 vigra_precondition(this->
size() > 0,
285 "ChunkedArrayHDF5(): invalid shape.");
286 typename detail::HDF5TypeTraits<T>::value_type init(this->fill_scalar_);
299 typedef detail::HDF5TypeTraits<T> TypeTraits;
300 if(TypeTraits::numberOfBands() > 1)
302 vigra_precondition(fileShape.size() == N+1,
303 "ChunkedArrayHDF5(file, dataset): dataset has wrong dimension.");
304 vigra_precondition(fileShape[0] == TypeTraits::numberOfBands(),
305 "ChunkedArrayHDF5(file, dataset): dataset has wrong number of bands.");
306 shape_type
shape(fileShape.begin()+1);
309 vigra_precondition(shape == this->shape_,
310 "ChunkedArrayHDF5(file, dataset, shape): shape mismatch between dataset and shape argument.");
314 this->shape_ =
shape;
319 vigra_precondition(fileShape.size() == N,
320 "ChunkedArrayHDF5(file, dataset): dataset has wrong dimension.");
321 shape_type
shape(fileShape.begin());
324 vigra_precondition(shape == this->shape_,
325 "ChunkedArrayHDF5(file, dataset, shape): shape mismatch between dataset and shape argument.");
329 this->shape_ =
shape;
330 ChunkStorage(detail::computeChunkArrayShape(shape, this->bits_, this->mask_)).swap(this->handle_array_);
334 end = this->handle_array_.
end();
337 i->chunk_state_.store(base_type::chunk_asleep);
352 void closeImpl(
bool force_destroy)
354 flushToDiskImpl(
true, force_destroy);
360 flushToDiskImpl(
false,
false);
363 void flushToDiskImpl(
bool destroy,
bool force_destroy)
365 if(file_.isReadOnly())
368 threading::lock_guard<threading::mutex> guard(*this->chunk_lock_);
370 end = this->handle_array_.
end();
371 if(destroy && !force_destroy)
375 vigra_precondition(i->chunk_state_.load() <= 0,
376 "ChunkedArrayHDF5::close(): cannot close file because there are active chunks.");
378 i = this->handle_array_.
begin();
382 Chunk * chunk =
static_cast<Chunk*
>(i->pointer_);
398 virtual bool isReadOnly()
const
400 return file_.isReadOnly();
403 virtual pointer loadChunk(ChunkBase<N, T> ** p, shape_type
const & index)
405 vigra_precondition(file_.isOpen(),
406 "ChunkedArrayHDF5::loadChunk(): file was already closed.");
409 *p =
new Chunk(this->
chunkShape(index), index*this->chunk_shape_,
this, alloc_);
410 this->overhead_bytes_ +=
sizeof(Chunk);
412 return static_cast<Chunk *
>(*p)->read();
415 virtual bool unloadChunk(ChunkBase<N, T> * chunk,
bool )
419 static_cast<Chunk *
>(chunk)->write();
423 virtual std::string backend()
const
425 return "ChunkedArrayHDF5<'" + file_.
filename() +
"/" + dataset_name_ +
"'>";
428 virtual std::size_t
dataBytes(ChunkBase<N,T> * c)
const
430 return c->pointer_ == 0
432 :
static_cast<Chunk*
>(c)->
size()*
sizeof(T);
437 return sizeof(Chunk) +
sizeof(SharedChunkHandle<N, T>);
440 std::string fileName()
const
445 std::string datasetName()
const
447 return dataset_name_;
451 std::string dataset_name_;
452 HDF5HandleShared dataset_;
453 CompressionMethod compression_;
461 #undef VIGRA_ASSERT_INSIDE
iterator end()
Definition: multi_array.hxx:1885
std::size_t dataBytes() const
Bytes of main memory occupied by the array's data.
Definition: multi_array_chunked.hxx:1677
Option object for ChunkedArray construction.
Definition: multi_array_chunked.hxx:1274
ArrayVector< hsize_t > getDatasetShape(std::string datasetName) const
Get the shape of each dimension of a certain dataset.
Definition: hdf5impex.hxx:1369
MultiArrayIndex size() const
Return the number of elements in this array.
ChunkedArrayHDF5(HDF5File const &file, std::string const &dataset, HDF5File::OpenMode mode=HDF5File::ReadOnly, ChunkedArrayOptions const &options=ChunkedArrayOptions(), Alloc const &alloc=Alloc())
Construct for an already existing dataset with given 'options', using 'alloc' to manage the in-memory...
Definition: multi_array_chunked_hdf5.hxx:221
view_type::iterator iterator
Definition: multi_array.hxx:2496
iterator begin()
Definition: multi_array.hxx:1869
Main MultiArray class containing the memory management.
Definition: multi_array.hxx:2422
shape_type const & shape() const
Return the shape in this array.
HDF5HandleShared getDatasetHandleShared(std::string const &datasetName) const
Obtain a shared HDF5 handle of a dataset.
Definition: hdf5impex.hxx:1464
Interface and base class for chunked arrays.
Definition: multi_array_chunked.hxx:470
HDF5HandleShared createDataset(std::string datasetName, TinyVector< MultiArrayIndex, N > const &shape, typename detail::HDF5TypeTraits< T >::value_type init=typename detail::HDF5TypeTraits< T >::value_type(), TinyVector< MultiArrayIndex, N > const &chunkSize=(TinyVector< MultiArrayIndex, N >()), int compressionParameter=0)
Create a new dataset. This function can be used to create a dataset filled with a default value init...
Definition: hdf5impex.hxx:2702
view_type::difference_type difference_type
Definition: multi_array.hxx:2470
shape_type const & chunkShape() const
Return the global chunk shape.
std::string filename() const
Get the name of the associated file.
Definition: hdf5impex.hxx:1322
NumericTraits< V >::Promote prod(TinyVectorBase< V, SIZE, D1, D2 > const &l)
product of the vector's elements
Definition: tinyvector.hxx:2097
Definition: multi_array_chunked_hdf5.hxx:68
ChunkedArrayHDF5(HDF5File const &file, std::string const &dataset, HDF5File::OpenMode mode, shape_type const &shape, shape_type const &chunk_shape=shape_type(), ChunkedArrayOptions const &options=ChunkedArrayOptions(), Alloc const &alloc=Alloc())
Construct with given 'shape', 'chunk_shape' and 'options', using 'alloc' to manage the in-memory vers...
Definition: multi_array_chunked_hdf5.hxx:183
iterator end()
Create the end iterator for scan-order iteration over the entire chunked array.
Definition: multi_array_chunked.hxx:2391
OpenMode
Set how a file is opened.
Definition: hdf5impex.hxx:1013
void flushToDisk()
Immediately write all data to disk.
Definition: hdf5impex.hxx:2171
Class for fixed size vectors.This class contains an array of size SIZE of the specified VALUETYPE...
Definition: accessor.hxx:940
void close()
Close the current file.
Definition: hdf5impex.hxx:1174
Base class for, and view to, vigra::MultiArray.
Definition: multi_array.hxx:652
virtual std::size_t overheadBytesPerChunk() const
Bytes of main memory needed to manage a single chunk.
Definition: multi_array_chunked_hdf5.hxx:435
bool existsDataset(std::string datasetName) const
Check if given datasetName exists.
Definition: hdf5impex.hxx:1329
Access to HDF5 files.
Definition: hdf5impex.hxx:956