44 #ifndef _INCLUDED_Field3D_SparseFile_H_
45 #define _INCLUDED_Field3D_SparseFile_H_
71 template <
typename Data_T>
76 template <
typename Data_T>
94 template <
class Data_T>
101 typedef std::vector<Sparse::SparseBlock<Data_T>*>
BlockPtrs;
137 Reference(
const std::string filename,
const std::string layerPath);
214 template <
class Data_T>
219 template <
class Data_T>
223 template <
class Data_T>
247 blockType(blockTypeIn), refIdx(refIdxIn), blockIdx(blockIdxIn)
321 template <
class Data_T>
378 template <
class Data_T>
385 template <
class Data_T>
391 template <
class Data_T>
397 template <
class Data_T>
402 template <
class Data_T>
403 int getNextId(
const std::string filename,
const std::string layerPath);
405 template <
class Data_T>
425 template <
class Data_T>
429 template <
class Data_T>
474 template <
class Data_T>
476 const std::string a_layerPath)
477 : filename(a_filename), layerPath(a_layerPath),
478 valuesPerBlock(-1), occupiedBlocks(-1),
479 blockMutex(NULL), m_fileHandle(-1), m_reader(NULL) {
485 template <
class Data_T>
492 delete [] blockMutex;
497 template <
class Data_T>
507 template <
class Data_T>
524 blockMutex =
new boost::mutex[blocks.size()];
529 if (m_fileHandle >= 0) {
530 m_layerGroup.open(m_fileHandle, layerPath.c_str());
542 template <
class Data_T>
545 return m_fileHandle >= 0;
550 template <
class Data_T>
553 boost::mutex::scoped_lock lock(m_mutex);
555 fileBlockIndices.resize(numBlocks);
556 blockLoaded.resize(numBlocks, 0);
557 blocks.resize(numBlocks, 0);
558 blockUsed.resize(numBlocks,
false);
559 loadCounts.resize(numBlocks, 0);
560 refCounts.resize(numBlocks, 0);
563 blockMutex =
new boost::mutex[numBlocks];
568 template <
class Data_T>
574 boost::mutex::scoped_lock lock_A(m_mutex);
581 m_fileHandle = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
582 if (m_fileHandle < 0)
583 throw NoSuchFileException(filename);
585 m_layerGroup.open(m_fileHandle, layerPath.c_str());
586 if (m_layerGroup.id() < 0) {
588 "Couldn't find layer group " + layerPath +
590 throw FileIntegrityException(filename);
599 template <
class Data_T>
602 boost::mutex::scoped_lock lock(m_mutex);
605 blocks[blockIdx]->resize(valuesPerBlock);
606 assert(blocks[blockIdx]->data.size() > 0);
609 m_reader->readBlock(fileBlockIndices[blockIdx], blocks[blockIdx]->dataRef());
611 blockLoaded[blockIdx] = 1;
616 template <
class Data_T>
620 blocks[blockIdx]->clear();
623 blockLoaded[blockIdx] = 0;
628 template <
class Data_T>
631 boost::mutex::scoped_lock lock(blockMutex[blockIdx]);
632 ++refCounts[blockIdx];
637 template <
class Data_T>
640 boost::mutex::scoped_lock lock(blockMutex[blockIdx]);
641 --refCounts[blockIdx];
646 template <
class Data_T>
649 return valuesPerBlock *
sizeof(Data_T);
654 template <
class Data_T>
657 std::vector<int>::const_iterator i = loadCounts.begin();
658 std::vector<int>::const_iterator end = loadCounts.end();
660 for (; i != end; ++i)
668 template <
class Data_T>
671 std::vector<int>::const_iterator i = blockLoaded.begin();
672 std::vector<int>::const_iterator end = blockLoaded.end();
674 for (; i != end; ++i)
683 template <
class Data_T>
686 std::vector<int>::const_iterator i = loadCounts.begin();
687 std::vector<int>::const_iterator li = blockLoaded.begin();
688 std::vector<int>::const_iterator end = loadCounts.end();
691 if (blockLoaded.size() == 0) {
692 for (; i != end; ++i)
696 assert(loadCounts.size() == blockLoaded.size());
698 for (; i != end; ++i, ++li)
708 template <
class Data_T>
711 std::vector<int>::const_iterator i = loadCounts.begin();
712 std::vector<int>::const_iterator end = loadCounts.end();
713 int numLoads = 0, numBlocks = 0;
714 for (; i != end; ++i) {
721 return (
float)numLoads / std::max(1, numBlocks);
726 template <
class Data_T>
729 std::vector<int>::iterator li = loadCounts.begin();
730 std::vector<int>::iterator lend = loadCounts.end();
731 for (; li != lend; ++li)
748 inline Reference<half>&
856 inline int FileReferences::numRefs<half>()
const
858 return m_hRefs.size();
864 inline int FileReferences::numRefs<V3h>()
const
866 return m_vhRefs.size();
872 inline int FileReferences::numRefs<float>()
const
874 return m_fRefs.size();
880 inline int FileReferences::numRefs<V3f>()
const
882 return m_vfRefs.size();
888 inline int FileReferences::numRefs<double>()
const
890 return m_dRefs.size();
896 inline int FileReferences::numRefs<V3d>()
const
898 return m_vdRefs.size();
905 template <
class Data_T>
908 assert(
false &&
"Do not use memory limiting on sparse fields that aren't "
909 "simple scalars or vectors!");
911 "FileReferences::ref(): Do not use memory limiting on sparse "
912 "fields that aren't simple scalars or vectors!");
919 template <
class Data_T>
922 assert(
false &&
"Do not use memory limiting on sparse fields that aren't "
923 "simple scalars or vectors!");
925 "FileReferences::append(): Do not use memory limiting on sparse "
926 "fields that aren't simple scalars or vectors!");
932 template <
class Data_T>
935 assert(
false &&
"Do not use memory limiting on sparse fields that aren't "
936 "simple scalars or vectors!");
938 "FileReferences::numRefs(): "
939 "Do not use memory limiting on sparse "
940 "fields that aren't "
941 "simple scalars or vectors!");
953 template <
class Data_T>
956 const std::string layerPath)
966 template <
class Data_T>
970 boost::mutex::scoped_lock lock(
m_mutex);
977 CacheList::iterator next;
982 if (it->blockType == blockType && it->refIdx == refIdx) {
988 bytesFreed += reference.
blockSize(it->blockIdx);
1002 reference.
blocks.clear();
1009 template <
class Data_T>
1018 template <
class Data_T>
1026 int blockSize = reference.
blockSize(blockIdx);
1037 boost::mutex::scoped_lock lock_A(
m_mutex);
1038 boost::mutex::scoped_lock lock_B(reference.
blockMutex[blockIdx]);
1054 template <
class Data_T>
1067 template <
class Data_T>
void removeFieldFromCache(int refIdx)
#define FIELD3D_NAMESPACE_HEADER_CLOSE
long long totalLoads()
Returns the total number of block loads in the cache.
void resetCacheStatistics()
Resets block load.
Contains utility functions and classes for Hdf5 files.
hid_t m_fileHandle
Holds the Hdf5 handle to the file.
int deallocateBlock(const SparseFile::CacheBlock &cb)
Utility function to attempt to deallocate a single block and advance the "hand".
CacheList m_blockCacheList
List of dynamically loaded blocks to be considered for unloading when the cache is full...
Namespace for Exception objects.
int blockSize(int blockIdx) const
Returns the number of bytes used by the data in the block.
void activateBlock(int fileId, int blockIdx)
Called by SparseField when it's about to read from a block. This should not be called by the user...
std::vector< int > blockLoaded
Whether each block is loaded. We don't use bools since vector is weird.
void incBlockRef(int fileId, int blockIdx)
Increments the usage reference count on the specified block, to prevent it from getting unloaded whil...
void incBlockRef(int blockIdx)
Increment reference count on a block, indicates the block is currently in use, so prevents it from be...
float cacheEfficiency()
Computes the efficiency, the ratio of the number of blocks ever loaded to the number of loads...
int m_memUse
Current amount of memory in use in bytes.
std::vector< bool > blockUsed
Flags of whether the blocks have been accessed since they were last considered for deallocation by th...
CacheBlock(DataTypeEnum blockTypeIn, int refIdxIn, int blockIdxIn)
std::vector< Sparse::SparseBlock< Data_T > * > BlockPtrs
void deallocateBlocks(int bytesNeeded)
Utility function to reclaim the specified number of bytes by deallocating unneeded blocks...
Reference< Data_T > & ref(int idx)
Returns a reference to the index. This is specialized so that the correct data member is accessed...
void print(Severity severity, const std::string &message)
Sends the string to the assigned output, prefixing the message with the severity. ...
long long totalLoadedBlocks()
Returns the total number of blocks loaded (max 1 per block) into cache.
std::vector< int > fileBlockIndices
Index in file for each block.
int m_maxMemUseInBytes
Max amount om memory to use in bytes.
void setNumBlocks(int numBlocks)
Sets the number of blocks used by the SparseField we're supporting.
static DataTypeEnum typeEnum()
void decBlockRef(int blockIdx)
Decrement reference count on a block.
std::vector< int > refCounts
Per-block counts of the number of current references to the blocks. If a block's ref count is non-zer...
Contains various utility functions for Hdf5.
std::vector< Reference< double > > m_dRefs
void openFile()
Opens the file. This is done just before the first request to loadBlock. This is delayed so that the ...
float m_maxMemUse
Max amount om memory to use in megabytes.
SparseDataReader< Data_T > * m_reader
Pointer to the reader object. NULL at construction time. Created in openFile().
boost::mutex m_mutex
Mutex to prevent multiple threads from deallocating blocks at the same time.
std::vector< Reference< V3f > > m_vfRefs
bool m_limitMemUse
Whether to limit memory use of sparse fields from disk. Enables the cache and dynamic loading when tr...
float averageLoads() const
Returns the average number of loads per accessed block in this file, for cache statistics.
std::vector< Reference< V3h > > m_vhRefs
static SparseFileManager * ms_singleton
Pointer to singleton.
void setLimitMemUse(bool enabled)
Sets whether to limit memory usage and do dynamic loading for sparse fields.
This class gets used by SparseFieldIO and SparseFileManager to read the block data. On creation it will open the data set and not close it until the object is destroyed.
std::vector< Reference< half > > m_hRefs
float cacheFractionLoaded()
Computes the ratio of blocks in the cache to the total number of blocks that have been loaded (includ...
BlockPtrs blocks
Pointers to each block. This is so we can go in and manipulate them as we please. ...
long long numLoadedBlocks()
Returns the total number of blocks currently loaded into cache.
CacheList::iterator m_nextBlock
Pointer to the next block to test for unloading in the cache, the "hand" of the clock.
void unloadBlock(int blockIdx)
Unloads the block with the given index from memory.
SparseFile::FileReferences m_fileData
Vector containing information for each of the managed fields. The order matches the index stored in e...
Namespace for sparse field specifics.
std::list< SparseFile::CacheBlock > CacheList
int totalLoadedBlocks() const
Returns the total number of blocks that were ever loaded (max 1 per block, not the number of blocks)...
Contains functions controlling the loading of sparse fields.
boost::mutex m_mutex
Mutex to prevent two threads from modifying conflicting data.
int totalLoads() const
Returns the total number of loads of the blocks of this file, for cache statistics.
void resetCacheStatistics()
Resets counts of total block loads.
Hdf5Util::H5ScopedGopen m_layerGroup
Hold the group containing the data open for the duration of the Reference's existence.
SparseFile::Reference< Data_T > & reference(int index)
Returns a reference to the Reference object with the given index.
std::vector< Reference< float > > m_fRefs
int getNextId(const std::string filename, const std::string layerPath)
Returns the id of the next cache item. This is stored in the SparseField in order to reference its fi...
int numRefs() const
Returns the number of file references of the corresponding collection.
void addBlockToCache(DataTypeEnum blockType, int fileId, int blockIdx)
Adds the newly loaded block to the cache, managed by the paging algorithm.
SparseFileManager()
Private to prevent instantiation.
bool doLimitMemUse() const
Returns whether to limit memory usage and do dynamic loading for sparse fields.
void decBlockRef(int fileId, int blockIdx)
Decrements the usage reference count on the specified block, after its value is no longer being used ...
void loadBlock(int blockIdx)
Loads the block with the given index into memory. We don't pass in a reference to where the data shou...
Reference & operator=(const Reference &o)
Assignment operator. Clears ref counts and rebuilds mutex array.
void flushCache()
Flushes the entire block cache for all files, should probably only be used for debugging.
float cacheLoadsPerBlock()
Computes the overall loaded-blocks-to-load ratio for cached files.
Reference(const std::string filename, const std::string layerPath)
Constructor. Requires the filename and layer path of the field to be known.
bool fileIsOpen()
Checks if the file used by this reference is open already.
std::vector< int > loadCounts
Per-block counts of the number of times each block has been loaded, for cache statistics.
This Field subclass stores voxel data in block-allocated arrays.
static SparseFileManager & singleton()
Returns a reference to the singleton instance.
void setMaxMemUse(float maxMemUse)
Sets the maximum memory usage, in MB, by dynamically loaded sparse fields.
Scoped object - opens a group on creation and closes it on destruction.
Contains Exception base class.
std::vector< Reference< V3d > > m_vdRefs
boost::mutex * blockMutex
Allocated array of mutexes, one per block, to lock each block individually, for guaranteeing thread-s...
int numLoadedBlocks() const
Returns the total number of blocks that are currently loaded, for statistics.
int append(const Reference< Data_T > &ref)
Appends a reference to the collection. This is specialized so that the correct data member is accesse...