Hypertable::CellStoreV2 Class Reference

#include <CellStoreV2.h>

Inheritance diagram for Hypertable::CellStoreV2:

Collaboration diagram for Hypertable::CellStoreV2:

Classes
class	IndexBuilder

Public Member Functions
	CellStoreV2 (Filesystem *filesys)
virtual	~CellStoreV2 ()
void	create (const char *fname, size_t max_entries, PropertiesPtr &props, const TableIdentifier *table_identifier=0) override
	Creates a new cell store. More...
void	add (const Key &key, const ByteString value) override
	Inserts a key/value pair into the cell list. More...
void	finalize (TableIdentifier *table_identifier) override
	Finalizes the creation of a cell store, by writing block index and metadata trailer. More...
void	open (const String &fname, const String &start_row, const String &end_row, int32_t fd, int64_t file_length, CellStoreTrailer *trailer) override
	Opens a cell store with possibly a restricted view. More...
int64_t	get_blocksize () override
	Returns the block size used for this cell store. More...
bool	may_contain (const void *ptr, size_t len)
bool	may_contain (const String &key)
bool	may_contain (ScanContext *scan_ctx) override
	Bloom filter lookup. More...
uint64_t	disk_usage () override
	Returns the disk used by this cell store. More...
float	compression_ratio () override
	Returns block compression ratio of this cell store. More...
int64_t	get_total_entries () override
	Returns the number of key/value pairs in the cell store. More...
std::string &	get_filename () override
	Pathname of cell store file. More...
int	get_file_id () override
	Returns a unique identifier which identifies the underlying file for caching purposes. More...
CellListScannerPtr	create_scanner (ScanContext *scan_ctx) override
	Creates a scanner on this cell list. More...
BlockCompressionCodec *	create_block_compression_codec () override
	Creates a block compression codec suitable for decompressing the cell store's blocks. More...
void	display_block_info () override
	Displays block information to stdout. More...
int64_t	end_of_last_block () override
	Returns the offset of the end of the last block in the cell store. More...
size_t	bloom_filter_size () override
	Return Bloom filter size. More...
int64_t	bloom_filter_memory_used () override
	Returns the amount of memory consumed by the bloom filter. More...
int64_t	block_index_memory_used () override
	Returns the amount of memory consumed by the block index. More...
uint64_t	purge_indexes () override
	Purges bloom filter and block indexes. More...
bool	restricted_range () override
	Returns true if the cellstore was opened with a restricted range. More...
int32_t	get_fd () override
	Returns the open file descriptor for the CellStore file. More...
int32_t	reopen_fd () override
	Closes and reopens the underlying CellStore file. More...
CellStoreTrailer *	get_trailer () override
	Return a pointer to the trailer object for this cell store. More...
uint16_t	block_header_format () override
Public Member Functions inherited from Hypertable::CellStore
	CellStore ()
virtual	~CellStore ()
virtual void	split_row_estimate_data (SplitRowDataMapT &split_row_data)
	Populates split_row_data with unique row and count estimates from block index. More...
virtual void	populate_index_pseudo_table_scanner (CellListScannerBuffer *scanner)
	Populates scanner with key/value pairs generated from CellStore index. More...
virtual void	rescope (const String &start_row, const String &end_row)
virtual size_t	block_count ()
	Returns the number of CellStore blocks covered by this object. More...
virtual KeyDecompressor *	create_key_decompressor ()
	Creates a key decompressor suitable for decompressing the keys stored in this cell store. More...
virtual void	set_replaced_files (const std::vector< String > &old_files)
	Sets the cell store files replaced by this CellStore. More...
virtual const std::vector < String > &	get_replaced_files ()
	Returns all the cell store files replaced by this CellStore. More...
virtual void	get_index_memory_stats (IndexMemoryStats *statsp)
	Returns amount of purgeable index memory available. More...
uint64_t	bytes_read ()
	Returns the number of "uncompressed" bytes read from the underlying filesystem. More...
void	decrement_index_refcount ()
	Decrement index reference count. More...
Public Member Functions inherited from Hypertable::CellList
virtual	~CellList ()
virtual const char *	get_start_row ()
	Returns the start row of this cell list. More...
virtual const char *	get_end_row ()
	Returns the end row of this cell list. More...

Protected Types
typedef BlobHashSet	BloomFilterItems

Protected Member Functions
void	create_bloom_filter (bool is_approx=false)
void	load_bloom_filter ()
void	load_block_index ()

Protected Attributes
Filesystem *	m_filesys
int32_t	m_fd
std::string	m_filename
CellStoreBlockIndexArray < uint32_t >	m_index_map32
CellStoreBlockIndexArray< int64_t >	m_index_map64
bool	m_64bit_index
CellStoreTrailerV2	m_trailer
BlockCompressionCodec *	m_compressor
DynamicBuffer	m_buffer
IndexBuilder	m_index_builder
DispatchHandlerSynchronizer	m_sync_handler
uint32_t	m_outstanding_appends
int64_t	m_offset
ByteString	m_last_key
int64_t	m_file_length
int64_t	m_disk_usage
int	m_file_id
float	m_uncompressed_data
float	m_compressed_data
int64_t	m_uncompressed_blocksize
BlockCompressionCodec::Args	m_compressor_args
size_t	m_max_entries
BloomFilterMode	m_bloom_filter_mode
BloomFilter *	m_bloom_filter
BloomFilterItems *	m_bloom_filter_items
int64_t	m_max_approx_items
float	m_bloom_bits_per_item
float	m_filter_false_positive_prob
bool	m_restricted_range
Protected Attributes inherited from Hypertable::CellStore
std::mutex	m_mutex
IndexMemoryStats	m_index_stats
std::vector< String >	m_replaced_files
uint64_t	m_bytes_read
size_t	m_block_count
uint32_t	m_index_refcount
Protected Attributes inherited from Hypertable::CellList
std::string	m_start_row
std::string	m_end_row

Additional Inherited Members
Public Types inherited from Hypertable::CellList
typedef std::pair< const char *, int64_t >	SplitRowDataValue
typedef StlAllocator < SplitRowDataValue >	SplitRowDataAlloc
typedef std::map< const char *, int64_t, LtCstr, SplitRowDataAlloc >	SplitRowDataMapT
Static Public Attributes inherited from Hypertable::CellStore
static const char	DATA_BLOCK_MAGIC [10]
static const char	INDEX_FIXED_BLOCK_MAGIC [10]
static const char	INDEX_VARIABLE_BLOCK_MAGIC [10]

Detailed Description

Definition at line 54 of file CellStoreV2.h.

Member Typedef Documentation

typedef BlobHashSet Hypertable::CellStoreV2::BloomFilterItems

protected

Definition at line 133 of file CellStoreV2.h.

Constructor & Destructor Documentation

CellStoreV2::CellStoreV2

(

Filesystem *

filesys

)

Definition at line 58 of file CellStoreV2.cc.

CellStoreV2::~CellStoreV2 ( )

virtual

Definition at line 70 of file CellStoreV2.cc.

Member Function Documentation

void CellStoreV2::add ( const Key &  key, const ByteString  value  )

overridevirtual

Inserts a key/value pair into the cell list.

Parameters

key	key object
value	ByteString representing value

Implements Hypertable::CellStore.

Definition at line 308 of file CellStoreV2.cc.

uint16_t CellStoreV2::block_header_format ( )

overridevirtual

Implements Hypertable::CellStore.

Definition at line 808 of file CellStoreV2.cc.

int64_t Hypertable::CellStoreV2::block_index_memory_used ( )

inlineoverridevirtual

Returns the amount of memory consumed by the block index.

Returns

memory used by block index

Implements Hypertable::CellStore.

Definition at line 107 of file CellStoreV2.h.

int64_t Hypertable::CellStoreV2::bloom_filter_memory_used ( )

inlineoverridevirtual

Returns the amount of memory consumed by the bloom filter.

Returns

memory used by bloom filter

Implements Hypertable::CellStore.

Definition at line 106 of file CellStoreV2.h.

size_t Hypertable::CellStoreV2::bloom_filter_size ( )

inlineoverridevirtual

Return Bloom filter size.

Returns

size of bloom filter

Implements Hypertable::CellStore.

Definition at line 105 of file CellStoreV2.h.

float Hypertable::CellStoreV2::compression_ratio ( )

inlineoverridevirtual

Returns block compression ratio of this cell store.

Returns

compression ratio

Implements Hypertable::CellStore.

Definition at line 97 of file CellStoreV2.h.

void CellStoreV2::create ( const char *  fname, size_t  max_entries, PropertiesPtr &  props, const TableIdentifier *  table_id = 0  )

overridevirtual

Creates a new cell store.

Parameters

fname	name of file to contain the cell store
max_entries	maximum number of entries the cell store is expected to have
props	cellstore specific properties
table_id	Table identifier

Implements Hypertable::CellStore.

Definition at line 110 of file CellStoreV2.cc.

BlockCompressionCodec * CellStoreV2::create_block_compression_codec ( )

overridevirtual

Creates a block compression codec suitable for decompressing the cell store's blocks.

Returns

pointer to compression codec

Implements Hypertable::CellStore.

Definition at line 88 of file CellStoreV2.cc.

void CellStoreV2::create_bloom_filter ( bool  is_approx = false )

protected

Definition at line 193 of file CellStoreV2.cc.

CellListScannerPtr CellStoreV2::create_scanner ( ScanContext *  scan_ctx )

overridevirtual

Creates a scanner on this cell list.

Parameters

scan_ctx

smart pointer to scan context

Returns

pointer to newly allocated scanner

Reimplemented from Hypertable::CellStore.

Definition at line 94 of file CellStoreV2.cc.

uint64_t Hypertable::CellStoreV2::disk_usage ( )

inlineoverridevirtual

Returns the disk used by this cell store.

If the cell store is opened with a restricted range, then it returns an estimate of the disk used by that range.

Returns

disk used by this cell store or portion thereof

Implements Hypertable::CellStore.

Definition at line 91 of file CellStoreV2.h.

void CellStoreV2::display_block_info ( )

overridevirtual

Displays block information to stdout.

Implements Hypertable::CellStore.

Definition at line 799 of file CellStoreV2.cc.

int64_t Hypertable::CellStoreV2::end_of_last_block ( )

inlineoverridevirtual

Returns the offset of the end of the last block in the cell store.

Returns

offset of end of last block

Implements Hypertable::CellStore.

Definition at line 104 of file CellStoreV2.h.

void CellStoreV2::finalize ( TableIdentifier *  table_identifier )

overridevirtual

Finalizes the creation of a cell store, by writing block index and metadata trailer.

Parameters

table_identifier

table identifier

Chop the Index buffers down to the exact length

Write fixed index

Write variable index

Set up index

close file for writing

Set file length

Re-open file for reading

Implements Hypertable::CellStore.

Definition at line 399 of file CellStoreV2.cc.

int64_t Hypertable::CellStoreV2::get_blocksize ( )

inlineoverridevirtual

Returns the block size used for this cell store.

The block size is the amount of uncompressed key/value pairs to collect before compressing and storing as a compressed block in the cell store.

Returns

block size

Implements Hypertable::CellStore.

Definition at line 83 of file CellStoreV2.h.

int32_t Hypertable::CellStoreV2::get_fd ( )

inlineoverridevirtual

Returns the open file descriptor for the CellStore file.

Returns

open file descriptor

Implements Hypertable::CellStore.

Definition at line 111 of file CellStoreV2.h.

int Hypertable::CellStoreV2::get_file_id ( )

inlineoverridevirtual

Returns a unique identifier which identifies the underlying file for caching purposes.

Returns

unique file id

Implements Hypertable::CellStore.

Definition at line 100 of file CellStoreV2.h.

std::string& Hypertable::CellStoreV2::get_filename ( )

inlineoverridevirtual

Pathname of cell store file.

Returns

string reference to the cell store path name

Implements Hypertable::CellStore.

Definition at line 99 of file CellStoreV2.h.

int64_t Hypertable::CellStoreV2::get_total_entries ( )

inlineoverridevirtual

Returns the number of key/value pairs in the cell store.

Returns

Number of key/value pairs in list

Implements Hypertable::CellStore.

Definition at line 98 of file CellStoreV2.h.

CellStoreTrailer* Hypertable::CellStoreV2::get_trailer ( )

inlineoverridevirtual

Return a pointer to the trailer object for this cell store.

Returns

pointer to the internal trailer of this cell store

Implements Hypertable::CellStore.

Definition at line 124 of file CellStoreV2.h.

void CellStoreV2::load_block_index ( )

protected

Read index data

inflate fixed index

inflate variable index

Set up index

Definition at line 654 of file CellStoreV2.cc.

void CellStoreV2::load_bloom_filter ( )

protected

Definition at line 225 of file CellStoreV2.cc.

bool CellStoreV2::may_contain	(	const void *	ptr,
		size_t	len
	)

Definition at line 783 of file CellStoreV2.cc.

bool Hypertable::CellStoreV2::may_contain ( const String &  key )

inline

Definition at line 85 of file CellStoreV2.h.

bool CellStoreV2::may_contain ( ScanContext *  scan_ctx )

overridevirtual

Bloom filter lookup.

Parameters

scan_ctx

Scan context

Returns

true if cell store may contain row referenced by scan_ctx

Implements Hypertable::CellStore.

Definition at line 746 of file CellStoreV2.cc.

void CellStoreV2::open ( const String &  fname, const String &  start_row, const String &  end_row, int32_t  fd, int64_t  file_length, CellStoreTrailer *  trailer  )

overridevirtual

Opens a cell store with possibly a restricted view.

When a range splits, the cell stores that comprise the range get shared between the two newly created ranges. This method allows each of the two ranges to open the same cell store but view different portions of it.

Parameters

fname	pathname of file containing cell store
start_row	restricts view of this store to key/value pairs that are greater than this value
end_row	restricts view of this store to key/value pairs that are less than or equal to this value
fd	cell store file descriptor
file_length	length of cell store file
trailer	pointer to trailer object

Sanity check trailer

Implements Hypertable::CellStore.

Definition at line 623 of file CellStoreV2.cc.

uint64_t CellStoreV2::purge_indexes ( )

overridevirtual

Purges bloom filter and block indexes.

Returns

amount of memory purged

Implements Hypertable::CellStore.

Definition at line 282 of file CellStoreV2.cc.

int32_t Hypertable::CellStoreV2::reopen_fd ( )

inlineoverridevirtual

Closes and reopens the underlying CellStore file.

Returns

new file descriptor

Implements Hypertable::CellStore.

Definition at line 116 of file CellStoreV2.h.

bool Hypertable::CellStoreV2::restricted_range ( )

inlineoverridevirtual

Returns true if the cellstore was opened with a restricted range.

Returns

true if cellstore opened with restricted range

Implements Hypertable::CellStore.

Definition at line 109 of file CellStoreV2.h.

Member Data Documentation

bool Hypertable::CellStoreV2::m_64bit_index

protected

Definition at line 140 of file CellStoreV2.h.

float Hypertable::CellStoreV2::m_bloom_bits_per_item

protected

Definition at line 162 of file CellStoreV2.h.

BloomFilter* Hypertable::CellStoreV2::m_bloom_filter

protected

Definition at line 159 of file CellStoreV2.h.

BloomFilterItems* Hypertable::CellStoreV2::m_bloom_filter_items

protected

Definition at line 160 of file CellStoreV2.h.

BloomFilterMode Hypertable::CellStoreV2::m_bloom_filter_mode

protected

Definition at line 158 of file CellStoreV2.h.

DynamicBuffer Hypertable::CellStoreV2::m_buffer

protected

Definition at line 143 of file CellStoreV2.h.

float Hypertable::CellStoreV2::m_compressed_data

protected

Definition at line 153 of file CellStoreV2.h.

BlockCompressionCodec* Hypertable::CellStoreV2::m_compressor

protected

Definition at line 142 of file CellStoreV2.h.

BlockCompressionCodec::Args Hypertable::CellStoreV2::m_compressor_args

protected

Definition at line 155 of file CellStoreV2.h.

int64_t Hypertable::CellStoreV2::m_disk_usage

protected

Definition at line 150 of file CellStoreV2.h.

int32_t Hypertable::CellStoreV2::m_fd

protected

Definition at line 136 of file CellStoreV2.h.

int Hypertable::CellStoreV2::m_file_id

protected

Definition at line 151 of file CellStoreV2.h.

int64_t Hypertable::CellStoreV2::m_file_length

protected

Definition at line 149 of file CellStoreV2.h.

std::string Hypertable::CellStoreV2::m_filename

protected

Definition at line 137 of file CellStoreV2.h.

Filesystem* Hypertable::CellStoreV2::m_filesys

protected

Definition at line 135 of file CellStoreV2.h.

float Hypertable::CellStoreV2::m_filter_false_positive_prob

protected

Definition at line 163 of file CellStoreV2.h.

IndexBuilder Hypertable::CellStoreV2::m_index_builder

protected

Definition at line 144 of file CellStoreV2.h.

CellStoreBlockIndexArray<uint32_t> Hypertable::CellStoreV2::m_index_map32

protected

Definition at line 138 of file CellStoreV2.h.

CellStoreBlockIndexArray<int64_t> Hypertable::CellStoreV2::m_index_map64

protected

Definition at line 139 of file CellStoreV2.h.

ByteString Hypertable::CellStoreV2::m_last_key

protected

Definition at line 148 of file CellStoreV2.h.

int64_t Hypertable::CellStoreV2::m_max_approx_items

protected

Definition at line 161 of file CellStoreV2.h.

size_t Hypertable::CellStoreV2::m_max_entries

protected

Definition at line 156 of file CellStoreV2.h.

int64_t Hypertable::CellStoreV2::m_offset

protected

Definition at line 147 of file CellStoreV2.h.

uint32_t Hypertable::CellStoreV2::m_outstanding_appends

protected

Definition at line 146 of file CellStoreV2.h.

bool Hypertable::CellStoreV2::m_restricted_range

protected

Definition at line 164 of file CellStoreV2.h.

DispatchHandlerSynchronizer Hypertable::CellStoreV2::m_sync_handler

protected

Definition at line 145 of file CellStoreV2.h.

CellStoreTrailerV2 Hypertable::CellStoreV2::m_trailer

protected

Definition at line 141 of file CellStoreV2.h.

int64_t Hypertable::CellStoreV2::m_uncompressed_blocksize

protected

Definition at line 154 of file CellStoreV2.h.

float Hypertable::CellStoreV2::m_uncompressed_data

protected

Definition at line 152 of file CellStoreV2.h.

---

The documentation for this class was generated from the following files:

• /home/doug/src/hypertable/src/cc/Hypertable/RangeServer/CellStoreV2.h
• /home/doug/src/hypertable/src/cc/Hypertable/RangeServer/CellStoreV2.cc