Commit log for persisting range updates. More...

#include <CommitLog.h>

Inheritance diagram for Hypertable::CommitLog:

Collaboration diagram for Hypertable::CommitLog:

Classes
struct	CumulativeFragmentData

Public Types
typedef std::map< int64_t, CumulativeFragmentData >	CumulativeSizeMap

Public Member Functions
	CommitLog (FilesystemPtr &fs, const std::string &log_dir, PropertiesPtr &props, CommitLogBase *init_log=0, bool is_meta=true)
	Constructs a CommitLog object using supplied properties. More...

	CommitLog (FilesystemPtr &fs, const std::string &log_dir, bool is_meta=true)
	Constructs a CommitLog object using default properties. More...

virtual	~CommitLog ()

int64_t	get_timestamp ()
	Atomically obtains a timestamp. More...

int	write (uint64_t cluster_id, DynamicBuffer &buffer, int64_t revision, Filesystem::Flags flags)
	Writes a block of updates to the commit log. More...

int	flush ()
	Flushes previous updates written to commit log. More...

int	sync ()
	Sync previous updates written to commit log. More...

int	link_log (uint64_t cluster_id, CommitLogBase *log_base)
	Links an external log into this log. More...

int	close ()
	Closes the log. More...

int	purge (int64_t revision, StringSet &remove_ok_logs, StringSet &removed_logs, std::string *trace)
	Purges the log. More...

void	load_cumulative_size_map (CumulativeSizeMap &cumulative_size_map)
	Fills up a map of cumulative fragment size data. More...

int64_t	get_max_fragment_size ()
	Returns the maximum size of each log fragment file. More...

void	get_stats (const std::string &prefix, std::string &result)
	Returns the stats on all commit log fragments. More...

int64_t	size ()
	Returns total size of commit log. More...

const std::string &	get_current_fragment_file ()

Public Member Functions inherited from Hypertable::CommitLogBase
	CommitLogBase (const std::string &log_dir)

void	stitch_in (CommitLogBase *other)
	This method assumes that the other commit log is not being concurrently used which is why it doesn't lock it's mutex. More...

std::string &	get_log_dir ()

int64_t	get_latest_revision ()

bool	empty ()

bool	range_reference_required ()

LogFragmentQueue &	fragment_queue ()

uint32_t	toplevel_fragment_id (CommitLogFileInfo *finfo)

Static Public Attributes
static const char	MAGIC_DATA [10]

static const char	MAGIC_LINK [10]

Private Member Functions
void	initialize (const std::string &log_dir, PropertiesPtr &, CommitLogBase *init_log, bool is_meta)

int	roll (CommitLogFileInfo **clfip=0)

int	compress_and_write (DynamicBuffer &input, BlockHeader *header, int64_t revision, Filesystem::Flags flags)

void	remove_file_info (CommitLogFileInfo *fi, StringSet &removed_logs)

Private Attributes
FilesystemPtr	m_fs

std::set< CommitLogFileInfo * >	m_reap_set

std::unique_ptr < BlockCompressionCodec >	m_compressor

std::string	m_cur_fragment_fname

int64_t	m_cur_fragment_length

int64_t	m_max_fragment_size

uint32_t	m_cur_fragment_num

int32_t	m_fd

int32_t	m_replication

bool	m_needs_roll

Additional Inherited Members
Protected Attributes inherited from Hypertable::CommitLogBase
std::mutex	m_mutex

std::string	m_log_dir

std::string	m_log_name

LogFragmentQueue	m_fragment_queue

int64_t	m_latest_revision

std::set< int64_t >	m_linked_log_hashes

bool	m_range_reference_required

Detailed Description

Commit log for persisting range updates.

The commit log is a directory that contains a growing number of files that contain compressed blocks of "commits". The files are named starting with '0' and will periodically roll, which means that a trailer is written to the end of the file, the file is closed, and then the numeric name is incremented by one and opened. Periodically when old parts of the log are no longer needed, they get purged. The size of each log fragment file is determined by the following config file property:

  Hypertable.RangeServer.CommitLog.RollLimit
*

Definition at line 63 of file CommitLog.h.

Member Typedef Documentation

typedef std::map<int64_t, CumulativeFragmentData> Hypertable::CommitLog::CumulativeSizeMap

Definition at line 74 of file CommitLog.h.

Constructor & Destructor Documentation

Hypertable::CommitLog::CommitLog	(	FilesystemPtr &	fs,
		const std::string &	log_dir,
		PropertiesPtr &	props,
		CommitLogBase *	init_log = `0`,
		bool	is_meta = `true`
	)

inline

Constructs a CommitLog object using supplied properties.

Parameters

fs	filesystem to write log into
log_dir	directory of the commit log
props	reference to properties map
init_log	base log to pull fragments from
is_meta	true for root, system and metadata logs

Definition at line 85 of file CommitLog.h.

CommitLog::CommitLog	(	FilesystemPtr &	fs,
		const std::string &	log_dir,
		bool	is_meta = `true`
	)

Constructs a CommitLog object using default properties.

Parameters

fs	filesystem to write log into
log_dir	directory of the commit log
is_meta	true for root, system and metadata logs

Definition at line 60 of file CommitLog.cc.

CommitLog::~CommitLog ( )

virtual

Definition at line 65 of file CommitLog.cc.

Member Function Documentation

int CommitLog::close ( )

Closes the log.

Writes the trailer and closes the file

Returns: Error::OK on success or error code on failure

Definition at line 290 of file CommitLog.cc.

int CommitLog::compress_and_write	(	DynamicBuffer &	input,
		BlockHeader *	header,
		int64_t	revision,
		Filesystem::Flags	flags
	)

private

Definition at line 477 of file CommitLog.cc.

int CommitLog::flush ( )

Flushes previous updates written to commit log.

Returns: Error::OK on success or error code on failure

Definition at line 143 of file CommitLog.cc.

const std::string& Hypertable::CommitLog::get_current_fragment_file ( )

inline

Definition at line 194 of file CommitLog.h.

int64_t Hypertable::CommitLog::get_max_fragment_size ( )

inline

Returns the maximum size of each log fragment file.

Definition at line 171 of file CommitLog.h.

void CommitLog::get_stats	(	const std::string &	prefix,
		std::string &	result
	)

Returns the stats on all commit log fragments.

Parameters

prefix	stat line prefix string
result	reference to return stats string

Definition at line 544 of file CommitLog.cc.

int64_t CommitLog::get_timestamp ( )

Atomically obtains a timestamp.

Returns: nanoseconds since the epoch

Definition at line 139 of file CommitLog.cc.

void CommitLog::initialize	(	const std::string &	log_dir,
		PropertiesPtr &	props,
		CommitLogBase *	init_log,
		bool	is_meta
	)

private

Definition at line 70 of file CommitLog.cc.

int CommitLog::link_log	(	uint64_t	cluster_id,
		CommitLogBase *	log_base
	)

Links an external log into this log.

Parameters

cluster_id	Originating cluster ID
log_base	pointer to commit log object to link in

Returns: Error::OK on success or error code on failure

Definition at line 211 of file CommitLog.cc.

void CommitLog::load_cumulative_size_map ( CumulativeSizeMap & cumulative_size_map )

Fills up a map of cumulative fragment size data.

One entry per log fragment is inserted into this map. The key is the revision of the fragment (e.g. the real revision of the most recent data in the fragment file). The value is a structure that contains information regarding how expensive it is to keep this fragment around.

Parameters

cumulative_size_map reference to map of log fragment priority data

Definition at line 510 of file CommitLog.cc.

int CommitLog::purge	(	int64_t	revision,
		StringSet &	remove_ok_logs,
		StringSet &	removed_logs,
		std::string *	trace
	)

Purges the log.

Removes all of the log fragments that have a revision that is less than the given revision.

Parameters

revision	real cutoff revision
remove_ok_logs	Set of log pathnames that can be safely removed
removed_logs	Set of logs that were removed by this call
trace	Address of trace string to add trace info to if non-NULL

Definition at line 310 of file CommitLog.cc.

void CommitLog::remove_file_info	(	CommitLogFileInfo *	fi,
		StringSet &	removed_logs
	)

private

Definition at line 366 of file CommitLog.cc.

int CommitLog::roll ( CommitLogFileInfo ** clfip = 0 )

private

Definition at line 406 of file CommitLog.cc.

int64_t Hypertable::CommitLog::size ( )

inline

Returns total size of commit log.

Definition at line 185 of file CommitLog.h.

int CommitLog::sync ( )

Sync previous updates written to commit log.

Returns: Error::OK on success or error code on failure

Definition at line 161 of file CommitLog.cc.

int CommitLog::write	(	uint64_t	cluster_id,
		DynamicBuffer &	buffer,
		int64_t	revision,
		Filesystem::Flags	flags
	)

Writes a block of updates to the commit log.

Parameters

cluster_id	Originating cluster ID
buffer	block of updates to commit
revision	most recent revision in buffer
flags	Flags to pass to underlying append operation

Returns: Error::OK on success or error code on failure

Compress and write the commit block

Roll the log

Definition at line 181 of file CommitLog.cc.

Member Data Documentation

std::unique_ptr<BlockCompressionCodec> Hypertable::CommitLog::m_compressor

private

Definition at line 212 of file CommitLog.h.

std::string Hypertable::CommitLog::m_cur_fragment_fname

private

Definition at line 213 of file CommitLog.h.

int64_t Hypertable::CommitLog::m_cur_fragment_length

private

Definition at line 214 of file CommitLog.h.

uint32_t Hypertable::CommitLog::m_cur_fragment_num

private

Definition at line 216 of file CommitLog.h.

int32_t Hypertable::CommitLog::m_fd

private

Definition at line 217 of file CommitLog.h.

FilesystemPtr Hypertable::CommitLog::m_fs

private

Definition at line 210 of file CommitLog.h.

int64_t Hypertable::CommitLog::m_max_fragment_size

private

Definition at line 215 of file CommitLog.h.

bool Hypertable::CommitLog::m_needs_roll

private

Definition at line 219 of file CommitLog.h.

std::set<CommitLogFileInfo *> Hypertable::CommitLog::m_reap_set

private

Definition at line 211 of file CommitLog.h.

int32_t Hypertable::CommitLog::m_replication

private

Definition at line 218 of file CommitLog.h.

const char CommitLog::MAGIC_DATA

static

Initial value:

=

{ 'C','O','M','M','I','T','D','A','T','A' }

Definition at line 199 of file CommitLog.h.

const char CommitLog::MAGIC_LINK

static

Initial value:

=

{ 'C','O','M','M','I','T','L','I','N','K' }

Definition at line 200 of file CommitLog.h.

The documentation for this class was generated from the following files:

/home/doug/src/hypertable/src/cc/Hypertable/Lib/CommitLog.h
/home/doug/src/hypertable/src/cc/Hypertable/Lib/CommitLog.cc

Classes

Public Types

Public Member Functions

Static Public Attributes

Private Member Functions

Private Attributes

Additional Inherited Members

Detailed Description

Member Typedef Documentation

Constructor & Destructor Documentation

Member Function Documentation

Member Data Documentation