0.9.8.10
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Modules Pages
CellStoreV0.h
Go to the documentation of this file.
1 /* -*- c++ -*-
2  * Copyright (C) 2007-2015 Hypertable, Inc.
3  *
4  * This file is part of Hypertable.
5  *
6  * Hypertable is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU General Public License
8  * as published by the Free Software Foundation; version 3 of the
9  * License, or any later version.
10  *
11  * Hypertable is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19  * 02110-1301, USA.
20  */
21 
22 #ifndef Hypertable_RangeServer_CellStoreV0_h
23 #define Hypertable_RangeServer_CellStoreV0_h
24 
25 #include <map>
26 #include <string>
27 #include <vector>
28 
30 
32 #include "Common/DynamicBuffer.h"
33 #include "Common/BloomFilter.h"
34 #include "Common/BlobHashSet.h"
35 #include "Common/Filesystem.h"
36 
39 
40 #include "CellStore.h"
41 #include "CellStoreTrailerV0.h"
42 
43 
47 namespace Hypertable {
49  class Client;
50  class Protocol;
51 }
52 
53 namespace Hypertable {
54 
55  class CellStoreV0 : public CellStore {
56 
57  public:
58  CellStoreV0(Filesystem *filesys);
59  virtual ~CellStoreV0();
60 
61  void create(const char *fname, size_t max_entries, PropertiesPtr &props,
62  const TableIdentifier *table_id=0) override;
63  void add(const Key &key, const ByteString value) override;
64  void finalize(TableIdentifier *table_identifier) override;
65  void open(const String &fname, const String &start_row,
66  const String &end_row, int32_t fd, int64_t file_length,
67  CellStoreTrailer *trailer) override;
68  int64_t get_blocksize() override { return m_trailer.blocksize; }
69  bool may_contain(const void *ptr, size_t len);
70  bool may_contain(const String &key) {
71  return may_contain(key.data(), key.size());
72  }
73 
74  bool may_contain(ScanContext *scan_ctx) override;
75 
76  uint64_t disk_usage() override {
77  if (m_disk_usage < 0)
78  HT_WARN_OUT << "[Issue 339] Disk usage for " << m_filename << "=" << m_disk_usage
79  << HT_END;
80  return m_disk_usage;
81  }
82  float compression_ratio() override { return m_trailer.compression_ratio; }
83  int64_t get_total_entries() override { return m_trailer.total_entries; }
84  std::string &get_filename() override { return m_filename; }
85  int get_file_id() override { return m_file_id; }
86  CellListScannerPtr create_scanner(ScanContext *scan_ctx) override;
88  void display_block_info() override;
89  int64_t end_of_last_block() override { return m_trailer.fix_index_offset; }
90  size_t bloom_filter_size() override { return m_bloom_filter ? m_bloom_filter->size() : 0; }
91  int64_t bloom_filter_memory_used() override { return 0; }
92  int64_t block_index_memory_used() override { return 0; }
93  uint64_t purge_indexes() override { return 0; }
94  bool restricted_range() override { return true; }
95 
96  int32_t get_fd() override {
97  std::lock_guard<std::mutex> lock(m_mutex);
98  return m_fd;
99  }
100 
101  int32_t reopen_fd() override {
102  std::lock_guard<std::mutex> lock(m_mutex);
103  if (m_fd != -1)
104  m_filesys->close(m_fd);
105  m_fd = m_filesys->open(m_filename, 0);
106  return m_fd;
107  }
108 
109  CellStoreTrailer *get_trailer() override { return &m_trailer; }
110 
111  uint16_t block_header_format() override;
112 
113  protected:
114  void add_index_entry(const SerializedKey key, uint32_t offset);
115  void create_bloom_filter(bool is_approx = false);
116  void load_index();
117 
118  typedef std::map<SerializedKey, uint32_t> IndexMap;
120 
122  int32_t m_fd;
123  std::string m_filename;
124  IndexMap m_index;
134  uint32_t m_offset;
136  int64_t m_file_length;
137  int64_t m_disk_usage;
144 
147  BloomFilterItems *m_bloom_filter_items;
149  };
150 
151 }
152 
153 #endif // Hypertable_RangeServer_CellStoreV0_h
Abstract base class for cell store trailer.
virtual void close(int fd, DispatchHandler *handler)=0
Closes a file asynchronously.
bool restricted_range() override
Returns true if the cellstore was opened with a restricted range.
Definition: CellStoreV0.h:94
Abstract base class for persistent cell lists (ones that are stored on disk).
Definition: CellStore.h:57
uint32_t m_outstanding_appends
Definition: CellStoreV0.h:133
Abstract base class for a filesystem.
std::string String
A String is simply a typedef to std::string.
Definition: String.h:44
CellStoreTrailer * get_trailer() override
Return a pointer to the trailer object for this cell store.
Definition: CellStoreV0.h:109
float compression_ratio() override
Returns block compression ratio of this cell store.
Definition: CellStoreV0.h:82
A Bloom Filter implementation.
BlockCompressionCodec * create_block_compression_codec() override
Creates a block compression codec suitable for decompressing the cell store's blocks.
Definition: CellStoreV0.cc:90
size_t bloom_filter_size() override
Return Bloom filter size.
Definition: CellStoreV0.h:90
CellStoreV0(Filesystem *filesys)
Definition: CellStoreV0.cc:56
int get_file_id() override
Returns a unique identifier which identifies the underlying file for caching purposes.
Definition: CellStoreV0.h:85
void add_index_entry(const SerializedKey key, uint32_t offset)
Definition: CellStoreV0.cc:429
uint32_t m_uncompressed_blocksize
Definition: CellStoreV0.h:141
CellStoreBlockIndexArray< uint32_t > m_index_map32
Definition: CellStoreV0.h:125
Scan context information.
Definition: ScanContext.h:52
Declarations for BlockCompressionCodec.
DispatchHandlerSynchronizer m_sync_handler
Definition: CellStoreV0.h:132
void create(const char *fname, size_t max_entries, PropertiesPtr &props, const TableIdentifier *table_id=0) override
Creates a new cell store.
Definition: CellStoreV0.cc:102
A dynamic, resizable and reference counted memory buffer.
Definition: DynamicBuffer.h:42
std::vector< String > Args
Compression codec argument vector.
int64_t get_blocksize() override
Returns the block size used for this cell store.
Definition: CellStoreV0.h:68
A class managing one or more serializable ByteStrings.
Definition: ByteString.h:47
bool may_contain(const void *ptr, size_t len)
Definition: CellStoreV0.cc:630
Declarations for CellStore.
A dynamic, resizable memory buffer.
uint64_t disk_usage() override
Returns the disk used by this cell store.
Definition: CellStoreV0.h:76
BloomFilterItems * m_bloom_filter_items
Definition: CellStoreV0.h:147
CellStoreTrailerV0 m_trailer
Definition: CellStoreV0.h:126
size_t size()
Getter for the bloom filter size.
Definition: BloomFilter.h:232
Filesystem * m_filesys
Definition: CellStoreV0.h:121
std::shared_ptr< Properties > PropertiesPtr
Definition: Properties.h:447
BloomFilterMode
Enumeration for bloom filter modes.
std::mutex m_mutex
Definition: CellStore.h:331
void display_block_info() override
Displays block information to stdout.
Definition: CellStoreV0.cc:637
#define HT_END
Definition: Logger.h:220
BlockCompressionCodec::Args m_compressor_args
Definition: CellStoreV0.h:142
A HashSet optimized for blobs.
#define HT_WARN_OUT
Definition: Logger.h:291
uint64_t purge_indexes() override
Purges bloom filter and block indexes.
Definition: CellStoreV0.h:93
Hypertable definitions
DispatchHandler class used to synchronize with response messages.
int64_t bloom_filter_memory_used() override
Returns the amount of memory consumed by the bloom filter.
Definition: CellStoreV0.h:91
std::map< SerializedKey, uint32_t > IndexMap
Definition: CellStoreV0.h:118
BloomFilterMode m_bloom_filter_mode
Definition: CellStoreV0.h:145
void finalize(TableIdentifier *table_identifier) override
Finalizes the creation of a cell store, by writing block index and metadata trailer.
Definition: CellStoreV0.cc:271
A HashSet for storing and looking up blobs efficiently.
Definition: BlobHashSet.h:44
BlockCompressionCodec * m_compressor
Definition: CellStoreV0.h:127
uint16_t block_header_format() override
Definition: CellStoreV0.cc:661
std::string & get_filename() override
Pathname of cell store file.
Definition: CellStoreV0.h:84
Abstract base class for server protocol drivers.
Definition: Protocol.h:49
Provides access to internal components of opaque key.
Definition: Key.h:40
void open(const String &fname, const String &start_row, const String &end_row, int32_t fd, int64_t file_length, CellStoreTrailer *trailer) override
Opens a cell store with possibly a restricted view.
Definition: CellStoreV0.cc:446
CellListScannerPtr create_scanner(ScanContext *scan_ctx) override
Creates a scanner on this cell list.
Definition: CellStoreV0.cc:96
void add(const Key &key, const ByteString value) override
Inserts a key/value pair into the cell list.
Definition: CellStoreV0.cc:192
int32_t get_fd() override
Returns the open file descriptor for the CellStore file.
Definition: CellStoreV0.h:96
int64_t block_index_memory_used() override
Returns the amount of memory consumed by the block index.
Definition: CellStoreV0.h:92
void create_bloom_filter(bool is_approx=false)
Definition: CellStoreV0.cc:165
bool may_contain(const String &key)
Definition: CellStoreV0.h:70
BloomFilter * m_bloom_filter
Definition: CellStoreV0.h:146
DynamicBuffer m_fix_index_buffer
Definition: CellStoreV0.h:129
std::shared_ptr< CellListScanner > CellListScannerPtr
Definition: CellList.h:35
DynamicBuffer m_var_index_buffer
Definition: CellStoreV0.h:130
virtual void open(const String &name, uint32_t flags, DispatchHandler *handler)=0
Opens a file asynchronously.
DynamicBuffer m_buffer
Definition: CellStoreV0.h:128
int64_t get_total_entries() override
Returns the number of key/value pairs in the cell store.
Definition: CellStoreV0.h:83
A space-efficent probabilistic set for membership test, false postives are possible, but false negatives are not.
Definition: BloomFilter.h:50
int32_t reopen_fd() override
Closes and reopens the underlying CellStore file.
Definition: CellStoreV0.h:101
int64_t end_of_last_block() override
Returns the offset of the end of the last block in the cell store.
Definition: CellStoreV0.h:89
BlobHashSet BloomFilterItems
Definition: CellStoreV0.h:119
Declarations for DispatchHandlerSynchronizer.
Abstract base class for block compression codecs.
Abstract base class for a filesystem.
Definition: Filesystem.h:72
Declarations for CellStoreBlockIndexArray.