LoadDataSourceFileLocal.cc

Go to the documentation of this file.

#include "Common/Compat.h"
#include <fstream>
#include <iostream>
#include <cerrno>
#include <cctype>
#include <cstdlib>
#include <cstring>

#include <boost/algorithm/string.hpp>
#include <boost/algorithm/string/predicate.hpp>
#include <boost/iostreams/filter/gzip.hpp>
#include <boost/shared_array.hpp>

extern "C" {
#include <strings.h>
#include <sys/types.h>
#include <unistd.h>
}

#include "Common/DynamicBuffer.h"
#include "Common/Error.h"
#include "Common/Logger.h"
#include "Common/FileUtils.h"

#include "Key.h"
#include "LoadDataSourceFileLocal.h"

using namespace boost::iostreams;
using namespace Hypertable;
using namespace std;

LoadDataSourceFileLocal::LoadDataSourceFileLocal(const string &fname,
  const string &header_fname, int row_uniquify_chars, int load_flags)
  : LoadDataSource(header_fname, row_uniquify_chars, load_flags),
    m_source(fname), m_fname(fname) {

  if (!FileUtils::exists(fname.c_str()))
    HT_THROW(Error::FILE_NOT_FOUND, fname);

  if (boost::algorithm::ends_with(fname, ".gz")) {
    m_fin.push(gzip_decompressor());
    m_zipped = true;
  }

  return;
}

void
LoadDataSourceFileLocal::init_src()
{
  m_fin.push(m_source);
  m_source_size = FileUtils::size(m_fname.c_str());
}

uint64_t
LoadDataSourceFileLocal::incr_consumed()
{
  uint64_t consumed=0;
  uint64_t new_offset = m_source.seek(0, BOOST_IOS::cur);
  consumed = new_offset - m_offset;
  m_offset = new_offset;
  return consumed;
}