leveldb::anonymous_namespace{repair.cc}::Repairer Class Reference

Collaboration diagram for leveldb::anonymous_namespace{repair.cc}::Repairer:

Classes
struct	TableInfo

Public Member Functions
	Repairer (const std::string &dbname, const Options &options)
	~Repairer ()
Status	Run ()

Private Member Functions
Status	FindFiles ()
void	ConvertLogFilesToTables ()
Status	ConvertLogToTable (uint64_t log)
void	ExtractMetaData ()
Iterator *	NewTableIterator (const FileMetaData &meta)
void	ScanTable (uint64_t number)
void	RepairTable (const std::string &src, TableInfo t)
Status	WriteDescriptor ()
void	ArchiveFile (const std::string &fname)

Private Attributes
std::string const	dbname_
Env *const	env_
InternalKeyComparator const	icmp_
InternalFilterPolicy const	ipolicy_
Options const	options_
bool	owns_info_log_
bool	owns_cache_
TableCache *	table_cache_
VersionEdit	edit_
std::vector< std::string >	manifests_
std::vector< uint64_t >	table_numbers_
std::vector< uint64_t >	logs_
std::vector< TableInfo >	tables_
uint64_t	next_file_number_

Detailed Description

Definition at line 45 of file repair.cc.

---

Class Documentation

leveldb::anonymous_namespace{repair.cc}::Repairer::TableInfo

struct leveldb::anonymous_namespace{repair.cc}::Repairer::TableInfo

Definition at line 95 of file repair.cc.

Collaboration diagram for leveldb::anonymous_namespace{repair.cc}::Repairer::TableInfo:

Class Members
SequenceNumber	max_sequence
FileMetaData	meta

Constructor & Destructor Documentation

Repairer()

leveldb::anonymous_namespace{repair.cc}::Repairer::Repairer ( const std::string &  dbname, const Options &  options  )

inline

Definition at line 47 of file repair.cc.

      : dbname_(dbname),
        env_(options.env),
        icmp_(options.comparator),
        ipolicy_(options.filter_policy),
        options_(SanitizeOptions(dbname, &icmp_, &ipolicy_, options)),
        owns_info_log_(options_.info_log != options.info_log),
        owns_cache_(options_.block_cache != options.block_cache),
        next_file_number_(1) {
    // TableCache can be small since we expect each table to be opened once.
    table_cache_ = new TableCache(dbname_, &options_, 10);
  }

~Repairer()

leveldb::anonymous_namespace{repair.cc}::Repairer::~Repairer ( )

inline

Definition at line 60 of file repair.cc.

              {
    delete table_cache_;
    if (owns_info_log_) {
      delete options_.info_log;
    }
    if (owns_cache_) {
      delete options_.block_cache;
    }
  }

Member Function Documentation

ArchiveFile()

void leveldb::anonymous_namespace{repair.cc}::Repairer::ArchiveFile ( const std::string &  fname )

inlineprivate

Definition at line 434 of file repair.cc.

                                           {
    // Move into another directory.  E.g., for
    //    dir/foo
    // rename to
    //    dir/lost/foo
    const char* slash = strrchr(fname.c_str(), '/');
    std::string new_dir;
    if (slash != NULL) {
      new_dir.assign(fname.data(), slash - fname.data());
    }
    new_dir.append("/lost");
    env_->CreateDir(new_dir);  // Ignore error
    std::string new_file = new_dir;
    new_file.append("/");
    new_file.append((slash == NULL) ? fname.c_str() : slash + 1);
    Status s = env_->RenameFile(fname, new_file);
    Log(options_.info_log, "Archiving %s: %s\n",
        fname.c_str(), s.ToString().c_str());
  }

Here is the call graph for this function:

ConvertLogFilesToTables()

void leveldb::anonymous_namespace{repair.cc}::Repairer::ConvertLogFilesToTables ( )

inlineprivate

Definition at line 149 of file repair.cc.

                                 {
    for (size_t i = 0; i < logs_.size(); i++) {
      std::string logname = LogFileName(dbname_, logs_[i]);
      Status status = ConvertLogToTable(logs_[i]);
      if (!status.ok()) {
        Log(options_.info_log, "Log #%llu: ignoring conversion error: %s",
            (unsigned long long) logs_[i],
            status.ToString().c_str());
      }
      ArchiveFile(logname);
    }
  }

Here is the call graph for this function:

ConvertLogToTable()

Status leveldb::anonymous_namespace{repair.cc}::Repairer::ConvertLogToTable ( uint64_t  log )

inlineprivate

Definition at line 162 of file repair.cc.

                                         {
    struct LogReporter : public log::Reader::Reporter {
      Env* env;
      Logger* info_log;
      uint64_t lognum;
      virtual void Corruption(size_t bytes, const Status& s) {
        // We print error messages for corruption, but continue repairing.
        Log(info_log, "Log #%llu: dropping %d bytes; %s",
            (unsigned long long) lognum,
            static_cast<int>(bytes),
            s.ToString().c_str());
      }
    };

    // Open the log file
    std::string logname = LogFileName(dbname_, log);
    SequentialFile* lfile;
    Status status = env_->NewSequentialFile(logname, &lfile);
    if (!status.ok()) {
      return status;
    }

    // Create the log reader.
    LogReporter reporter;
    reporter.env = env_;
    reporter.info_log = options_.info_log;
    reporter.lognum = log;
    // We intentionally make log::Reader do checksumming so that
    // corruptions cause entire commits to be skipped instead of
    // propagating bad information (like overly large sequence
    // numbers).
    log::Reader reader(lfile, &reporter, false/*do not checksum*/,
                       0/*initial_offset*/);

    // Read all the records and add to a memtable
    std::string scratch;
    Slice record;
    WriteBatch batch;
    MemTable* mem = new MemTable(icmp_);
    mem->Ref();
    int counter = 0;
    while (reader.ReadRecord(&record, &scratch)) {
      if (record.size() < 12) {
        reporter.Corruption(
            record.size(), Status::Corruption("log record too small"));
        continue;
      }
      WriteBatchInternal::SetContents(&batch, record);
      status = WriteBatchInternal::InsertInto(&batch, mem);
      if (status.ok()) {
        counter += WriteBatchInternal::Count(&batch);
      } else {
        Log(options_.info_log, "Log #%llu: ignoring %s",
            (unsigned long long) log,
            status.ToString().c_str());
        status = Status::OK();  // Keep going with rest of file
      }
    }
    delete lfile;

    // Do not record a version edit for this conversion to a Table
    // since ExtractMetaData() will also generate edits.
    FileMetaData meta;
    meta.number = next_file_number_++;
    Iterator* iter = mem->NewIterator();
    status = BuildTable(dbname_, env_, options_, table_cache_, iter, &meta);
    delete iter;
    mem->Unref();
    mem = NULL;
    if (status.ok()) {
      if (meta.file_size > 0) {
        table_numbers_.push_back(meta.number);
      }
    }
    Log(options_.info_log, "Log #%llu: %d ops saved to Table #%llu %s",
        (unsigned long long) log,
        counter,
        (unsigned long long) meta.number,
        status.ToString().c_str());
    return status;
  }

Here is the call graph for this function:

ExtractMetaData()

void leveldb::anonymous_namespace{repair.cc}::Repairer::ExtractMetaData ( )

inlineprivate

Definition at line 244 of file repair.cc.

                         {
    for (size_t i = 0; i < table_numbers_.size(); i++) {
      ScanTable(table_numbers_[i]);
    }
  }

FindFiles()

Status leveldb::anonymous_namespace{repair.cc}::Repairer::FindFiles ( )

inlineprivate

Definition at line 116 of file repair.cc.

                     {
    std::vector<std::string> filenames;
    Status status = env_->GetChildren(dbname_, &filenames);
    if (!status.ok()) {
      return status;
    }
    if (filenames.empty()) {
      return Status::IOError(dbname_, "repair found no files");
    }

    uint64_t number;
    FileType type;
    for (size_t i = 0; i < filenames.size(); i++) {
      if (ParseFileName(filenames[i], &number, &type)) {
        if (type == kDescriptorFile) {
          manifests_.push_back(filenames[i]);
        } else {
          if (number + 1 > next_file_number_) {
            next_file_number_ = number + 1;
          }
          if (type == kLogFile) {
            logs_.push_back(number);
          } else if (type == kTableFile) {
            table_numbers_.push_back(number);
          } else {
            // Ignore other files
          }
        }
      }
    }
    return status;
  }

Here is the call graph for this function:

NewTableIterator()

Iterator* leveldb::anonymous_namespace{repair.cc}::Repairer::NewTableIterator ( const FileMetaData &  meta )

inlineprivate

Definition at line 250 of file repair.cc.

                                                       {
    // Same as compaction iterators: if paranoid_checks are on, turn
    // on checksum verification.
    ReadOptions r;
    r.verify_checksums = options_.paranoid_checks;
    return table_cache_->NewIterator(r, meta.number, meta.file_size);
  }

Here is the call graph for this function:

RepairTable()

void leveldb::anonymous_namespace{repair.cc}::Repairer::RepairTable ( const std::string &  src, TableInfo  t  )

inlineprivate

Definition at line 321 of file repair.cc.

                                                      {
    // We will copy src contents to a new table and then rename the
    // new table over the source.

    // Create builder.
    std::string copy = TableFileName(dbname_, next_file_number_++);
    WritableFile* file;
    Status s = env_->NewWritableFile(copy, &file);
    if (!s.ok()) {
      return;
    }
    TableBuilder* builder = new TableBuilder(options_, file);

    // Copy data.
    Iterator* iter = NewTableIterator(t.meta);
    int counter = 0;
    for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
      builder->Add(iter->key(), iter->value());
      counter++;
    }
    delete iter;

    ArchiveFile(src);
    if (counter == 0) {
      builder->Abandon();  // Nothing to save
    } else {
      s = builder->Finish();
      if (s.ok()) {
        t.meta.file_size = builder->FileSize();
      }
    }
    delete builder;
    builder = NULL;

    if (s.ok()) {
      s = file->Close();
    }
    delete file;
    file = NULL;

    if (counter > 0 && s.ok()) {
      std::string orig = TableFileName(dbname_, t.meta.number);
      s = env_->RenameFile(copy, orig);
      if (s.ok()) {
        Log(options_.info_log, "Table #%llu: %d entries repaired",
            (unsigned long long) t.meta.number, counter);
        tables_.push_back(t);
      }
    }
    if (!s.ok()) {
      env_->DeleteFile(copy);
    }
  }

Here is the call graph for this function:

Run()

Status leveldb::anonymous_namespace{repair.cc}::Repairer::Run ( )

inline

Definition at line 70 of file repair.cc.

               {
    Status status = FindFiles();
    if (status.ok()) {
      ConvertLogFilesToTables();
      ExtractMetaData();
      status = WriteDescriptor();
    }
    if (status.ok()) {
      unsigned long long bytes = 0;
      for (size_t i = 0; i < tables_.size(); i++) {
        bytes += tables_[i].meta.file_size;
      }
      Log(options_.info_log,
          "**** Repaired leveldb %s; "
          "recovered %d files; %llu bytes. "
          "Some data may have been lost. "
          "****",
          dbname_.c_str(),
          static_cast<int>(tables_.size()),
          bytes);
    }
    return status;
  }

Here is the call graph for this function:

ScanTable()

void leveldb::anonymous_namespace{repair.cc}::Repairer::ScanTable ( uint64_t  number )

inlineprivate

Definition at line 258 of file repair.cc.

                                  {
    TableInfo t;
    t.meta.number = number;
    std::string fname = TableFileName(dbname_, number);
    Status status = env_->GetFileSize(fname, &t.meta.file_size);
    if (!status.ok()) {
      // Try alternate file name.
      fname = SSTTableFileName(dbname_, number);
      Status s2 = env_->GetFileSize(fname, &t.meta.file_size);
      if (s2.ok()) {
        status = Status::OK();
      }
    }
    if (!status.ok()) {
      ArchiveFile(TableFileName(dbname_, number));
      ArchiveFile(SSTTableFileName(dbname_, number));
      Log(options_.info_log, "Table #%llu: dropped: %s",
          (unsigned long long) t.meta.number,
          status.ToString().c_str());
      return;
    }

    // Extract metadata by scanning through table.
    int counter = 0;
    Iterator* iter = NewTableIterator(t.meta);
    bool empty = true;
    ParsedInternalKey parsed;
    t.max_sequence = 0;
    for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
      Slice key = iter->key();
      if (!ParseInternalKey(key, &parsed)) {
        Log(options_.info_log, "Table #%llu: unparsable key %s",
            (unsigned long long) t.meta.number,
            EscapeString(key).c_str());
        continue;
      }

      counter++;
      if (empty) {
        empty = false;
        t.meta.smallest.DecodeFrom(key);
      }
      t.meta.largest.DecodeFrom(key);
      if (parsed.sequence > t.max_sequence) {
        t.max_sequence = parsed.sequence;
      }
    }
    if (!iter->status().ok()) {
      status = iter->status();
    }
    delete iter;
    Log(options_.info_log, "Table #%llu: %d entries %s",
        (unsigned long long) t.meta.number,
        counter,
        status.ToString().c_str());

    if (status.ok()) {
      tables_.push_back(t);
    } else {
      RepairTable(fname, t);  // RepairTable archives input file.
    }
  }

Here is the call graph for this function:

WriteDescriptor()

Status leveldb::anonymous_namespace{repair.cc}::Repairer::WriteDescriptor ( )

inlineprivate

Definition at line 375 of file repair.cc.

                           {
    std::string tmp = TempFileName(dbname_, 1);
    WritableFile* file;
    Status status = env_->NewWritableFile(tmp, &file);
    if (!status.ok()) {
      return status;
    }

    SequenceNumber max_sequence = 0;
    for (size_t i = 0; i < tables_.size(); i++) {
      if (max_sequence < tables_[i].max_sequence) {
        max_sequence = tables_[i].max_sequence;
      }
    }

    edit_.SetComparatorName(icmp_.user_comparator()->Name());
    edit_.SetLogNumber(0);
    edit_.SetNextFile(next_file_number_);
    edit_.SetLastSequence(max_sequence);

    for (size_t i = 0; i < tables_.size(); i++) {
      // TODO(opt): separate out into multiple levels
      const TableInfo& t = tables_[i];
      edit_.AddFile(0, t.meta.number, t.meta.file_size,
                    t.meta.smallest, t.meta.largest);
    }

    //fprintf(stderr, "NewDescriptor:\n%s\n", edit_.DebugString().c_str());
    {
      log::Writer log(file);
      std::string record;
      edit_.EncodeTo(&record);
      status = log.AddRecord(record);
    }
    if (status.ok()) {
      status = file->Close();
    }
    delete file;
    file = NULL;

    if (!status.ok()) {
      env_->DeleteFile(tmp);
    } else {
      // Discard older manifests
      for (size_t i = 0; i < manifests_.size(); i++) {
        ArchiveFile(dbname_ + "/" + manifests_[i]);
      }

      // Install new manifest
      status = env_->RenameFile(tmp, DescriptorFileName(dbname_, 1));
      if (status.ok()) {
        status = SetCurrentFile(env_, dbname_, 1);
      } else {
        env_->DeleteFile(tmp);
      }
    }
    return status;
  }

Here is the call graph for this function:

Member Data Documentation

dbname_

std::string const leveldb::anonymous_namespace{repair.cc}::Repairer::dbname_

private

Definition at line 100 of file repair.cc.

edit_

VersionEdit leveldb::anonymous_namespace{repair.cc}::Repairer::edit_

private

Definition at line 108 of file repair.cc.

env_

Env* const leveldb::anonymous_namespace{repair.cc}::Repairer::env_

private

Definition at line 101 of file repair.cc.

icmp_

InternalKeyComparator const leveldb::anonymous_namespace{repair.cc}::Repairer::icmp_

private

Definition at line 102 of file repair.cc.

ipolicy_

InternalFilterPolicy const leveldb::anonymous_namespace{repair.cc}::Repairer::ipolicy_

private

Definition at line 103 of file repair.cc.

logs_

std::vector<uint64_t> leveldb::anonymous_namespace{repair.cc}::Repairer::logs_

private

Definition at line 112 of file repair.cc.

manifests_

std::vector<std::string> leveldb::anonymous_namespace{repair.cc}::Repairer::manifests_

private

Definition at line 110 of file repair.cc.

next_file_number_

uint64_t leveldb::anonymous_namespace{repair.cc}::Repairer::next_file_number_

private

Definition at line 114 of file repair.cc.

options_

Options const leveldb::anonymous_namespace{repair.cc}::Repairer::options_

private

Definition at line 104 of file repair.cc.

owns_cache_

bool leveldb::anonymous_namespace{repair.cc}::Repairer::owns_cache_

private

Definition at line 106 of file repair.cc.

owns_info_log_

bool leveldb::anonymous_namespace{repair.cc}::Repairer::owns_info_log_

private

Definition at line 105 of file repair.cc.

table_cache_

TableCache* leveldb::anonymous_namespace{repair.cc}::Repairer::table_cache_

private

Definition at line 107 of file repair.cc.

table_numbers_

std::vector<uint64_t> leveldb::anonymous_namespace{repair.cc}::Repairer::table_numbers_

private

Definition at line 111 of file repair.cc.

tables_

std::vector<TableInfo> leveldb::anonymous_namespace{repair.cc}::Repairer::tables_

private

Definition at line 113 of file repair.cc.

---

The documentation for this class was generated from the following file:

• db/repair.cc