blockchain_export can now export to a blocks.dat format

Also make the number of blocks endian independant, and add
support for testnet
This commit is contained in:
moneromooo-monero 2015-10-16 19:45:35 +01:00
parent 11db442a6c
commit b13e7f284b
No known key found for this signature in database
GPG Key ID: 686F07454D6CEFC3
11 changed files with 368 additions and 38 deletions

View File

@ -38,11 +38,13 @@ bitmonero_private_headers(blockchain_converter
set(blockchain_import_sources
blockchain_import.cpp
bootstrap_file.cpp
blocksdat_file.cpp
)
set(blockchain_import_private_headers
fake_core.h
bootstrap_file.h
blocksdat_file.h
bootstrap_serialization.h
)
@ -52,10 +54,12 @@ bitmonero_private_headers(blockchain_import
set(blockchain_export_sources
blockchain_export.cpp
bootstrap_file.cpp
blocksdat_file.cpp
)
set(blockchain_export_private_headers
bootstrap_file.h
blocksdat_file.h
bootstrap_serialization.h
)

View File

@ -27,6 +27,7 @@
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "bootstrap_file.h"
#include "blocksdat_file.h"
#include "common/command_line.h"
#include "version.h"
@ -39,6 +40,7 @@ int main(int argc, char* argv[])
{
uint32_t log_level = 0;
uint64_t block_stop = 0;
bool blocks_dat = false;
boost::filesystem::path default_data_path {tools::get_default_data_dir()};
boost::filesystem::path default_testnet_data_path {default_data_path / "testnet"};
@ -54,6 +56,7 @@ int main(int argc, char* argv[])
, "Run on testnet."
, false
};
const command_line::arg_descriptor<bool> arg_blocks_dat = {"blocksdat", "Output in blocks.dat format", blocks_dat};
command_line::add_arg(desc_cmd_sett, command_line::arg_data_dir, default_data_path.string());
@ -62,6 +65,7 @@ int main(int argc, char* argv[])
command_line::add_arg(desc_cmd_sett, arg_testnet_on);
command_line::add_arg(desc_cmd_sett, arg_log_level);
command_line::add_arg(desc_cmd_sett, arg_block_stop);
command_line::add_arg(desc_cmd_sett, arg_blocks_dat);
command_line::add_arg(desc_cmd_only, command_line::arg_help);
@ -94,6 +98,7 @@ int main(int argc, char* argv[])
LOG_PRINT_L0("Setting log level = " << log_level);
bool opt_testnet = command_line::get_arg(vm, arg_testnet_on);
bool opt_blocks_dat = command_line::get_arg(vm, arg_blocks_dat);
std::string m_config_folder;
@ -155,8 +160,16 @@ int main(int argc, char* argv[])
LOG_PRINT_L0("Source blockchain storage initialized OK");
LOG_PRINT_L0("Exporting blockchain raw data...");
BootstrapFile bootstrap;
r = bootstrap.store_blockchain_raw(core_storage, NULL, output_file_path, block_stop);
if (opt_blocks_dat)
{
BlocksdatFile blocksdat;
r = blocksdat.store_blockchain_raw(core_storage, NULL, output_file_path, block_stop);
}
else
{
BootstrapFile bootstrap;
r = bootstrap.store_blockchain_raw(core_storage, NULL, output_file_path, block_stop);
}
CHECK_AND_ASSERT_MES(r, false, "Failed to export blockchain raw data");
LOG_PRINT_L0("Blockchain raw data exported OK");
}

View File

@ -0,0 +1,51 @@
// Copyright (c) 2014-2015, The Monero Project
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#pragma once
#include "version.h"
// CONFIG: choose one of the three #define's
//
// DB_MEMORY is a sensible default for users migrating to LMDB, as it allows
// the exporter to use the in-memory blockchain while the other binaries
// work with LMDB, without recompiling anything.
//
#define SOURCE_DB DB_MEMORY
//#define SOURCE_DB DB_LMDB
// to use global compile-time setting (DB_MEMORY or DB_LMDB):
// #define SOURCE_DB BLOCKCHAIN_DB
// bounds checking is done before writing to buffer, but buffer size
// should be a sensible maximum
#define BUFFER_SIZE 1000000
#define NUM_BLOCKS_PER_CHUNK 1
#define BLOCKCHAIN_RAW "blockchain.raw"

View File

@ -0,0 +1,170 @@
// Copyright (c) 2014-2015, The Monero Project
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "blocksdat_file.h"
namespace po = boost::program_options;
using namespace cryptonote;
using namespace epee;
namespace
{
std::string refresh_string = "\r \r";
}
bool BlocksdatFile::open_writer(const boost::filesystem::path& file_path, uint64_t block_stop)
{
const boost::filesystem::path dir_path = file_path.parent_path();
if (!dir_path.empty())
{
if (boost::filesystem::exists(dir_path))
{
if (!boost::filesystem::is_directory(dir_path))
{
LOG_PRINT_RED_L0("export directory path is a file: " << dir_path);
return false;
}
}
else
{
if (!boost::filesystem::create_directory(dir_path))
{
LOG_PRINT_RED_L0("Failed to create directory " << dir_path);
return false;
}
}
}
m_raw_data_file = new std::ofstream();
uint64_t num_blocks = 0;
LOG_PRINT_L0("creating file");
m_raw_data_file->open(file_path.string(), std::ios_base::binary | std::ios_base::out | std::ios::trunc);
if (m_raw_data_file->fail())
return false;
initialize_file(block_stop);
return true;
}
bool BlocksdatFile::initialize_file(uint64_t block_stop)
{
const uint32_t nblocks = block_stop + 1;
unsigned char nblocksc[4];
nblocksc[0] = nblocks & 0xff;
nblocksc[1] = (nblocks >> 8) & 0xff;
nblocksc[2] = (nblocks >> 16) & 0xff;
nblocksc[3] = (nblocks >> 24) & 0xff;
// 4 bytes little endian
*m_raw_data_file << nblocksc[0];
*m_raw_data_file << nblocksc[1];
*m_raw_data_file << nblocksc[2];
*m_raw_data_file << nblocksc[3];
return true;
}
void BlocksdatFile::write_block(const crypto::hash& block_hash)
{
const std::string data(block_hash.data, sizeof(block_hash));
*m_raw_data_file << data;
}
bool BlocksdatFile::close()
{
if (m_raw_data_file->fail())
return false;
m_raw_data_file->flush();
delete m_raw_data_file;
return true;
}
#if SOURCE_DB == DB_MEMORY
bool BlocksdatFile::store_blockchain_raw(blockchain_storage* _blockchain_storage, tx_memory_pool* _tx_pool, boost::filesystem::path& output_file, uint64_t requested_block_stop)
#else
bool BlocksdatFile::store_blockchain_raw(Blockchain* _blockchain_storage, tx_memory_pool* _tx_pool, boost::filesystem::path& output_file, uint64_t requested_block_stop)
#endif
{
uint64_t num_blocks_written = 0;
m_blockchain_storage = _blockchain_storage;
uint64_t progress_interval = 100;
block b;
uint64_t block_start = 0;
uint64_t block_stop = 0;
LOG_PRINT_L0("source blockchain height: " << m_blockchain_storage->get_current_blockchain_height()-1);
if ((requested_block_stop > 0) && (requested_block_stop < m_blockchain_storage->get_current_blockchain_height()))
{
LOG_PRINT_L0("Using requested block height: " << requested_block_stop);
block_stop = requested_block_stop;
}
else
{
block_stop = m_blockchain_storage->get_current_blockchain_height() - 1;
LOG_PRINT_L0("Using block height of source blockchain: " << block_stop);
}
LOG_PRINT_L0("Storing blocks raw data...");
if (!BlocksdatFile::open_writer(output_file, block_stop))
{
LOG_PRINT_RED_L0("failed to open raw file for write");
return false;
}
for (m_cur_height = block_start; m_cur_height <= block_stop; ++m_cur_height)
{
// this method's height refers to 0-based height (genesis block = height 0)
crypto::hash hash = m_blockchain_storage->get_block_id_by_height(m_cur_height);
write_block(hash);
if (m_cur_height % NUM_BLOCKS_PER_CHUNK == 0) {
num_blocks_written += NUM_BLOCKS_PER_CHUNK;
}
if (m_cur_height % progress_interval == 0) {
std::cout << refresh_string;
std::cout << "block " << m_cur_height << "/" << block_stop << std::flush;
}
}
// print message for last block, which may not have been printed yet due to progress_interval
std::cout << refresh_string;
std::cout << "block " << m_cur_height-1 << "/" << block_stop << ENDL;
LOG_PRINT_L0("Number of blocks exported: " << num_blocks_written);
return BlocksdatFile::close();
}

View File

@ -0,0 +1,90 @@
// Copyright (c) 2014-2015, The Monero Project
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#pragma once
#include <boost/iostreams/stream_buffer.hpp>
#include <boost/iostreams/stream.hpp>
#include <boost/iostreams/device/back_inserter.hpp>
#include <boost/iostreams/filtering_streambuf.hpp>
#include "cryptonote_core/cryptonote_basic.h"
#include "cryptonote_core/cryptonote_boost_serialization.h"
#include "cryptonote_core/blockchain_storage.h"
#include "cryptonote_core/blockchain.h"
#include "blockchain_db/blockchain_db.h"
#include "blockchain_db/lmdb/db_lmdb.h"
#include <algorithm>
#include <cstdio>
#include <fstream>
#include <boost/iostreams/copy.hpp>
#include <atomic>
#include "common/command_line.h"
#include "version.h"
#include "blockchain_utilities.h"
using namespace cryptonote;
class BlocksdatFile
{
public:
#if SOURCE_DB == DB_MEMORY
bool store_blockchain_raw(cryptonote::blockchain_storage* cs, cryptonote::tx_memory_pool* txp,
boost::filesystem::path& output_file, uint64_t use_block_height=0);
#else
bool store_blockchain_raw(cryptonote::Blockchain* cs, cryptonote::tx_memory_pool* txp,
boost::filesystem::path& output_file, uint64_t use_block_height=0);
#endif
protected:
#if SOURCE_DB == DB_MEMORY
blockchain_storage* m_blockchain_storage;
#else
Blockchain* m_blockchain_storage;
#endif
std::ofstream * m_raw_data_file;
// open export file for write
bool open_writer(const boost::filesystem::path& file_path, uint64_t block_stop);
bool initialize_file(uint64_t block_stop);
bool close();
void write_block(const crypto::hash &block_hash);
private:
uint64_t m_cur_height; // tracks current height during export
};

View File

@ -49,24 +49,7 @@
#include "common/command_line.h"
#include "version.h"
// CONFIG: choose one of the three #define's
//
// DB_MEMORY is a sensible default for users migrating to LMDB, as it allows
// the exporter to use the in-memory blockchain while the other binaries
// work with LMDB, without recompiling anything.
//
#define SOURCE_DB DB_MEMORY
// #define SOURCE_DB DB_LMDB
// to use global compile-time setting (DB_MEMORY or DB_LMDB):
// #define SOURCE_DB BLOCKCHAIN_DB
// bounds checking is done before writing to buffer, but buffer size
// should be a sensible maximum
#define BUFFER_SIZE 1000000
#define NUM_BLOCKS_PER_CHUNK 1
#define BLOCKCHAIN_RAW "blockchain.raw"
#include "blockchain_utilities.h"
using namespace cryptonote;

View File

@ -30,9 +30,9 @@ if(APPLE)
add_library(blocks STATIC blockexports.c)
set_target_properties(blocks PROPERTIES LINKER_LANGUAGE C)
else()
add_custom_command(OUTPUT blocks.o COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && ld -r -b binary -o ${CMAKE_CURRENT_BINARY_DIR}/blocks.o blocks.dat)
add_library(blocks STATIC blocks.o blockexports.c)
add_custom_command(OUTPUT blocks.o COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && ld -r -b binary -o ${CMAKE_CURRENT_BINARY_DIR}/blocks.o blocks.dat)
add_custom_command(OUTPUT testnet_blocks.o COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && ld -r -b binary -o ${CMAKE_CURRENT_BINARY_DIR}/testnet_blocks.o testnet_blocks.dat)
add_library(blocks STATIC blocks.o testnet_blocks.o blockexports.c)
set_target_properties(blocks PROPERTIES LINKER_LANGUAGE C)
endif()

View File

@ -9,16 +9,22 @@ extern const struct mach_header _mh_execute_header;
extern const struct mach_header_64 _mh_execute_header;
#endif
const unsigned char *get_blocks_dat_start()
const unsigned char *get_blocks_dat_start(int testnet)
{
size_t size;
return getsectiondata(&_mh_execute_header, "__DATA", "__blocks_dat", &size);
if (testnet)
return getsectiondata(&_mh_execute_header, "__DATA", "__testnet_blocks_dat", &size);
else
return getsectiondata(&_mh_execute_header, "__DATA", "__blocks_dat", &size);
}
size_t get_blocks_dat_size()
size_t get_blocks_dat_size(int testnet)
{
size_t size;
getsectiondata(&_mh_execute_header, "__DATA", "__blocks_dat", &size);
if (testnet)
getsectiondata(&_mh_execute_header, "__DATA", "__testnet_blocks_dat", &size);
else
getsectiondata(&_mh_execute_header, "__DATA", "__blocks_dat", &size);
return size;
}
@ -27,22 +33,34 @@ size_t get_blocks_dat_size()
#if defined(_WIN32) && !defined(_WIN64)
#define _binary_blocks_start binary_blocks_dat_start
#define _binary_blocks_end binary_blocks_dat_end
#define _binary_testnet_blocks_start binary_testnet_blocks_dat_start
#define _binary_testnet_blocks_end binary_testnet_blocks_dat_end
#else
#define _binary_blocks_start _binary_blocks_dat_start
#define _binary_blocks_end _binary_blocks_dat_end
#define _binary_testnet_blocks_start _binary_testnet_blocks_dat_start
#define _binary_testnet_blocks_end _binary_testnet_blocks_dat_end
#endif
extern const unsigned char _binary_blocks_start[];
extern const unsigned char _binary_blocks_end[];
extern const unsigned char _binary_testnet_blocks_start[];
extern const unsigned char _binary_testnet_blocks_end[];
const unsigned char *get_blocks_dat_start(void)
const unsigned char *get_blocks_dat_start(int testnet)
{
return _binary_blocks_start;
if (testnet)
return _binary_testnet_blocks_start;
else
return _binary_blocks_start;
}
size_t get_blocks_dat_size(void)
size_t get_blocks_dat_size(int testnet)
{
return (size_t) (_binary_blocks_end - _binary_blocks_start);
if (testnet)
return (size_t) (_binary_testnet_blocks_end - _binary_testnet_blocks_start);
else
return (size_t) (_binary_blocks_end - _binary_blocks_start);
}
#endif

View File

@ -5,8 +5,8 @@
extern "C" {
#endif
const unsigned char *get_blocks_dat_start();
size_t get_blocks_dat_size();
const unsigned char *get_blocks_dat_start(int testnet);
size_t get_blocks_dat_size(int testnet);
#ifdef __cplusplus
}

View File

View File

@ -352,13 +352,14 @@ bool Blockchain::init(BlockchainDB* db, const bool testnet)
m_async_pool.create_thread(boost::bind(&boost::asio::io_service::run, &m_async_service));
#if defined(PER_BLOCK_CHECKPOINT)
if (m_fast_sync && !testnet && get_blocks_dat_start() != nullptr)
if (m_fast_sync && get_blocks_dat_start(testnet) != nullptr)
{
if (get_blocks_dat_size() > 4)
if (get_blocks_dat_size(testnet) > 4)
{
const unsigned char *p = get_blocks_dat_start();
uint32_t nblocks = *(uint32_t *) p;
if(nblocks > 0 && nblocks > m_db->height())
const unsigned char *p = get_blocks_dat_start(testnet);
const uint32_t nblocks = *p | ((*(p+1))<<8) | ((*(p+2))<<16) | ((*(p+3))<<24);
const size_t size_needed = 4 + nblocks * sizeof(crypto::hash);
if(nblocks > 0 && nblocks > m_db->height() && get_blocks_dat_size(testnet) >= size_needed)
{
LOG_PRINT_L0("Loading precomputed blocks: " << nblocks);
p += sizeof(uint32_t);