vitalif
/
zbackup
mirror of https://github.com/vitalif/zbackup
1
0
Fork 0
Code Issues Releases Wiki Activity

Added support for -o lzma.compression_level: 

* Closes https://github.com/zbackup/zbackup/pull/10 completely
* editConfigInteractively moved to ZBackupBase again (to make possible
  to use Config almost anywhere)
master
Vladimir Stackov 2015-01-23 14:32:40 +03:00
parent c5b821dba4
commit 30fe01a2a0
12 changed files with 151 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
bundle.cc
View File

@ -90,7 +90,8 @@ void Creator::write( std::string const & fileName, EncryptionKey const & key,
}
}
void Creator::write( std::string const & fileName, EncryptionKey const & key )
void Creator::write( Config const & config, std::string const & fileName,
EncryptionKey const & key )
{
EncryptedFile::OutputStream os( fileName.c_str(), key, Encryption::ZeroIv );
@ -98,7 +99,8 @@ void Creator::write( std::string const & fileName, EncryptionKey const & key )
BundleFileHeader header;
const_sptr<Compression::CompressionMethod> compression = Compression::CompressionMethod::selectedCompression;
const_sptr<Compression::CompressionMethod> compression =
Compression::CompressionMethod::selectedCompression;
header.set_compression_method( compression->getName() );
// The old code only support lzma, so we will bump up the version, if we're
@ -115,7 +117,8 @@ void Creator::write( std::string const & fileName, EncryptionKey const & key )
// Compress
sptr<Compression::EnDecoder> encoder = compression->createEncoder();
sptr<Compression::EnDecoder> encoder = compression->createEncoder(
config );
encoder->setInput( payload.data(), payload.size() );
@ -135,7 +138,7 @@ void Creator::write( std::string const & fileName, EncryptionKey const & key )
}
// Perform the compression
if ( encoder->process(true) )
if ( encoder->process( true ) )
{
if ( encoder->getAvailableOutput() )
os.BackUp( encoder->getAvailableOutput() );
@ -190,7 +193,8 @@ Reader::Reader( string const & fileName, EncryptionKey const & key, bool prohibi
decoder->setInput( data, size );
}
if ( decoder->process(false) ) {
if ( decoder->process( false ) )
{
if ( decoder->getAvailableInput() )
is.BackUp( decoder->getAvailableInput() );
break;

3
bundle.hh
View File

@ -17,6 +17,7 @@
#include "static_assert.hh"
#include "zbackup.pb.h"
#include "encrypted_file.hh"
#include "config.hh"
namespace Bundle {
@ -101,7 +102,7 @@ public:
/// Compresses and writes the bundle to the given file. The operation is
/// time-consuming - calling this function from a worker thread could be
/// warranted
void write( string const & fileName, EncryptionKey const & );
void write( Config const &, string const & fileName, EncryptionKey const & );
void write( string const & fileName, EncryptionKey const &,
Bundle::Reader & reader );

10
chunk_storage.cc
View File

@ -130,7 +130,8 @@ void Writer::finishCurrentBundle()
while ( runningCompressors >= maxCompressorsToRun )
runningCompressorsCondition.wait( runningCompressorsMutex );
Compressor * compressor = new Compressor( *this, currentBundle,
Compressor * compressor = new Compressor( config,
*this, currentBundle,
file->getFileName() );
currentBundle.reset();
@ -159,10 +160,11 @@ Bundle::Id const & Writer::getCurrentBundleId()
return currentBundleId;
}
Writer::Compressor::Compressor( Writer & writer,
Writer::Compressor::Compressor( Config const & configIn, Writer & writer,
sptr< Bundle::Creator > const & bundleCreator,
string const & fileName ):
writer( writer ), bundleCreator( bundleCreator ), fileName( fileName )
writer( writer ), bundleCreator( bundleCreator ), fileName( fileName ),
config( configIn )
{
}
@ -170,7 +172,7 @@ void * Writer::Compressor::Compressor::threadFunction() throw()
{
try
{
bundleCreator->write( fileName, writer.encryptionKey );
bundleCreator->write( config, fileName, writer.encryptionKey );
}
catch( std::exception & e )
{

3
chunk_storage.hh
View File

@ -68,8 +68,9 @@ private:
Writer & writer;
sptr< Bundle::Creator > bundleCreator;
string fileName;
Config const & config;
public:
Compressor( Writer &, sptr< Bundle::Creator > const &,
Compressor( Config const &, Writer &, sptr< Bundle::Creator > const &,
string const & fileName );
protected:
virtual void * threadFunction() throw();

68
compression.cc
View File

@ -8,11 +8,17 @@
namespace Compression {
EnDecoder::EnDecoder() { }
EnDecoder::~EnDecoder() { }
EnDecoder::EnDecoder()
{
}
CompressionMethod::~CompressionMethod() { }
EnDecoder::~EnDecoder()
{
}
CompressionMethod::~CompressionMethod()
{
}
// LZMA
@ -72,8 +78,17 @@ class LZMAEncoder : public LZMAEnDecoder
public:
LZMAEncoder()
{
uint32_t preset = 6; // TODO: make this customizable, although 6 seems to be
// the best option
uint32_t preset = 6;
lzma_ret ret = lzma_easy_encoder( &strm, preset, LZMA_CHECK_CRC64 );
CHECK( ret == LZMA_OK, "lzma_easy_encoder error: %d", (int) ret );
}
LZMAEncoder( Config const & config )
{
uint32_t compressionLevel = config.GET_STORABLE( lzma, compression_level );
uint32_t preset = ( compressionLevel > 9 ) ?
( compressionLevel - 10 ) | LZMA_PRESET_EXTREME :
compressionLevel;
lzma_ret ret = lzma_easy_encoder( &strm, preset, LZMA_CHECK_CRC64 );
CHECK( ret == LZMA_OK, "lzma_easy_encoder error: %d", (int) ret );
}
@ -92,6 +107,11 @@ public:
class LZMACompression : public CompressionMethod
{
public:
sptr<EnDecoder> createEncoder( Config const & config ) const
{
return new LZMAEncoder( config );
}
sptr<EnDecoder> createEncoder() const
{
return new LZMAEncoder();
@ -105,7 +125,6 @@ public:
std::string getName() const { return "lzma"; }
};
// LZO
// liblzo implements a lot of algorithms "for unlimited backward compatibility"
@ -232,7 +251,7 @@ public:
{
// data has been encoded or decoded, remaining output is in accDataOut
// -> copy to output
if (availOut > 0 && accDataOut.size() - posInAccDataOut > 0)
if ( availOut > 0 && accDataOut.size() - posInAccDataOut > 0 )
{
size_t sz = availOut;
if ( sz > accDataOut.size() - posInAccDataOut )
@ -273,7 +292,7 @@ private:
// we use our own buffer
size_t bufferSize = suggestOutputSize( dataIn, availIn );
do {
accDataOut.resize(bufferSize);
accDataOut.resize( bufferSize );
size_t outputSize;
//TODO doc says we mustn't modify the pointer returned by data()...
@ -312,7 +331,6 @@ protected:
virtual bool doProcessNoSize( const char* dataIn, size_t availIn,
char* dataOut, size_t availOut, size_t& outputSize ) =0;
bool shouldTryWith( const char* dataIn, size_t availIn, size_t availOut )
{
return suggestOutputSize( dataIn, availIn ) <= availOut;
@ -388,7 +406,6 @@ protected:
virtual bool doProcessNoSize( const char* dataIn, size_t availIn,
char* dataOut, size_t availOut, size_t& outputSize ) =0;
bool shouldTryWith( const char*, size_t, size_t availOut )
{
// If the compression doesn't use any spaces...
@ -443,7 +460,6 @@ protected:
}
};
#ifdef HAVE_LIBLZO
#include <lzo/lzo1x.h>
@ -472,9 +488,9 @@ class LZO1X_1_Compression;
class LZO1X_1_Encoder : public NoStreamAndUnknownSizeEncoder
{
const LZO1X_1_Compression* compression;
static size_t calcMaxCompressedSize(size_t availIn);
static size_t calcMaxCompressedSize( size_t availIn );
public:
LZO1X_1_Encoder(const LZO1X_1_Compression* compression)
LZO1X_1_Encoder( const LZO1X_1_Compression* compression )
{
this->compression = compression;
}
@ -499,13 +515,19 @@ class LZO1X_1_Compression : public CompressionMethod
}
}
public:
sptr<EnDecoder> createEncoder() const
sptr< EnDecoder > createEncoder( Config const & config ) const
{
init();
return new LZO1X_1_Encoder(this);
}
sptr<EnDecoder> createDecoder() const
sptr< EnDecoder > createEncoder() const
{
init();
return new LZO1X_1_Encoder(this);
}
sptr< EnDecoder > createDecoder() const
{
init();
return new LZO1X_1_Decoder();
@ -513,7 +535,6 @@ public:
std::string getName() const { return "lzo1x_1"; }
lzo_voidp getWorkmem( size_t size ) const
{
return new char[size];
@ -531,7 +552,6 @@ public:
bool LZO1X_1_Compression::initialized = false;
size_t LZO1X_1_Encoder::calcMaxCompressedSize( size_t availIn )
{
// It seems that lzo1x_1_compress does NOT check whether the buffer is big enough.
@ -596,12 +616,12 @@ const_sptr< CompressionMethod > const CompressionMethod::compressions[] = {
};
const_sptr< CompressionMethod > CompressionMethod::selectedCompression =
compressions[0];
compressions[ 0 ];
const_sptr< CompressionMethod > CompressionMethod::findCompression(
const std::string& name, bool optional )
{
for ( const const_sptr<CompressionMethod>* c = compressions+0; *c; ++c )
for ( const const_sptr<CompressionMethod>* c = compressions + 0; *c; ++c )
{
if ( (*c)->getName() == name )
{
@ -616,9 +636,15 @@ const_sptr< CompressionMethod > CompressionMethod::findCompression(
}
// iterator over compressions
CompressionMethod::iterator::iterator( const const_sptr< CompressionMethod > * ptr ):
ptr( ptr )
{
}
CompressionMethod::iterator::iterator( const const_sptr<CompressionMethod>* ptr ) : ptr( ptr) { }
CompressionMethod::iterator::iterator( const iterator& it ) : ptr(it.ptr) { }
CompressionMethod::iterator::iterator( const iterator & it ):
ptr( it.ptr )
{
}
CompressionMethod::iterator& CompressionMethod::iterator::operator =( const iterator& it )
{

2
compression.hh
View File

@ -7,6 +7,7 @@
#include "sptr.hh"
#include "ex.hh"
#include "nocopy.hh"
#include "config.hh"
namespace Compression {
@ -48,6 +49,7 @@ public:
// This name is saved in the file header of the compressed file.
virtual std::string getName() const = 0;
virtual sptr< EnDecoder > createEncoder( Config const & ) const = 0;
virtual sptr< EnDecoder > createEncoder() const = 0;
virtual sptr< EnDecoder > createDecoder() const = 0;

72
config.cc
View File

@ -22,6 +22,9 @@
return true; \
}
// Some configurables could be just a switch
// So we introducing a macros that would indicate
// that this configurable is not a switch
#define REQUIRE_VALUE \
{ \
if ( !hasValue && !validate ) \
@ -52,7 +55,9 @@ static struct
Config::oChunk_max_size,
Config::Storable,
"Maximum chunk size used when storing chunks\n"
"Affects deduplication ratio directly"
"Affects deduplication ratio directly\n"
"Default is %s",
Utils::numberToString( defaultConfig.GET_STORABLE( chunk, max_size ) )
},
{
"bundle.max_payload_size",
@ -61,13 +66,26 @@ static struct
"Maximum number of bytes a bundle can hold. Only real chunk bytes are\n"
"counted, not metadata. Any bundle should be able to contain at least\n"
"one arbitrary single chunk, so this should not be smaller than\n"
"chunk.max_size"
"chunk.max_size\n"
"Default is %s",
Utils::numberToString( defaultConfig.GET_STORABLE( bundle, max_payload_size ) )
},
{
"bundle.compression_method",
Config::oBundle_compression_method,
Config::Storable,
"Compression method for new bundles"
"Compression method for new bundles\n"
"Default is %s",
defaultConfig.GET_STORABLE( bundle, compression_method )
},
{
"lzma.compression_level",
Config::oLZMA_compression_level,
Config::Storable,
"Compression level for new LZMA-compressed files\n"
"Valid values: 0-19 (values over 9 enables extreme mode)\n"
"Default is %s",
Utils::numberToString( defaultConfig.GET_STORABLE( lzma, compression_level ) )
},
// Shortcuts for storable options
@ -75,7 +93,9 @@ static struct
"compression",
Config::oBundle_compression_method,
Config::Storable,
"Shortcut for bundle.compression_method"
"Shortcut for bundle.compression_method\n"
"Default is %s",
defaultConfig.GET_STORABLE( bundle, compression_method )
},
// Runtime options
@ -239,6 +259,25 @@ bool Config::parseOrValidate( const char * option, const OptionType type,
/* NOTREACHED */
break;
case oLZMA_compression_level:
REQUIRE_VALUE;
if ( PARSE_OR_VALIDATE(
sscanf( optionValue, "%zu %n", &uint32Value, &n ) != 1 ||
optionValue[ n ] || uint32Value > 19,
GET_STORABLE( lzma, compression_level ) > 19 )
)
return false;
SKIP_ON_VALIDATION;
SET_STORABLE( lzma, compression_level, uint32Value );
dPrintf( "storable[lzma][compression_level] = %zu\n",
GET_STORABLE( lzma, compression_level ) );
return true;
/* NOTREACHED */
break;
case oBundle_compression_method:
REQUIRE_VALUE;
@ -504,6 +543,7 @@ void Config::reset_storable()
SET_STORABLE( chunk, max_size, GET_STORABLE( chunk, max_size ) );
SET_STORABLE( bundle, max_payload_size, GET_STORABLE( bundle, max_payload_size ) );
SET_STORABLE( bundle, compression_method, GET_STORABLE( bundle, compression_method ) );
SET_STORABLE( lzma, compression_level, GET_STORABLE( lzma, compression_level ) );
}
void Config::show()
@ -515,27 +555,3 @@ void Config::show( const ConfigInfo & config )
{
printf( "%s", toString( config ).c_str() );
}
bool Config::editInteractively( ZBackupBase * zbb )
{
string configData( toString( *zbb->config.storable ) );
if ( !zbb->spawnEditor( configData, &validateProto ) )
return false;
ConfigInfo newConfig;
parseProto( configData, &newConfig );
if ( toString( *zbb->config.storable ) == toString( newConfig ) )
{
verbosePrintf( "No changes made to config\n" );
return false;
}
verbosePrintf( "Updating configuration...\n" );
zbb->config.storable->MergeFrom( newConfig );
verbosePrintf(
"Configuration successfully updated!\n"
"Updated configuration:\n%s", toString( *zbb->config.storable ).c_str() );
return true;
}

9
config.hh
View File

@ -21,8 +21,6 @@
using std::string;
using std::bitset;
class ZBackupBase;
class Config
{
public:
@ -55,6 +53,7 @@ public:
oChunk_max_size,
oBundle_max_payload_size,
oBundle_compression_method,
oLZMA_compression_level,
oRuntime_threads,
oRuntime_cacheSize,
@ -72,10 +71,6 @@ public:
static string toString( google::protobuf::Message const & );
// Edit current configuration
// returns true if configuration is changed
static bool editInteractively( ZBackupBase * );
// Print configuration to screen
static void show( const ConfigInfo & );
void show();
@ -96,6 +91,4 @@ private:
bool want_cleanup;
};
#include "zbackup_base.hh"
#endif

2
zbackup.cc
View File

@ -649,7 +649,7 @@ invalid_option:
{
ZBackupBase zbb( ZBackupBase::deriveStorageDirFromBackupsFile( args[ fieldStorage ], true ),
passwords[ 0 ], true );
if ( Config::editInteractively( &zbb ) )
if ( zbb.editConfigInteractively() )
zbb.saveExtendedStorageInfo();
}
else

7
zbackup.proto
View File

@ -42,6 +42,12 @@ message StorageInfo
optional string default_compression_method = 4 [default = "lzma", deprecated = true];
}
message LZMAConfigInfo
{
// Compression level for new LZMA-compressed files
optional uint32 compression_level = 1 [default = 6];
}
message ChunkConfigInfo
{
// Maximum chunk size used when storing chunks
@ -64,6 +70,7 @@ message ConfigInfo
{
required ChunkConfigInfo chunk = 1;
required BundleConfigInfo bundle = 2;
required LZMAConfigInfo lzma = 3;
}
message ExtendedStorageInfo

27
zbackup_base.cc
View File

@ -116,7 +116,7 @@ ZBackupBase::ZBackupBase( string const & storageDir, string const & password,
storageDir.c_str() );
}
// Update all internal variables after configuration change
// Update all internal variables according to real configuration
// Dunno why someone need to store duplicate information
// in deduplication utility
void ZBackupBase::propagateUpdate()
@ -419,3 +419,28 @@ end:
return isChanged;
}
bool ZBackupBase::editConfigInteractively()
{
string configData( Config::toString( *config.storable ) );
if ( !spawnEditor( configData, &Config::validateProto ) )
return false;
ConfigInfo newConfig;
Config::parseProto( configData, &newConfig );
if ( Config::toString( *config.storable ) ==
Config::toString( newConfig ) )
{
verbosePrintf( "No changes made to config\n" );
return false;
}
verbosePrintf( "Updating configuration...\n" );
config.storable->MergeFrom( newConfig );
verbosePrintf(
"Configuration successfully updated!\n"
"Updated configuration:\n%s", Config::toString( *config.storable ).c_str() );
return true;
}

4
zbackup_base.hh
View File

@ -67,6 +67,10 @@ public:
bool spawnEditor( std::string & data, bool( * validator )
( string const &, string const & ) );
// Edit current configuration
// returns true if configuration is changed
bool editConfigInteractively();
StorageInfo storageInfo;
EncryptionKey encryptionkey;
ExtendedStorageInfo extendedStorageInfo;