Initial prototype of protected config storage

chunk_index.cc

@@ -33,7 +33,7 @@ void ChunkIndex::loadIndex( IndexProcessor & ip )
 
   while( lst.getNext( entry ) )
   {
-    verbosePrintf( "Loading index file %s... ", entry.getFileName().c_str() );
+    verbosePrintf( "Loading index file %s...\n", entry.getFileName().c_str() );
     try
     {
       string indexFn = Dir::addPath( indexPath, entry.getFileName() );
@@ -73,7 +73,6 @@ void ChunkIndex::loadIndex( IndexProcessor & ip )
       fprintf( stderr, "error: %s\n", e.what() );
       continue;
     }
-    verbosePrintf( "\n" );
   }
 
   verbosePrintf( "Index loaded.\n" );
@@ -106,6 +105,8 @@ ChunkIndex::ChunkIndex( EncryptionKey const & key, TmpMgr & tmpMgr,
   key( key ), tmpMgr( tmpMgr ), indexPath( indexPath ), storage( 65536, 1 ),
   lastBundleId( NULL )
 {
+  dPrintf( "Chunk index (%s) instantiated and initialized, hasKey: %s\n", indexPath.c_str(),
+      key.hasKey() ? "true" : "false" );
   if ( !prohibitChunkIndexLoading )
     loadIndex( *this );
 }

debug.hh

@@ -10,7 +10,11 @@
 
 #ifndef NDEBUG
 
-#define dPrintf( ... ) (fprintf( stderr, __VA_ARGS__ ))
+#define __FILE_BASE (strrchr(__FILE__, '/') ? strrchr(__FILE__, '/') + 1 : __FILE__)
+
+#define dPrintf( ... ) ({ fprintf( stderr, "[DEBUG] at %s( %s:%d ): ", __func__,\
+      __FILE_BASE, __LINE__ );\
+    fprintf( stderr, __VA_ARGS__ ); })
 
 #else
 

encrypted_file.cc

@@ -9,6 +9,7 @@
 #include "endian.hh"
 #include "page_size.hh"
 #include "random.hh"
+#include "debug.hh"
 
 namespace EncryptedFile {
 
@@ -22,6 +23,7 @@ InputStream::InputStream( char const * fileName, EncryptionKey const & key,
   buffer( std::max( getPageSize(), ( unsigned ) BlockSize * 2 ) ),
   fill( 0 ), remainder( 0 ), backedUp( false )
 {
+  dPrintf( "Loading %s, hasKey: %s\n", fileName, key.hasKey() ? "true" : "false" );
   if ( key.hasKey() )
   {
     memcpy( iv, iv_, sizeof( iv ) );
@@ -239,6 +241,7 @@ OutputStream::OutputStream( char const * fileName, EncryptionKey const & key,
   file( fileName, UnbufferedFile::WriteOnly ), filePos( 0 ), key( key ),
   buffer( getPageSize() ), start( buffer.data() ), avail( 0 ), backedUp( false )
 {
+  dPrintf( "Saving %s, hasKey: %s\n", fileName, key.hasKey() ? "true" : "false" );
   if ( key.hasKey() )
     memcpy( iv, iv_, sizeof( iv ) );
 }

storage_info_file.cc

@@ -6,6 +6,7 @@
 #include "encrypted_file.hh"
 #include "message.hh"
 #include "storage_info_file.hh"
+#include "debug.hh"
 
 namespace StorageInfoFile {
 
@@ -16,6 +17,7 @@ enum
 
 void save( string const & fileName, StorageInfo const & storageInfo )
 {
+  dPrintf( "Saving storage info...\n" );
   EncryptedFile::OutputStream os( fileName.c_str(), EncryptionKey::noKey(),
                                   NULL );
   FileHeader header;
@@ -28,6 +30,7 @@ void save( string const & fileName, StorageInfo const & storageInfo )
 
 void load( string const & fileName, StorageInfo & storageInfo )
 {
+  dPrintf( "Loading storage info...\n" );
   EncryptedFile::InputStream is( fileName.c_str(), EncryptionKey::noKey(),
                                  NULL );
   FileHeader header;
@@ -40,3 +43,47 @@ void load( string const & fileName, StorageInfo & storageInfo )
 }
 
 }
+
+namespace ExtendedStorageInfoFile {
+
+enum
+{
+  FileFormatVersion = 1
+};
+
+void save( string const & fileName, EncryptionKey const & encryptionKey,
+           ExtendedStorageInfo const & extendedStorageInfo )
+{
+  dPrintf( "Saving extended storage info, hasKey: %s\n",
+      encryptionKey.hasKey() ? "true" : "false" );
+  EncryptedFile::OutputStream os( fileName.c_str(), encryptionKey,
+                                  Encryption::ZeroIv );
+  os.writeRandomIv();
+
+  FileHeader header;
+  header.set_version( FileFormatVersion );
+  Message::serialize( header, os );
+
+  Message::serialize( extendedStorageInfo, os );
+  os.writeAdler32();
+}
+
+void load( string const & fileName, EncryptionKey const & encryptionKey,
+           ExtendedStorageInfo & extendedStorageInfo )
+{
+  dPrintf( "Loading extended storage info, hasKey: %s\n",
+      encryptionKey.hasKey() ? "true" : "false" );
+  EncryptedFile::InputStream is( fileName.c_str(), encryptionKey,
+                                 Encryption::ZeroIv );
+  is.consumeRandomIv();
+
+  FileHeader header;
+  Message::parse( header, is );
+  if ( header.version() != FileFormatVersion )
+    throw exUnsupportedVersion();
+
+  Message::parse( extendedStorageInfo, is );
+  is.checkAdler32();
+}
+
+}

storage_info_file.hh

@@ -25,4 +25,18 @@ void save( string const & fileName, StorageInfo const & );
 void load( string const & fileName, StorageInfo & );
 }
 
+namespace ExtendedStorageInfoFile {
+
+using std::string;
+
+DEF_EX( Ex, "Extended storage info file exception", std::exception )
+DEF_EX( exUnsupportedVersion, "Unsupported version of the extended storage info file format", Ex )
+
+/// Saves the given ExtendedStorageInfo data into the given file
+void save( string const & fileName, EncryptionKey const &, ExtendedStorageInfo const & );
+
+/// Loads the given ExtendedStorageInfo data from the given file
+void load( string const & fileName, EncryptionKey const &, ExtendedStorageInfo & );
+}
+
 #endif

zbackup.cc

@@ -441,7 +441,8 @@ int main( int argc, char *argv[] )
         ++x;
         if ( strcmp( argv[ x ], "lzma" ) == 0 )
         {
-          const_sptr<Compression::CompressionMethod> lzma = Compression::CompressionMethod::findCompression( "lzma" );
+          const_sptr<Compression::CompressionMethod> lzma =
+            Compression::CompressionMethod::findCompression( "lzma" );
           if ( !lzma )
           {
             fprintf( stderr, "zbackup is compiled without LZMA support, but the code "
@@ -454,7 +455,8 @@ int main( int argc, char *argv[] )
         else
         if ( strcmp( argv[ x ], "lzo" ) == 0 )
         {
-          const_sptr<Compression::CompressionMethod> lzo = Compression::CompressionMethod::findCompression( "lzo1x_1" );
+          const_sptr<Compression::CompressionMethod> lzo =
+            Compression::CompressionMethod::findCompression( "lzo1x_1" );
           if ( !lzo )
           {
             fprintf( stderr, "zbackup is compiled without LZO support, but the code "
@@ -509,7 +511,8 @@ int main( int argc, char *argv[] )
 "    import <source storage path> <destination storage path> -\n"
 "            performs import from source to destination storage;\n"
 "    gc <storage path> - performs chunk garbage collection;\n"
-"    passwd <storage path> - changes repository info file passphrase.\n"
+"    passwd <storage path> - changes repository info file passphrase;\n"
+"    info <storage path> - shows information about storage.\n"
 "  For export/import storage path must be valid (initialized) storage.\n"
 "", *argv,
                defaultThreads, defaultCacheSizeMb );
@@ -637,18 +640,34 @@ int main( int argc, char *argv[] )
         return EXIT_FAILURE;
       }
 
-      ZBackupBase zbb( args[ 1 ], passwords[ 0 ], true );
+      ZBackupBase zbb( ZBackupBase::deriveStorageDirFromBackupsFile( args[ 1 ], true ),
+          passwords[ 0 ], true );
+
       if ( passwords[ 0 ].empty() != passwords[ 1 ].empty() )
       {
         fprintf( stderr,
-"Changing repo encryption type (non-encrypted to encrypted and vice versa) "
+"Changing repo encryption type (non-encrypted to encrypted and vice versa) is possible "
-"is not supported yet.\n"
+"only via import/export operations.\n"
-"Current repo type: %s\n", zbb.encryptionkey.hasKey() ? "encrypted" : "non-encrypted" );
+"Current repo type: %s.\n", zbb.encryptionkey.hasKey() ? "encrypted" : "non-encrypted" );
         return EXIT_FAILURE;
       }
       zbb.setPassword( passwords[ 1 ] );
     }
     else
+    if ( strcmp( args[ 0 ], "info" ) == 0 )
+    {
+      // Show repo info
+      if ( args.size() != 2 )
+      {
+        fprintf( stderr, "Usage: %s info <storage path>\n",
+                 *argv );
+        return EXIT_FAILURE;
+      }
+
+      ZBackupBase zbb( ZBackupBase::deriveStorageDirFromBackupsFile( args[ 1 ], true ),
+          passwords[ 0 ] );
+    }
+    else
     {
       fprintf( stderr, "Error: unknown command line option: %s\n", args[ 0 ] );
       return EXIT_FAILURE;

zbackup.proto

@@ -30,16 +30,34 @@ message EncryptionKeyInfo
 message StorageInfo
 {
   // Maximum chunk size used when storing chunks
-  required uint32 chunk_max_size = 1;
+  required uint32 chunk_max_size = 1 [deprecated = true];
   // Maximum number of bytes a bundle can hold. Only real chunk bytes are
   // counted, not metadata. Any bundle should be able to contain at least
   // one arbitrary single chunk, so this should not be smaller than
   // chunk_max_size
-  required uint32 bundle_max_payload_size = 2;
+  required uint32 bundle_max_payload_size = 2 [deprecated = true];
   // If present, used for encryption/decryption of all data
   optional EncryptionKeyInfo encryption_key = 3;
   // Default compression for new bundles
-  optional string default_compression_method = 4 [default = "lzma"];
+  optional string default_compression_method = 4 [default = "lzma", deprecated = true];
+}
+
+message ConfigInfo
+{
+  required uint32 chunk_max_size = 1 [default = 65536];
+  // Maximum number of bytes a bundle can hold. Only real chunk bytes are
+  // counted, not metadata. Any bundle should be able to contain at least
+  // one arbitrary single chunk, so this should not be smaller than
+  // chunk_max_size
+  required uint32 bundle_max_payload_size = 2 [default = 0x200000];
+  // Default compression for new bundles
+  optional string default_compression_method = 3 [default = "lzma"];
+}
+
+message ExtendedStorageInfo
+{
+  // Config data storage
+  optional ConfigInfo config = 1;
 }
 
 message BundleInfo

zbackup_base.cc

@@ -5,6 +5,7 @@
 
 #include "storage_info_file.hh"
 #include "compression.hh"
+#include "debug.hh"
 
 using std::string;
 
@@ -27,6 +28,11 @@ string Paths::getStorageInfoPath()
   return string( Dir::addPath( storageDir, "info" ) );
 }
 
+string Paths::getExtendedStorageInfoPath()
+{
+  return string( Dir::addPath( storageDir, "info_extended" ) );
+}
+
 string Paths::getIndexPath()
 {
   return string( Dir::addPath( storageDir, "index" ) );
@@ -41,9 +47,11 @@ ZBackupBase::ZBackupBase( string const & storageDir, string const & password ):
   Paths( storageDir ), storageInfo( loadStorageInfo() ),
   encryptionkey( password, storageInfo.has_encryption_key() ?
                    &storageInfo.encryption_key() : 0 ),
+  extendedStorageInfo( loadExtendedStorageInfo( encryptionkey ) ),
   tmpMgr( getTmpPath() ),
   chunkIndex( encryptionkey, tmpMgr, getIndexPath(), false )
 {
+  dPrintf("%s repo instantiated and initialized\n", storageDir.c_str() );
 }
 
 ZBackupBase::ZBackupBase( string const & storageDir, string const & password,
@@ -51,9 +59,11 @@ ZBackupBase::ZBackupBase( string const & storageDir, string const & password,
   Paths( storageDir ), storageInfo( loadStorageInfo() ),
   encryptionkey( password, storageInfo.has_encryption_key() ?
                    &storageInfo.encryption_key() : 0 ),
+  extendedStorageInfo( loadExtendedStorageInfo( encryptionkey ) ),
   tmpMgr( getTmpPath() ),
   chunkIndex( encryptionkey, tmpMgr, getIndexPath(), prohibitChunkIndexLoading )
 {
+  dPrintf("%s repo instantiated and initialized\n", storageDir.c_str() );
 }
 
 StorageInfo ZBackupBase::loadStorageInfo()
@@ -65,14 +75,30 @@ StorageInfo ZBackupBase::loadStorageInfo()
   return storageInfo;
 }
 
+ExtendedStorageInfo ZBackupBase::loadExtendedStorageInfo( EncryptionKey const & encryptionkey )
+{
+  ExtendedStorageInfo extendedStorageInfo;
+
+  ExtendedStorageInfoFile::load( getExtendedStorageInfoPath(), encryptionkey, extendedStorageInfo );
+
+  return extendedStorageInfo;
+}
+
 void ZBackupBase::initStorage( string const & storageDir,
                                string const & password,
                                bool isEncrypted )
 {
   StorageInfo storageInfo;
+  ExtendedStorageInfo extendedStorageInfo;
+
   // TODO: make the following configurable
   storageInfo.set_chunk_max_size( 65536 );
   storageInfo.set_bundle_max_payload_size( 0x200000 );
+  extendedStorageInfo.mutable_config()->set_chunk_max_size(
+      extendedStorageInfo.config().chunk_max_size() );
+  extendedStorageInfo.mutable_config()->set_bundle_max_payload_size(
+      extendedStorageInfo.config().bundle_max_payload_size() );
+
   EncryptionKey encryptionkey = EncryptionKey::noKey();
 
   if ( isEncrypted )
@@ -98,11 +124,16 @@ void ZBackupBase::initStorage( string const & storageDir,
     Dir::create( paths.getIndexPath() );
 
   string storageInfoPath( paths.getStorageInfoPath() );
+  string extendedStorageInfoPath( paths.getExtendedStorageInfoPath() );
 
   if ( File::exists( storageInfoPath ) )
     throw exWontOverwrite( storageInfoPath );
 
+  encryptionkey = EncryptionKey( password, storageInfo.has_encryption_key() ?
+      &storageInfo.encryption_key() : 0 );
+
   StorageInfoFile::save( storageInfoPath, storageInfo );
+  ExtendedStorageInfoFile::save( extendedStorageInfoPath, encryptionkey, extendedStorageInfo );
 }
 
 string ZBackupBase::deriveStorageDirFromBackupsFile( string const &
@@ -145,3 +176,7 @@ void ZBackupBase::setPassword( string const & password )
                    &storageInfo.encryption_key() : 0 );
 }
 
+void ZBackupBase::saveExtendedStorageInfo()
+{
+  ExtendedStorageInfoFile::save( getExtendedStorageInfoPath(), encryptionkey, extendedStorageInfo );
+}

zbackup_base.hh

@@ -21,6 +21,7 @@ struct Paths
   std::string getCreatePath();
   std::string getBundlesPath();
   std::string getStorageInfoPath();
+  std::string getExtendedStorageInfoPath();
   std::string getIndexPath();
   std::string getBackupsPath();
 };
@@ -53,15 +54,19 @@ public:
 
   void useDefaultCompressionMethod();
 
+  void saveExtendedStorageInfo();
+
   void setPassword( std::string const & password );
 
   StorageInfo storageInfo;
   EncryptionKey encryptionkey;
+  ExtendedStorageInfo extendedStorageInfo;
   TmpMgr tmpMgr;
   ChunkIndex chunkIndex;
 
 private:
   StorageInfo loadStorageInfo();
+  ExtendedStorageInfo loadExtendedStorageInfo( EncryptionKey const & );
 };