Implement ReconstructSome() to reconstruct only specific data shards (#189)
Co-authored-by: Vitaliy Filippov <vitalif@yourcmc.ru>master
parent
10e7890be7
commit
19a04effc5
11
README.md
11
README.md
|
@ -188,6 +188,17 @@ If you are only interested in the data shards (for reading purposes) you can cal
|
|||
err := enc.ReconstructData(data)
|
||||
```
|
||||
|
||||
If you don't need all data shards you can use `ReconstructSome()`:
|
||||
|
||||
```Go
|
||||
// Delete two data shards
|
||||
data[3] = nil
|
||||
data[7] = nil
|
||||
|
||||
// Reconstruct just the shard 3
|
||||
err := enc.ReconstructSome(data, []bool{false, false, false, true, false, false, false, false})
|
||||
```
|
||||
|
||||
So to sum up reconstruction:
|
||||
* The number of data/parity shards must match the numbers used for encoding.
|
||||
* The order of shards must be the same as used when encoding.
|
||||
|
|
|
@ -77,6 +77,24 @@ type Encoder interface {
|
|||
// calling the Verify function is likely to fail.
|
||||
ReconstructData(shards [][]byte) error
|
||||
|
||||
// ReconstructSome will recreate only requested data shards, if possible.
|
||||
//
|
||||
// Given a list of shards, some of which contain data, fills in the
|
||||
// data shards indicated by true values in the "required" parameter.
|
||||
// The length of "required" array must be equal to DataShards.
|
||||
//
|
||||
// The length of "shards" array must be equal to Shards.
|
||||
// You indicate that a shard is missing by setting it to nil or zero-length.
|
||||
// If a shard is zero-length but has sufficient capacity, that memory will
|
||||
// be used, otherwise a new []byte will be allocated.
|
||||
//
|
||||
// If there are too few shards to reconstruct the missing
|
||||
// ones, ErrTooFewShards will be returned.
|
||||
//
|
||||
// As the reconstructed shard set may contain missing parity shards,
|
||||
// calling the Verify function is likely to fail.
|
||||
ReconstructSome(shards [][]byte, required []bool) error
|
||||
|
||||
// Update parity is use for change a few data shards and update it's parity.
|
||||
// Input 'newDatashards' containing data shards changed.
|
||||
// Input 'shards' containing old data shards (if data shard not changed, it can be nil) and old parity shards.
|
||||
|
@ -995,7 +1013,7 @@ func shardSize(shards [][]byte) int {
|
|||
// The reconstructed shard set is complete, but integrity is not verified.
|
||||
// Use the Verify function to check if data set is ok.
|
||||
func (r *reedSolomon) Reconstruct(shards [][]byte) error {
|
||||
return r.reconstruct(shards, false)
|
||||
return r.reconstruct(shards, false, nil)
|
||||
}
|
||||
|
||||
// ReconstructData will recreate any missing data shards, if possible.
|
||||
|
@ -1014,19 +1032,39 @@ func (r *reedSolomon) Reconstruct(shards [][]byte) error {
|
|||
// As the reconstructed shard set may contain missing parity shards,
|
||||
// calling the Verify function is likely to fail.
|
||||
func (r *reedSolomon) ReconstructData(shards [][]byte) error {
|
||||
return r.reconstruct(shards, true)
|
||||
return r.reconstruct(shards, true, nil)
|
||||
}
|
||||
|
||||
// ReconstructSome will recreate only requested data shards, if possible.
|
||||
//
|
||||
// Given a list of shards, some of which contain data, fills in the
|
||||
// data shards indicated by true values in the "required" parameter.
|
||||
// The length of "required" array must be equal to DataShards.
|
||||
//
|
||||
// The length of "shards" array must be equal to Shards.
|
||||
// You indicate that a shard is missing by setting it to nil or zero-length.
|
||||
// If a shard is zero-length but has sufficient capacity, that memory will
|
||||
// be used, otherwise a new []byte will be allocated.
|
||||
//
|
||||
// If there are too few shards to reconstruct the missing
|
||||
// ones, ErrTooFewShards will be returned.
|
||||
//
|
||||
// As the reconstructed shard set may contain missing parity shards,
|
||||
// calling the Verify function is likely to fail.
|
||||
func (r *reedSolomon) ReconstructSome(shards [][]byte, required []bool) error {
|
||||
return r.reconstruct(shards, true, required)
|
||||
}
|
||||
|
||||
// reconstruct will recreate the missing data shards, and unless
|
||||
// dataOnly is true, also the missing parity shards
|
||||
//
|
||||
// The length of the array must be equal to Shards.
|
||||
// The length of "shards" array must be equal to Shards.
|
||||
// You indicate that a shard is missing by setting it to nil.
|
||||
//
|
||||
// If there are too few shards to reconstruct the missing
|
||||
// ones, ErrTooFewShards will be returned.
|
||||
func (r *reedSolomon) reconstruct(shards [][]byte, dataOnly bool) error {
|
||||
if len(shards) != r.Shards {
|
||||
func (r *reedSolomon) reconstruct(shards [][]byte, dataOnly bool, required []bool) error {
|
||||
if len(shards) != r.Shards || required != nil && len(required) < r.DataShards {
|
||||
return ErrTooFewShards
|
||||
}
|
||||
// Check arguments.
|
||||
|
@ -1041,15 +1079,19 @@ func (r *reedSolomon) reconstruct(shards [][]byte, dataOnly bool) error {
|
|||
// nothing to do.
|
||||
numberPresent := 0
|
||||
dataPresent := 0
|
||||
missingRequired := 0
|
||||
for i := 0; i < r.Shards; i++ {
|
||||
if len(shards[i]) != 0 {
|
||||
numberPresent++
|
||||
if i < r.DataShards {
|
||||
dataPresent++
|
||||
}
|
||||
} else if required != nil && required[i] {
|
||||
missingRequired++
|
||||
}
|
||||
}
|
||||
if numberPresent == r.Shards || dataOnly && dataPresent == r.DataShards {
|
||||
if numberPresent == r.Shards || dataOnly && dataPresent == r.DataShards ||
|
||||
required != nil && missingRequired == 0 {
|
||||
// Cool. All of the shards data data. We don't
|
||||
// need to do anything.
|
||||
return nil
|
||||
|
@ -1127,7 +1169,7 @@ func (r *reedSolomon) reconstruct(shards [][]byte, dataOnly bool) error {
|
|||
outputCount := 0
|
||||
|
||||
for iShard := 0; iShard < r.DataShards; iShard++ {
|
||||
if len(shards[iShard]) == 0 {
|
||||
if len(shards[iShard]) == 0 && (required == nil || required[iShard]) {
|
||||
if cap(shards[iShard]) >= shardSize {
|
||||
shards[iShard] = shards[iShard][0:shardSize]
|
||||
} else {
|
||||
|
@ -1153,7 +1195,7 @@ func (r *reedSolomon) reconstruct(shards [][]byte, dataOnly bool) error {
|
|||
// data shards were missing.
|
||||
outputCount = 0
|
||||
for iShard := r.DataShards; iShard < r.Shards; iShard++ {
|
||||
if len(shards[iShard]) == 0 {
|
||||
if len(shards[iShard]) == 0 && (required == nil || required[iShard]) {
|
||||
if cap(shards[iShard]) >= shardSize {
|
||||
shards[iShard] = shards[iShard][0:shardSize]
|
||||
} else {
|
||||
|
|
|
@ -686,6 +686,32 @@ func testReconstructData(t *testing.T, o ...Option) {
|
|||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Reconstruct 3 shards with 3 data and 5 parity shards
|
||||
shardsCopy := make([][]byte, 13)
|
||||
copy(shardsCopy, shards)
|
||||
shardsCopy[2] = nil
|
||||
shardsCopy[3] = nil
|
||||
shardsCopy[4] = nil
|
||||
shardsCopy[5] = nil
|
||||
shardsCopy[6] = nil
|
||||
|
||||
shardsRequired := make([]bool, 8)
|
||||
shardsRequired[3] = true
|
||||
shardsRequired[4] = true
|
||||
err = r.ReconstructSome(shardsCopy, shardsRequired)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if 0 != bytes.Compare(shardsCopy[3], shards[3]) ||
|
||||
0 != bytes.Compare(shardsCopy[4], shards[4]) {
|
||||
t.Fatal("ReconstructSome did not reconstruct required shards correctly")
|
||||
}
|
||||
|
||||
if shardsCopy[2] != nil || shardsCopy[5] != nil || shardsCopy[6] != nil {
|
||||
t.Fatal("ReconstructSome reconstructed extra shards")
|
||||
}
|
||||
|
||||
// Reconstruct with 10 shards present. Use pre-allocated memory for one of them.
|
||||
shards[0] = nil
|
||||
shards[2] = nil
|
||||
|
|
Loading…
Reference in New Issue