Implement ReconstructSome() to reconstruct only specific data shards (#189)
Co-authored-by: Vitaliy Filippov <vitalif@yourcmc.ru>master
parent
10e7890be7
commit
19a04effc5
11
README.md
11
README.md
|
@ -188,6 +188,17 @@ If you are only interested in the data shards (for reading purposes) you can cal
|
||||||
err := enc.ReconstructData(data)
|
err := enc.ReconstructData(data)
|
||||||
```
|
```
|
||||||
|
|
||||||
|
If you don't need all data shards you can use `ReconstructSome()`:
|
||||||
|
|
||||||
|
```Go
|
||||||
|
// Delete two data shards
|
||||||
|
data[3] = nil
|
||||||
|
data[7] = nil
|
||||||
|
|
||||||
|
// Reconstruct just the shard 3
|
||||||
|
err := enc.ReconstructSome(data, []bool{false, false, false, true, false, false, false, false})
|
||||||
|
```
|
||||||
|
|
||||||
So to sum up reconstruction:
|
So to sum up reconstruction:
|
||||||
* The number of data/parity shards must match the numbers used for encoding.
|
* The number of data/parity shards must match the numbers used for encoding.
|
||||||
* The order of shards must be the same as used when encoding.
|
* The order of shards must be the same as used when encoding.
|
||||||
|
|
|
@ -77,6 +77,24 @@ type Encoder interface {
|
||||||
// calling the Verify function is likely to fail.
|
// calling the Verify function is likely to fail.
|
||||||
ReconstructData(shards [][]byte) error
|
ReconstructData(shards [][]byte) error
|
||||||
|
|
||||||
|
// ReconstructSome will recreate only requested data shards, if possible.
|
||||||
|
//
|
||||||
|
// Given a list of shards, some of which contain data, fills in the
|
||||||
|
// data shards indicated by true values in the "required" parameter.
|
||||||
|
// The length of "required" array must be equal to DataShards.
|
||||||
|
//
|
||||||
|
// The length of "shards" array must be equal to Shards.
|
||||||
|
// You indicate that a shard is missing by setting it to nil or zero-length.
|
||||||
|
// If a shard is zero-length but has sufficient capacity, that memory will
|
||||||
|
// be used, otherwise a new []byte will be allocated.
|
||||||
|
//
|
||||||
|
// If there are too few shards to reconstruct the missing
|
||||||
|
// ones, ErrTooFewShards will be returned.
|
||||||
|
//
|
||||||
|
// As the reconstructed shard set may contain missing parity shards,
|
||||||
|
// calling the Verify function is likely to fail.
|
||||||
|
ReconstructSome(shards [][]byte, required []bool) error
|
||||||
|
|
||||||
// Update parity is use for change a few data shards and update it's parity.
|
// Update parity is use for change a few data shards and update it's parity.
|
||||||
// Input 'newDatashards' containing data shards changed.
|
// Input 'newDatashards' containing data shards changed.
|
||||||
// Input 'shards' containing old data shards (if data shard not changed, it can be nil) and old parity shards.
|
// Input 'shards' containing old data shards (if data shard not changed, it can be nil) and old parity shards.
|
||||||
|
@ -995,7 +1013,7 @@ func shardSize(shards [][]byte) int {
|
||||||
// The reconstructed shard set is complete, but integrity is not verified.
|
// The reconstructed shard set is complete, but integrity is not verified.
|
||||||
// Use the Verify function to check if data set is ok.
|
// Use the Verify function to check if data set is ok.
|
||||||
func (r *reedSolomon) Reconstruct(shards [][]byte) error {
|
func (r *reedSolomon) Reconstruct(shards [][]byte) error {
|
||||||
return r.reconstruct(shards, false)
|
return r.reconstruct(shards, false, nil)
|
||||||
}
|
}
|
||||||
|
|
||||||
// ReconstructData will recreate any missing data shards, if possible.
|
// ReconstructData will recreate any missing data shards, if possible.
|
||||||
|
@ -1014,19 +1032,39 @@ func (r *reedSolomon) Reconstruct(shards [][]byte) error {
|
||||||
// As the reconstructed shard set may contain missing parity shards,
|
// As the reconstructed shard set may contain missing parity shards,
|
||||||
// calling the Verify function is likely to fail.
|
// calling the Verify function is likely to fail.
|
||||||
func (r *reedSolomon) ReconstructData(shards [][]byte) error {
|
func (r *reedSolomon) ReconstructData(shards [][]byte) error {
|
||||||
return r.reconstruct(shards, true)
|
return r.reconstruct(shards, true, nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ReconstructSome will recreate only requested data shards, if possible.
|
||||||
|
//
|
||||||
|
// Given a list of shards, some of which contain data, fills in the
|
||||||
|
// data shards indicated by true values in the "required" parameter.
|
||||||
|
// The length of "required" array must be equal to DataShards.
|
||||||
|
//
|
||||||
|
// The length of "shards" array must be equal to Shards.
|
||||||
|
// You indicate that a shard is missing by setting it to nil or zero-length.
|
||||||
|
// If a shard is zero-length but has sufficient capacity, that memory will
|
||||||
|
// be used, otherwise a new []byte will be allocated.
|
||||||
|
//
|
||||||
|
// If there are too few shards to reconstruct the missing
|
||||||
|
// ones, ErrTooFewShards will be returned.
|
||||||
|
//
|
||||||
|
// As the reconstructed shard set may contain missing parity shards,
|
||||||
|
// calling the Verify function is likely to fail.
|
||||||
|
func (r *reedSolomon) ReconstructSome(shards [][]byte, required []bool) error {
|
||||||
|
return r.reconstruct(shards, true, required)
|
||||||
}
|
}
|
||||||
|
|
||||||
// reconstruct will recreate the missing data shards, and unless
|
// reconstruct will recreate the missing data shards, and unless
|
||||||
// dataOnly is true, also the missing parity shards
|
// dataOnly is true, also the missing parity shards
|
||||||
//
|
//
|
||||||
// The length of the array must be equal to Shards.
|
// The length of "shards" array must be equal to Shards.
|
||||||
// You indicate that a shard is missing by setting it to nil.
|
// You indicate that a shard is missing by setting it to nil.
|
||||||
//
|
//
|
||||||
// If there are too few shards to reconstruct the missing
|
// If there are too few shards to reconstruct the missing
|
||||||
// ones, ErrTooFewShards will be returned.
|
// ones, ErrTooFewShards will be returned.
|
||||||
func (r *reedSolomon) reconstruct(shards [][]byte, dataOnly bool) error {
|
func (r *reedSolomon) reconstruct(shards [][]byte, dataOnly bool, required []bool) error {
|
||||||
if len(shards) != r.Shards {
|
if len(shards) != r.Shards || required != nil && len(required) < r.DataShards {
|
||||||
return ErrTooFewShards
|
return ErrTooFewShards
|
||||||
}
|
}
|
||||||
// Check arguments.
|
// Check arguments.
|
||||||
|
@ -1041,15 +1079,19 @@ func (r *reedSolomon) reconstruct(shards [][]byte, dataOnly bool) error {
|
||||||
// nothing to do.
|
// nothing to do.
|
||||||
numberPresent := 0
|
numberPresent := 0
|
||||||
dataPresent := 0
|
dataPresent := 0
|
||||||
|
missingRequired := 0
|
||||||
for i := 0; i < r.Shards; i++ {
|
for i := 0; i < r.Shards; i++ {
|
||||||
if len(shards[i]) != 0 {
|
if len(shards[i]) != 0 {
|
||||||
numberPresent++
|
numberPresent++
|
||||||
if i < r.DataShards {
|
if i < r.DataShards {
|
||||||
dataPresent++
|
dataPresent++
|
||||||
}
|
}
|
||||||
|
} else if required != nil && required[i] {
|
||||||
|
missingRequired++
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if numberPresent == r.Shards || dataOnly && dataPresent == r.DataShards {
|
if numberPresent == r.Shards || dataOnly && dataPresent == r.DataShards ||
|
||||||
|
required != nil && missingRequired == 0 {
|
||||||
// Cool. All of the shards data data. We don't
|
// Cool. All of the shards data data. We don't
|
||||||
// need to do anything.
|
// need to do anything.
|
||||||
return nil
|
return nil
|
||||||
|
@ -1127,7 +1169,7 @@ func (r *reedSolomon) reconstruct(shards [][]byte, dataOnly bool) error {
|
||||||
outputCount := 0
|
outputCount := 0
|
||||||
|
|
||||||
for iShard := 0; iShard < r.DataShards; iShard++ {
|
for iShard := 0; iShard < r.DataShards; iShard++ {
|
||||||
if len(shards[iShard]) == 0 {
|
if len(shards[iShard]) == 0 && (required == nil || required[iShard]) {
|
||||||
if cap(shards[iShard]) >= shardSize {
|
if cap(shards[iShard]) >= shardSize {
|
||||||
shards[iShard] = shards[iShard][0:shardSize]
|
shards[iShard] = shards[iShard][0:shardSize]
|
||||||
} else {
|
} else {
|
||||||
|
@ -1153,7 +1195,7 @@ func (r *reedSolomon) reconstruct(shards [][]byte, dataOnly bool) error {
|
||||||
// data shards were missing.
|
// data shards were missing.
|
||||||
outputCount = 0
|
outputCount = 0
|
||||||
for iShard := r.DataShards; iShard < r.Shards; iShard++ {
|
for iShard := r.DataShards; iShard < r.Shards; iShard++ {
|
||||||
if len(shards[iShard]) == 0 {
|
if len(shards[iShard]) == 0 && (required == nil || required[iShard]) {
|
||||||
if cap(shards[iShard]) >= shardSize {
|
if cap(shards[iShard]) >= shardSize {
|
||||||
shards[iShard] = shards[iShard][0:shardSize]
|
shards[iShard] = shards[iShard][0:shardSize]
|
||||||
} else {
|
} else {
|
||||||
|
|
|
@ -686,6 +686,32 @@ func testReconstructData(t *testing.T, o ...Option) {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Reconstruct 3 shards with 3 data and 5 parity shards
|
||||||
|
shardsCopy := make([][]byte, 13)
|
||||||
|
copy(shardsCopy, shards)
|
||||||
|
shardsCopy[2] = nil
|
||||||
|
shardsCopy[3] = nil
|
||||||
|
shardsCopy[4] = nil
|
||||||
|
shardsCopy[5] = nil
|
||||||
|
shardsCopy[6] = nil
|
||||||
|
|
||||||
|
shardsRequired := make([]bool, 8)
|
||||||
|
shardsRequired[3] = true
|
||||||
|
shardsRequired[4] = true
|
||||||
|
err = r.ReconstructSome(shardsCopy, shardsRequired)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if 0 != bytes.Compare(shardsCopy[3], shards[3]) ||
|
||||||
|
0 != bytes.Compare(shardsCopy[4], shards[4]) {
|
||||||
|
t.Fatal("ReconstructSome did not reconstruct required shards correctly")
|
||||||
|
}
|
||||||
|
|
||||||
|
if shardsCopy[2] != nil || shardsCopy[5] != nil || shardsCopy[6] != nil {
|
||||||
|
t.Fatal("ReconstructSome reconstructed extra shards")
|
||||||
|
}
|
||||||
|
|
||||||
// Reconstruct with 10 shards present. Use pre-allocated memory for one of them.
|
// Reconstruct with 10 shards present. Use pre-allocated memory for one of them.
|
||||||
shards[0] = nil
|
shards[0] = nil
|
||||||
shards[2] = nil
|
shards[2] = nil
|
||||||
|
|
Loading…
Reference in New Issue