Implement jerasure algorithm of matrix generation for interoperability
parent
7b7dbe6919
commit
b933ef1add
17
options.go
17
options.go
|
@ -16,6 +16,7 @@ type options struct {
|
||||||
perRound int
|
perRound int
|
||||||
|
|
||||||
useAVX512, useAVX2, useSSSE3, useSSE2 bool
|
useAVX512, useAVX2, useSSSE3, useSSE2 bool
|
||||||
|
useJerasureMatrix bool
|
||||||
usePAR1Matrix bool
|
usePAR1Matrix bool
|
||||||
useCauchy bool
|
useCauchy bool
|
||||||
fastOneParity bool
|
fastOneParity bool
|
||||||
|
@ -163,12 +164,25 @@ func WithAVX512(enabled bool) Option {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// WithJerasureMatrix causes the encoder to build the Reed-Solomon-Vandermonde
|
||||||
|
// matrix in the same way as done by the Jerasure library.
|
||||||
|
// The first row and column of the coding matrix only contains 1's in this method
|
||||||
|
// so the first parity chunk is always equal to XOR of all data chunks.
|
||||||
|
func WithJerasureMatrix() Option {
|
||||||
|
return func(o *options) {
|
||||||
|
o.useJerasureMatrix = true
|
||||||
|
o.usePAR1Matrix = false
|
||||||
|
o.useCauchy = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// WithPAR1Matrix causes the encoder to build the matrix how PARv1
|
// WithPAR1Matrix causes the encoder to build the matrix how PARv1
|
||||||
// does. Note that the method they use is buggy, and may lead to cases
|
// does. Note that the method they use is buggy, and may lead to cases
|
||||||
// where recovery is impossible, even if there are enough parity
|
// where recovery is impossible, even if there are enough parity
|
||||||
// shards.
|
// shards.
|
||||||
func WithPAR1Matrix() Option {
|
func WithPAR1Matrix() Option {
|
||||||
return func(o *options) {
|
return func(o *options) {
|
||||||
|
o.useJerasureMatrix = false
|
||||||
o.usePAR1Matrix = true
|
o.usePAR1Matrix = true
|
||||||
o.useCauchy = false
|
o.useCauchy = false
|
||||||
}
|
}
|
||||||
|
@ -180,8 +194,9 @@ func WithPAR1Matrix() Option {
|
||||||
// but will result in slightly faster start-up time.
|
// but will result in slightly faster start-up time.
|
||||||
func WithCauchyMatrix() Option {
|
func WithCauchyMatrix() Option {
|
||||||
return func(o *options) {
|
return func(o *options) {
|
||||||
o.useCauchy = true
|
o.useJerasureMatrix = false
|
||||||
o.usePAR1Matrix = false
|
o.usePAR1Matrix = false
|
||||||
|
o.useCauchy = true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -191,6 +191,87 @@ func buildMatrix(dataShards, totalShards int) (matrix, error) {
|
||||||
return vm.Multiply(topInv)
|
return vm.Multiply(topInv)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// buildMatrixJerasure creates the same encoding matrix as Jerasure library
|
||||||
|
//
|
||||||
|
// The top square of the matrix is guaranteed to be an identity
|
||||||
|
// matrix, which means that the data shards are unchanged after
|
||||||
|
// encoding.
|
||||||
|
func buildMatrixJerasure(dataShards, totalShards int) (matrix, error) {
|
||||||
|
// Start with a Vandermonde matrix. This matrix would work,
|
||||||
|
// in theory, but doesn't have the property that the data
|
||||||
|
// shards are unchanged after encoding.
|
||||||
|
vm, err := vandermonde(totalShards, dataShards)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Jerasure does this:
|
||||||
|
// first row is always 100..00
|
||||||
|
vm[0][0] = 1
|
||||||
|
for i := 1; i < dataShards; i++ {
|
||||||
|
vm[0][i] = 0
|
||||||
|
}
|
||||||
|
// last row is always 000..01
|
||||||
|
for i := 0; i < dataShards-1; i++ {
|
||||||
|
vm[totalShards-1][i] = 0
|
||||||
|
}
|
||||||
|
vm[totalShards-1][dataShards-1] = 1
|
||||||
|
|
||||||
|
for i := 0; i < dataShards; i++ {
|
||||||
|
// Find the row where i'th col is not 0
|
||||||
|
r := i
|
||||||
|
for ; r < totalShards && vm[r][i] == 0; r++ {
|
||||||
|
}
|
||||||
|
if r != i {
|
||||||
|
// Swap it with i'th row if not already
|
||||||
|
t := vm[r]
|
||||||
|
vm[r] = vm[i]
|
||||||
|
vm[i] = t
|
||||||
|
}
|
||||||
|
// Multiply by the inverted matrix (same as vm.Multiply(vm[0:dataShards].Invert()))
|
||||||
|
if vm[i][i] != 1 {
|
||||||
|
// Make vm[i][i] = 1 by dividing the column by vm[i][i]
|
||||||
|
tmp := galDivide(1, vm[i][i])
|
||||||
|
for j := 0; j < totalShards; j++ {
|
||||||
|
vm[j][i] = galMultiply(vm[j][i], tmp)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for j := 0; j < dataShards; j++ {
|
||||||
|
// Make vm[i][j] = 0 where j != i by adding vm[i][j]*vm[.][i] to each column
|
||||||
|
tmp := vm[i][j]
|
||||||
|
if j != i && tmp != 0 {
|
||||||
|
for r := 0; r < totalShards; r++ {
|
||||||
|
vm[r][j] = galAdd(vm[r][j], galMultiply(tmp, vm[r][i]))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Make vm[dataShards] row all ones - divide each column j by vm[dataShards][j]
|
||||||
|
for j := 0; j < dataShards; j++ {
|
||||||
|
tmp := vm[dataShards][j]
|
||||||
|
if tmp != 1 {
|
||||||
|
tmp = galDivide(1, tmp)
|
||||||
|
for i := dataShards; i < totalShards; i++ {
|
||||||
|
vm[i][j] = galMultiply(vm[i][j], tmp)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Make vm[dataShards...totalShards-1][0] column all ones - divide each row
|
||||||
|
for i := dataShards + 1; i < totalShards; i++ {
|
||||||
|
tmp := vm[i][0]
|
||||||
|
if tmp != 1 {
|
||||||
|
tmp = galDivide(1, tmp)
|
||||||
|
for j := 0; j < dataShards; j++ {
|
||||||
|
vm[i][j] = galMultiply(vm[i][j], tmp)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return vm, nil
|
||||||
|
}
|
||||||
|
|
||||||
// buildMatrixPAR1 creates the matrix to use for encoding according to
|
// buildMatrixPAR1 creates the matrix to use for encoding according to
|
||||||
// the PARv1 spec, given the number of data shards and the number of
|
// the PARv1 spec, given the number of data shards and the number of
|
||||||
// total shards. Note that the method they use is buggy, and may lead
|
// total shards. Note that the method they use is buggy, and may lead
|
||||||
|
@ -323,6 +404,8 @@ func New(dataShards, parityShards int, opts ...Option) (Encoder, error) {
|
||||||
r.m, err = buildMatrixCauchy(dataShards, r.Shards)
|
r.m, err = buildMatrixCauchy(dataShards, r.Shards)
|
||||||
case r.o.usePAR1Matrix:
|
case r.o.usePAR1Matrix:
|
||||||
r.m, err = buildMatrixPAR1(dataShards, r.Shards)
|
r.m, err = buildMatrixPAR1(dataShards, r.Shards)
|
||||||
|
case r.o.useJerasureMatrix:
|
||||||
|
r.m, err = buildMatrixJerasure(dataShards, r.Shards)
|
||||||
default:
|
default:
|
||||||
r.m, err = buildMatrix(dataShards, r.Shards)
|
r.m, err = buildMatrix(dataShards, r.Shards)
|
||||||
}
|
}
|
||||||
|
|
|
@ -110,6 +110,35 @@ func findSingularSubMatrix(m matrix) (matrix, error) {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestBuildMatrixJerasure(t *testing.T) {
|
||||||
|
totalShards := 12
|
||||||
|
dataShards := 8
|
||||||
|
m, err := buildMatrixJerasure(dataShards, totalShards)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
refMatrix := matrix{
|
||||||
|
{1, 1, 1, 1, 1, 1, 1, 1},
|
||||||
|
{1, 55, 39, 73, 84, 181, 225, 217},
|
||||||
|
{1, 39, 217, 161, 92, 60, 172, 90},
|
||||||
|
{1, 172, 70, 235, 143, 34, 200, 101},
|
||||||
|
}
|
||||||
|
for i := 0; i < 8; i++ {
|
||||||
|
for j := 0; j < 8; j++ {
|
||||||
|
if i != j && m[i][j] != 0 || i == j && m[i][j] != 1 {
|
||||||
|
t.Fatal("Top part of the matrix is not identity")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for i := 0; i < 4; i++ {
|
||||||
|
for j := 0; j < 8; j++ {
|
||||||
|
if m[8+i][j] != refMatrix[i][j] {
|
||||||
|
t.Fatal("Coding matrix for EC 8+4 differs from Jerasure")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestBuildMatrixPAR1Singular(t *testing.T) {
|
func TestBuildMatrixPAR1Singular(t *testing.T) {
|
||||||
totalShards := 8
|
totalShards := 8
|
||||||
dataShards := 4
|
dataShards := 4
|
||||||
|
|
Loading…
Reference in New Issue