reedsolomon-go/galois_amd64.go

//go:build !noasm && !appengine && !gccgo
// +build !noasm,!appengine,!gccgo

// Copyright 2015, Klaus Post, see LICENSE for details.

package reedsolomon

//go:noescape
func galMulSSSE3(low, high, in, out []byte)

//go:noescape
func galMulSSSE3Xor(low, high, in, out []byte)

//go:noescape
func galMulAVX2Xor(low, high, in, out []byte)

//go:noescape
func galMulAVX2(low, high, in, out []byte)

//go:noescape
func sSE2XorSlice(in, out []byte)

//go:noescape
func galMulAVX2Xor_64(low, high, in, out []byte)

//go:noescape
func galMulAVX2_64(low, high, in, out []byte)

//go:noescape
func sSE2XorSlice_64(in, out []byte)

// This is what the assembler routines do in blocks of 16 bytes:
/*
func galMulSSSE3(low, high, in, out []byte) {
	for n, input := range in {
		l := input & 0xf
		h := input >> 4
		out[n] = low[l] ^ high[h]
	}
}

func galMulSSSE3Xor(low, high, in, out []byte) {
	for n, input := range in {
		l := input & 0xf
		h := input >> 4
		out[n] ^= low[l] ^ high[h]
	}
}
*/

// bigSwitchover is the size where 64 bytes are processed per loop.
const bigSwitchover = 128

func galMulSlice(c byte, in, out []byte, o *options) {
	if c == 1 {
		copy(out, in)
		return
	}
	if o.useAVX2 {
		if len(in) >= bigSwitchover {
			galMulAVX2_64(mulTableLow[c][:], mulTableHigh[c][:], in, out)
			done := (len(in) >> 6) << 6
			in = in[done:]
			out = out[done:]
		}
		if len(in) > 32 {
			galMulAVX2(mulTableLow[c][:], mulTableHigh[c][:], in, out)
			done := (len(in) >> 5) << 5
			in = in[done:]
			out = out[done:]
		}
	} else if o.useSSSE3 {
		galMulSSSE3(mulTableLow[c][:], mulTableHigh[c][:], in, out)
		done := (len(in) >> 4) << 4
		in = in[done:]
		out = out[done:]
	}
	out = out[:len(in)]
	mt := mulTable[c][:256]
	for i := range in {
		out[i] = mt[in[i]]
	}
}

func galMulSliceXor(c byte, in, out []byte, o *options) {
	if c == 1 {
		sliceXor(in, out, o)
		return
	}

	if o.useAVX2 {
		if len(in) >= bigSwitchover {
			galMulAVX2Xor_64(mulTableLow[c][:], mulTableHigh[c][:], in, out)
			done := (len(in) >> 6) << 6
			in = in[done:]
			out = out[done:]
		}
		if len(in) >= 32 {
			galMulAVX2Xor(mulTableLow[c][:], mulTableHigh[c][:], in, out)
			done := (len(in) >> 5) << 5
			in = in[done:]
			out = out[done:]
		}
	} else if o.useSSSE3 {
		galMulSSSE3Xor(mulTableLow[c][:], mulTableHigh[c][:], in, out)
		done := (len(in) >> 4) << 4
		in = in[done:]
		out = out[done:]
	}
	if len(in) == 0 {
		return
	}
	out = out[:len(in)]
	mt := mulTable[c][:256]
	for i := range in {
		out[i] ^= mt[in[i]]
	}
}

// simple slice xor
func sliceXor(in, out []byte, o *options) {
	if o.useSSE2 {
		if len(in) >= bigSwitchover {
			sSE2XorSlice_64(in, out)
			done := (len(in) >> 6) << 6
			in = in[done:]
			out = out[done:]
		}
		if len(in) >= 16 {
			sSE2XorSlice(in, out)
			done := (len(in) >> 4) << 4
			in = in[done:]
			out = out[done:]
		}
	} else {
		sliceXorGo(in, out, o)
		return
	}
	out = out[:len(in)]
	for i := range in {
		out[i] ^= in[i]
	}
}

// 4-way butterfly
func ifftDIT4(work [][]byte, dist int, log_m01, log_m23, log_m02 ffe, o *options) {
	if o.useAVX2 || o.useAVX512 {
		if len(work[0]) > 0 {
			var mask uint8
			if log_m01 == modulus {
				mask |= 1 << 0
			}
			if log_m23 == modulus {
				mask |= 1 << 1
			}
			if log_m02 == modulus {
				mask |= 1 << 2
			}
			t01 := &multiply256LUT[log_m01]
			t23 := &multiply256LUT[log_m23]
			t02 := &multiply256LUT[log_m02]
			if o.useAVX512 {
				ifftDIT4_avx512(work, dist*24, t01, t23, t02, mask)
			} else {
				ifftDIT4_avx2(work, dist*24, t01, t23, t02, mask)
			}
		}
		return
	}
	ifftDIT4Ref(work, dist, log_m01, log_m23, log_m02, o)
}

func fftDIT4(work [][]byte, dist int, log_m01, log_m23, log_m02 ffe, o *options) {
	if o.useAVX2 || o.useAVX512 {
		if len(work[0]) > 0 {
			var mask uint8
			if log_m02 == modulus {
				mask |= 1 << 0
			}
			if log_m01 == modulus {
				mask |= 1 << 1
			}
			if log_m23 == modulus {
				mask |= 1 << 2
			}
			t01 := &multiply256LUT[log_m01]
			t23 := &multiply256LUT[log_m23]
			t02 := &multiply256LUT[log_m02]
			if o.useAVX512 {
				fftDIT4_avx512(work, dist*24, t01, t23, t02, mask)
			} else {
				fftDIT4_avx2(work, dist*24, t01, t23, t02, mask)
			}
		}
		return
	}
	fftDIT4Ref(work, dist, log_m01, log_m23, log_m02, o)
}

// 2-way butterfly forward
func fftDIT2(x, y []byte, log_m ffe, o *options) {
	if o.useAVX2 {
		if len(x) > 0 {
			tmp := &multiply256LUT[log_m]
			fftDIT2_avx2(x, y, tmp)
		}
	} else if o.useSSSE3 {
		if len(x) > 0 {
			tmp := &multiply256LUT[log_m]
			fftDIT2_ssse3(x, y, tmp)
		}
	} else {
		// Reference version:
		refMulAdd(x, y, log_m)
		sliceXor(x, y, o)
	}
}

// 2-way butterfly
func ifftDIT2(x, y []byte, log_m ffe, o *options) {
	if o.useAVX2 {
		if len(x) > 0 {
			tmp := &multiply256LUT[log_m]
			ifftDIT2_avx2(x, y, tmp)
		}
	} else if o.useSSSE3 {
		if len(x) > 0 {
			tmp := &multiply256LUT[log_m]
			ifftDIT2_ssse3(x, y, tmp)
		}
	} else {
		// Reference version:
		sliceXor(x, y, o)
		refMulAdd(x, y, log_m)
	}
}

func mulgf16(x, y []byte, log_m ffe, o *options) {
	if o.useAVX2 {
		if len(x) > 0 {
			tmp := &multiply256LUT[log_m]
			mulgf16_avx2(x, y, tmp)
		}
	} else if o.useSSSE3 {
		if len(x) > 0 {
			tmp := &multiply256LUT[log_m]
			mulgf16_ssse3(x, y, tmp)
		}
	} else {
		refMul(x, y, log_m)
	}
}