vitalif
/
jerasure
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge commit 'de1739cc8483696506829b52e7fda4f6bb195e6a' into debian/pike

master
Ondřej Nový 2017-08-04 03:46:26 +02:00
parent e1e6a02efd de1739cc84
commit 7ecb6f3c3b
15 changed files with 127 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
Examples/.gitignore vendored
View File

@ -5,4 +5,5 @@
/liberation_[0-9][0-9]
/reed_sol_[0-9][0-9]
/reed_sol_test_gf
/reed_sol_time_gf
/reed_sol_time_gf
/test_galois

7
Examples/Makefile.am
View File

@ -25,10 +25,15 @@ bin_PROGRAMS = jerasure_01 \
encoder \
decoder
TESTS=test_all_gfs.sh
check_PROGRAMS =
TESTS=test_all_gfs.sh encode_decode.sh $(check_PROGRAMS)
dist_noinst_SCRIPTS = test_all_gfs.sh time_all_gfs_argv_init.sh
test_galois_SOURCES = test_galois.c
check_PROGRAMS += test_galois
jerasure_01_SOURCES = jerasure_01.c
jerasure_02_SOURCES = jerasure_02.c
jerasure_03_SOURCES = jerasure_03.c

16
Examples/decoder.c
View File

@ -62,6 +62,7 @@ same arguments, and encoder.c does error check.
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include <assert.h>
#include <unistd.h>
#include <sys/time.h>
#include <sys/stat.h>
@ -104,12 +105,11 @@ int main (int argc, char **argv) {
char *c_tech;
int i, j; // loop control variable, s
int blocksize; // size of individual files
int blocksize = 0; // size of individual files
int origsize; // size of file before padding
int total; // used to write data, not padding to file
struct stat status; // used to find size of individual files
int numerased; // number of erased files
int dummy;
/* Used to recreate file names */
char *temp;
@ -138,8 +138,8 @@ int main (int argc, char **argv) {
fprintf(stderr, "usage: inputfile\n");
exit(0);
}
curdir = (char *)malloc(sizeof(char)*100);
getcwd(curdir, 100);
curdir = (char *)malloc(sizeof(char)*1000);
assert(curdir == getcwd(curdir, 1000));
/* Begin recreation of file names */
cs1 = (char*)malloc(sizeof(char)*strlen(argv[1]));
@ -270,11 +270,11 @@ int main (int argc, char **argv) {
stat(fname, &status);
blocksize = status.st_size;
data[i-1] = (char *)malloc(sizeof(char)*blocksize);
dummy = fread(data[i-1], sizeof(char), blocksize, fp);
assert(blocksize == fread(data[i-1], sizeof(char), blocksize, fp));
}
else {
fseek(fp, blocksize*(n-1), SEEK_SET);
dummy = fread(data[i-1], sizeof(char), buffersize/k, fp);
assert(buffersize/k == fread(data[i-1], sizeof(char), buffersize/k, fp));
}
fclose(fp);
}
@ -293,11 +293,11 @@ int main (int argc, char **argv) {
stat(fname, &status);
blocksize = status.st_size;
coding[i-1] = (char *)malloc(sizeof(char)*blocksize);
dummy = fread(coding[i-1], sizeof(char), blocksize, fp);
assert(blocksize == fread(coding[i-1], sizeof(char), blocksize, fp));
}
else {
fseek(fp, blocksize*(n-1), SEEK_SET);
dummy = fread(coding[i-1], sizeof(char), blocksize, fp);
assert(blocksize == fread(coding[i-1], sizeof(char), blocksize, fp));
}
fclose(fp);
}

21
Examples/encode_decode.sh Executable file
View File

@ -0,0 +1,21 @@
#!/bin/bash -e
#
# Copyright (C) 2014 Red Hat <contact@redhat.com>
#
# Author: Loic Dachary <loic@dachary.org>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Library Public License as published by
# the Free Software Foundation; either version 2, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library Public License for more details.
#
trap "rm -fr T Coding" EXIT
dd if=/dev/urandom of=T bs=4096 count=1
./encoder T 3 2 reed_sol_van 8 0 0
./decoder T

2
Examples/encoder.c
View File

@ -326,7 +326,7 @@ int main (int argc, char **argv) {
/* Get current working directory for construction of file names */
curdir = (char*)malloc(sizeof(char)*1000);
getcwd(curdir, 1000);
assert(curdir == getcwd(curdir, 1000));
if (argv[1][0] != '-') {

2
Examples/jerasure_05.c
View File

@ -111,7 +111,6 @@ static void print_data_and_coding(int k, int m, int w, int size,
int main(int argc, char **argv)
{
long l;
int k, m, w, size;
int i, j;
int *matrix;
@ -168,7 +167,6 @@ int main(int argc, char **argv)
erasures = talloc(int, (m+1));
erased = talloc(int, (k+m));
for (i = 0; i < m+k; i++) erased[i] = 0;
l = 0;
for (i = 0; i < m; ) {
erasures[i] = (MOA_Random_W(w, 1))%(k+m);
if (erased[erasures[i]] == 0) {

7
Examples/test_all_gfs.sh
View File

@ -1,4 +1,4 @@
#
#!/bin/bash
#
# Copyright (c) 2013, James S. Plank and Kevin Greenan
# All rights reserved.
@ -42,6 +42,11 @@ k=12
m=3
seed=1370
if ! test -x ${GF_METHODS} ; then
${GF_METHODS}
exit 1
fi
# Test all w=8
${GF_METHODS} 8 -B -L | awk -F: '{ if ($1 == "w=8") print $2; }' |
while read method; do

23
Examples/test_galois.c Normal file
View File

@ -0,0 +1,23 @@
#include <assert.h>
#include "galois.h"
int main(int argc, char **argv)
{
assert(galois_init_default_field(4) == 0);
assert(galois_uninit_field(4) == 0);
assert(galois_init_default_field(4) == 0);
assert(galois_uninit_field(4) == 0);
assert(galois_init_default_field(8) == 0);
assert(galois_uninit_field(8) == 0);
assert(galois_init_default_field(8) == 0);
assert(galois_uninit_field(8) == 0);
return 0;
}
/*
* Local Variables:
* compile-command: "make test_galois &&
* libtool --mode=execute valgrind --tool=memcheck --leak-check=full ./test_galois"
* End:
*/

BIN
Manual.pdf Normal file
View File

Binary file not shown.

13
README
View File

@ -7,14 +7,17 @@ Authors: James S. Plank (University of Tennessee)
------------------------------------------------------------
The online home for jerasure is:
- http://jerasure.org/jerasure/jerasure
------------------------------------------------------------
External Documentation:
The programmer's manual and tutorial is provided in two places:
See the file Manual.pdf for the programmer's manual and tutorial.
1.) A copy is hosted on BitBucket at https://bitbucket.org/jimplank/jerasure/downloads/Jerasure-Manual.pdf
2.) A copy is also available at http://web.eecs.utk.edu/~plank/plank/papers/UT-EECS-14-721.html
See https://bitbucket.org/jimplank/gf-complete for GF-Complete.
See http://jerasure.org/jerasure/gf-complete for GF-Complete.
NOTE: You must have GF-Complete installed (or compiled) in order to use Jerasure 2.0.

12
configure.ac
View File

@ -2,7 +2,7 @@
AC_PREREQ([2.65])
AC_INIT([Jerasure], [2.0], [], [],
[https://bitbucket.org/jimplank/jerasure])
[https://jerasure.org/jerasure/jerasure])
AC_CONFIG_SRCDIR([src/jerasure.c])
AC_CONFIG_HEADERS([include/config.h])
@ -24,7 +24,7 @@ AC_PROG_CC
AC_CHECK_LIB([gf_complete], [gf_init_easy], [],
[AC_MSG_FAILURE(
[You need to have gf_complete installed.
gf_complete is available from http://web.eecs.utk.edu/~plank/plank/papers/CS-13-703.html])
gf_complete is available from http://jerasure.org/jerasure/gf-complete])
])
# Checks for header files.
@ -36,6 +36,14 @@ AC_TYPE_UINT32_T
AC_TYPE_UINT64_T
AX_EXT
AC_ARG_ENABLE([sse],
AS_HELP_STRING([--disable-sse], [Build without SSE optimizations]),
[if test "x$enableval" = "xno" ; then
SIMD_FLAGS=""
echo "DISABLED SSE!!!"
fi]
)
# Checks for library functions.
AC_FUNC_MALLOC
AC_CHECK_FUNCS([bzero getcwd gettimeofday mkdir strchr strdup strrchr])

1
include/galois.h
View File

@ -47,6 +47,7 @@ extern "C" {
#endif
extern int galois_init_default_field(int w);
extern int galois_uninit_field(int w);
extern void galois_change_technique(gf_t *gf, int w);
extern int galois_single_multiply(int a, int b, int w);

43
src/galois.c
View File

@ -48,6 +48,7 @@
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include <assert.h>
#include "galois.h"
@ -78,25 +79,25 @@ gf_t* galois_init_field(int w,
if (w <= 0 || w > 32) {
fprintf(stderr, "ERROR -- cannot init default Galois field for w=%d\n", w);
exit(1);
assert(0);
}
gfp = (gf_t *) malloc(sizeof(gf_t));
if (!gfp) {
fprintf(stderr, "ERROR -- cannot allocate memory for Galois field w=%d\n", w);
exit(1);
assert(0);
}
scratch_size = gf_scratch_size(w, mult_type, region_type, divide_type, arg1, arg2);
if (!scratch_size) {
fprintf(stderr, "ERROR -- cannot get scratch size for base field w=%d\n", w);
exit(1);
assert(0);
}
scratch_memory = malloc(scratch_size);
if (!scratch_memory) {
fprintf(stderr, "ERROR -- cannot get scratch memory for base field w=%d\n", w);
exit(1);
assert(0);
}
if(!gf_init_hard(gfp,
@ -111,7 +112,7 @@ gf_t* galois_init_field(int w,
scratch_memory))
{
fprintf(stderr, "ERROR -- cannot init default Galois field for w=%d\n", w);
exit(1);
assert(0);
}
gfp_is_composite[w] = 0;
@ -130,25 +131,25 @@ gf_t* galois_init_composite_field(int w,
if (w <= 0 || w > 32) {
fprintf(stderr, "ERROR -- cannot init composite field for w=%d\n", w);
exit(1);
assert(0);
}
gfp = (gf_t *) malloc(sizeof(gf_t));
if (!gfp) {
fprintf(stderr, "ERROR -- cannot allocate memory for Galois field w=%d\n", w);
exit(1);
assert(0);
}
scratch_size = gf_scratch_size(w, GF_MULT_COMPOSITE, region_type, divide_type, degree, 0);
if (!scratch_size) {
fprintf(stderr, "ERROR -- cannot get scratch size for composite field w=%d\n", w);
exit(1);
assert(0);
}
scratch_memory = malloc(scratch_size);
if (!scratch_memory) {
fprintf(stderr, "ERROR -- cannot get scratch memory for composite field w=%d\n", w);
exit(1);
assert(0);
}
if(!gf_init_hard(gfp,
@ -163,7 +164,7 @@ gf_t* galois_init_composite_field(int w,
scratch_memory))
{
fprintf(stderr, "ERROR -- cannot init default composite field for w=%d\n", w);
exit(1);
assert(0);
}
gfp_is_composite[w] = 1;
return gfp;
@ -181,21 +182,33 @@ int galois_init_default_field(int w)
return 0;
}
int galois_uninit_field(int w)
{
int ret = 0;
if (gfp_array[w] != NULL) {
int recursive = 1;
ret = gf_free(gfp_array[w], recursive);
free(gfp_array[w]);
gfp_array[w] = NULL;
}
return ret;
}
static void galois_init(int w)
{
if (w <= 0 || w > 32) {
fprintf(stderr, "ERROR -- cannot init default Galois field for w=%d\n", w);
exit(1);
assert(0);
}
switch (galois_init_default_field(w)) {
case ENOMEM:
fprintf(stderr, "ERROR -- cannot allocate memory for Galois field w=%d\n", w);
exit(1);
assert(0);
break;
case EINVAL:
fprintf(stderr, "ERROR -- cannot init default Galois field for w=%d\n", w);
exit(1);
assert(0);
break;
}
}
@ -234,12 +247,12 @@ void galois_change_technique(gf_t *gf, int w)
{
if (w <= 0 || w > 32) {
fprintf(stderr, "ERROR -- cannot support Galois field for w=%d\n", w);
exit(1);
assert(0);
}
if (!is_valid_gf(gf, w)) {
fprintf(stderr, "ERROR -- overriding with invalid Galois field for w=%d\n", w);
exit(1);
assert(0);
}
if (gfp_array[w] != NULL) {

13
src/jerasure.c
View File

@ -47,6 +47,7 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <assert.h>
#include "galois.h"
#include "jerasure.h"
@ -311,7 +312,7 @@ void jerasure_matrix_encode(int k, int m, int w, int *matrix,
if (w != 8 && w != 16 && w != 32) {
fprintf(stderr, "ERROR: jerasure_matrix_encode() and w is not 8, 16 or 32\n");
exit(1);
assert(0);
}
for (i = 0; i < m; i++) {
@ -328,7 +329,7 @@ void jerasure_bitmatrix_dotprod(int k, int w, int *bitmatrix_row,
if (size%(w*packetsize) != 0) {
fprintf(stderr, "jerasure_bitmatrix_dotprod - size%c(w*packetsize)) must = 0\n", '%');
exit(1);
assert(0);
}
bpptr = (dest_id < k) ? data_ptrs[dest_id] : coding_ptrs[dest_id-k];
@ -567,7 +568,7 @@ void jerasure_free_schedule_cache(int k, int m, int ***cache)
if (m != 2) {
fprintf(stderr, "jerasure_free_schedule_cache(): m must equal 2\n");
exit(1);
assert(0);
}
for (e1 = 0; e1 < k+m; e1++) {
@ -589,7 +590,7 @@ void jerasure_matrix_dotprod(int k, int w, int *matrix_row,
if (w != 1 && w != 8 && w != 16 && w != 32) {
fprintf(stderr, "ERROR: jerasure_matrix_dotprod() called and w is not 1, 8, 16 or 32\n");
exit(1);
assert(0);
}
init = 0;
@ -1456,12 +1457,12 @@ void jerasure_bitmatrix_encode(int k, int m, int w, int *bitmatrix,
if (packetsize%sizeof(long) != 0) {
fprintf(stderr, "jerasure_bitmatrix_encode - packetsize(%d) %c sizeof(long) != 0\n", packetsize, '%');
exit(1);
assert(0);
}
if (size%(packetsize*w) != 0) {
fprintf(stderr, "jerasure_bitmatrix_encode - size(%d) %c (packetsize(%d)*w(%d))) != 0\n",
size, '%', packetsize, w);
exit(1);
assert(0);
}
for (i = 0; i < m; i++) {

11
src/reed_sol.c
View File

@ -47,6 +47,7 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <assert.h>
#include <gf_complete.h>
#include "galois.h"
@ -107,7 +108,7 @@ void reed_sol_galois_w08_region_multby_2(char *region, int nbytes)
if (!gf_init_hard(&GF08, 8, GF_MULT_BYTWO_b, GF_REGION_DEFAULT, GF_DIVIDE_DEFAULT,
prim08, 0, 0, NULL, NULL)) {
fprintf(stderr, "Error: Can't initialize the GF for reed_sol_galois_w08_region_multby_2\n");
exit(1);
assert(0);
}
}
GF08.multiply_region.w32(&GF08, region, region, 2, nbytes, 0);
@ -123,7 +124,7 @@ void reed_sol_galois_w16_region_multby_2(char *region, int nbytes)
if (!gf_init_hard(&GF16, 16, GF_MULT_BYTWO_b, GF_REGION_DEFAULT, GF_DIVIDE_DEFAULT,
prim16, 0, 0, NULL, NULL)) {
fprintf(stderr, "Error: Can't initialize the GF for reed_sol_galois_w16_region_multby_2\n");
exit(1);
assert(0);
}
}
GF16.multiply_region.w32(&GF16, region, region, 2, nbytes, 0);
@ -135,11 +136,11 @@ static gf_t GF32;
void reed_sol_galois_w32_region_multby_2(char *region, int nbytes)
{
if (prim32 == -1) {
prim32 = galois_single_multiply((1 << 31), 2, 32);
prim32 = galois_single_multiply(((gf_val_32_t)1 << 31), 2, 32);
if (!gf_init_hard(&GF32, 32, GF_MULT_BYTWO_b, GF_REGION_DEFAULT, GF_DIVIDE_DEFAULT,
prim32, 0, 0, NULL, NULL)) {
fprintf(stderr, "Error: Can't initialize the GF for reed_sol_galois_w32_region_multby_2\n");
exit(1);
assert(0);
}
}
GF32.multiply_region.w32(&GF32, region, region, 2, nbytes, 0);
@ -223,7 +224,7 @@ int *reed_sol_big_vandermonde_distribution_matrix(int rows, int cols, int w)
if (j >= rows) { /* This should never happen if rows/w are correct */
fprintf(stderr, "reed_sol_big_vandermonde_distribution_matrix(%d,%d,%d) - couldn't make matrix\n",
rows, cols, w);
exit(1);
assert(0);
}
/* If necessary, swap rows */