http://git-wip-us.apache.org/repos/asf/hadoop/blob/658ee95f/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawDecoder.h ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawDecoder.h b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawDecoder.h new file mode 100644 index 0000000..40da4e1 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawDecoder.h @@ -0,0 +1,37 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include <jni.h> +/* Header for class org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawDecoder */ + +#ifndef _Included_org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawDecoder +#define _Included_org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawDecoder +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawDecoder + * Method: initImpl + * Signature: (II[I)V + */ +JNIEXPORT void JNICALL Java_org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawDecoder_initImpl + (JNIEnv *, jobject, jint, jint); + +/* + * Class: org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawDecoder + * Method: decodeImpl + * Signature: ([Ljava/nio/ByteBuffer;[II[I[Ljava/nio/ByteBuffer;[I)V + */ +JNIEXPORT void JNICALL Java_org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawDecoder_decodeImpl + (JNIEnv *, jobject, jobjectArray, jintArray, jint, jintArray, jobjectArray, jintArray); + +/* + * Class: org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawDecoder + * Method: destroyImpl + * Signature: ()V + */ +JNIEXPORT void JNICALL Java_org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawDecoder_destroyImpl + (JNIEnv *, jobject); + +#ifdef __cplusplus +} +#endif +#endif
http://git-wip-us.apache.org/repos/asf/hadoop/blob/658ee95f/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawEncoder.h ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawEncoder.h b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawEncoder.h new file mode 100644 index 0000000..db094cf --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawEncoder.h @@ -0,0 +1,37 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include <jni.h> +/* Header for class org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawEncoder */ + +#ifndef _Included_org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawEncoder +#define _Included_org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawEncoder +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawEncoder + * Method: initImpl + * Signature: (II[I)V + */ +JNIEXPORT void JNICALL Java_org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawEncoder_initImpl + (JNIEnv *, jobject, jint, jint); + +/* + * Class: org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawEncoder + * Method: encodeImpl + * Signature: ([Ljava/nio/ByteBuffer;[II[Ljava/nio/ByteBuffer;[I)V + */ +JNIEXPORT void JNICALL Java_org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawEncoder_encodeImpl + (JNIEnv *, jobject, jobjectArray, jintArray, jint, jobjectArray, jintArray); + +/* + * Class: org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawEncoder + * Method: destroyImpl + * Signature: ()V + */ +JNIEXPORT void JNICALL Java_org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawEncoder_destroyImpl + (JNIEnv *, jobject); + +#ifdef __cplusplus +} +#endif +#endif http://git-wip-us.apache.org/repos/asf/hadoop/blob/658ee95f/hadoop-common-project/hadoop-common/src/main/native/src/test/org/apache/hadoop/io/erasurecode/erasure_code_test.c ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/test/org/apache/hadoop/io/erasurecode/erasure_code_test.c b/hadoop-common-project/hadoop-common/src/main/native/src/test/org/apache/hadoop/io/erasurecode/erasure_code_test.c index 9817a76..331bb21 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/test/org/apache/hadoop/io/erasurecode/erasure_code_test.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/test/org/apache/hadoop/io/erasurecode/erasure_code_test.c @@ -17,185 +17,36 @@ */ /** - * This is a lightweight version of the same file in Intel ISA-L library to test - * and verify the basic functions of ISA-L integration. Note it's not serving as - * a complete ISA-L library test nor as any sample to write an erasure coder - * using the library. A sample is to be written and provided separately. + * This is a sample program illustrating how to use the Intel ISA-L library. + * Note it's adapted from erasure_code_test.c test program, but trying to use + * variable names and styles we're more familiar with already similar to Java + * coders. */ -#include "org_apache_hadoop.h" +#include "isal_load.h" #include "erasure_code.h" #include "gf_util.h" +#include "erasure_coder.h" #include <stdio.h> #include <stdlib.h> #include <string.h> -#define TEST_LEN 8192 -#define TEST_SOURCES 127 -#define MMAX TEST_SOURCES -#define KMAX TEST_SOURCES -#define TEST_SEED 11 - -static void dump(unsigned char *buf, int len) -{ - int i; - for (i = 0; i < len;) { - printf(" %2x", 0xff & buf[i++]); - if (i % 32 == 0) - printf("\n"); - } - printf("\n"); -} - -static void dump_matrix(unsigned char **s, int k, int m) -{ +int main(int argc, char *argv[]) { int i, j; - for (i = 0; i < k; i++) { - for (j = 0; j < m; j++) { - printf(" %2x", s[i][j]); - } - printf("\n"); - } - printf("\n"); -} - -static void dump_u8xu8(unsigned char *s, int k, int m) -{ - int i, j; - for (i = 0; i < k; i++) { - for (j = 0; j < m; j++) { - printf(" %2x", 0xff & s[j + (i * m)]); - } - printf("\n"); - } - printf("\n"); -} - -// Generate Random errors -static void gen_err_list(unsigned char *src_err_list, - unsigned char *src_in_err, int *pnerrs, int *pnsrcerrs, int k, int m) -{ - int i, err; - int nerrs = 0, nsrcerrs = 0; - - for (i = 0, nerrs = 0, nsrcerrs = 0; i < m && nerrs < m - k; i++) { - err = 1 & rand(); - src_in_err[i] = err; - if (err) { - src_err_list[nerrs++] = i; - if (i < k) { - nsrcerrs++; - } - } - } - if (nerrs == 0) { // should have at least one error - while ((err = (rand() % KMAX)) >= m) ; - src_err_list[nerrs++] = err; - src_in_err[err] = 1; - if (err < k) - nsrcerrs = 1; - } - *pnerrs = nerrs; - *pnsrcerrs = nsrcerrs; - return; -} - -#define NO_INVERT_MATRIX -2 -// Generate decode matrix from encode matrix -static int gf_gen_decode_matrix(unsigned char *encode_matrix, - unsigned char *decode_matrix, - unsigned char *invert_matrix, - unsigned int *decode_index, - unsigned char *src_err_list, - unsigned char *src_in_err, - int nerrs, int nsrcerrs, int k, int m) -{ - int i, j, p; - int r; - unsigned char *backup, *b, s; - int incr = 0; - - b = malloc(MMAX * KMAX); - backup = malloc(MMAX * KMAX); - - if (b == NULL || backup == NULL) { - printf("Test failure! Error with malloc\n"); - free(b); - free(backup); - return -1; - } - // Construct matrix b by removing error rows - for (i = 0, r = 0; i < k; i++, r++) { - while (src_in_err[r]) - r++; - for (j = 0; j < k; j++) { - b[k * i + j] = encode_matrix[k * r + j]; - backup[k * i + j] = encode_matrix[k * r + j]; - } - decode_index[i] = r; - } - incr = 0; - while (h_gf_invert_matrix(b, invert_matrix, k) < 0) { - if (nerrs == (m - k)) { - free(b); - free(backup); - printf("BAD MATRIX\n"); - return NO_INVERT_MATRIX; - } - incr++; - memcpy(b, backup, MMAX * KMAX); - for (i = nsrcerrs; i < nerrs - nsrcerrs; i++) { - if (src_err_list[i] == (decode_index[k - 1] + incr)) { - // skip the erased parity line - incr++; - continue; - } - } - if (decode_index[k - 1] + incr >= m) { - free(b); - free(backup); - printf("BAD MATRIX\n"); - return NO_INVERT_MATRIX; - } - decode_index[k - 1] += incr; - for (j = 0; j < k; j++) - b[k * (k - 1) + j] = encode_matrix[k * decode_index[k - 1] + j]; - - }; - - for (i = 0; i < nsrcerrs; i++) { - for (j = 0; j < k; j++) { - decode_matrix[k * i + j] = invert_matrix[k * src_err_list[i] + j]; - } - } - /* src_err_list from encode_matrix * invert of b for parity decoding */ - for (p = nsrcerrs; p < nerrs; p++) { - for (i = 0; i < k; i++) { - s = 0; - for (j = 0; j < k; j++) - s ^= h_gf_mul(invert_matrix[j * k + i], - encode_matrix[k * src_err_list[p] + j]); - - decode_matrix[k * p + i] = s; - } - } - free(b); - free(backup); - return 0; -} - -int main(int argc, char *argv[]) -{ char err[256]; size_t err_len = sizeof(err); - int re, i, j, p, m, k; - int nerrs, nsrcerrs; - unsigned int decode_index[MMAX]; - unsigned char *temp_buffs[TEST_SOURCES], *buffs[TEST_SOURCES]; - unsigned char *encode_matrix, *decode_matrix, *invert_matrix, *g_tbls; - unsigned char src_in_err[TEST_SOURCES], src_err_list[TEST_SOURCES]; - unsigned char *recov[TEST_SOURCES]; + int chunkSize = 1024; + int numDataUnits = 6; + int numParityUnits = 3; + unsigned char** dataUnits; + unsigned char** parityUnits; + IsalEncoder* pEncoder; + int erasedIndexes[2]; + unsigned char* allUnits[MMAX]; + IsalDecoder* pDecoder; + unsigned char* decodingOutput[2]; + unsigned char** backupUnits; if (0 == build_support_erasurecode()) { printf("The native library isn't available, skipping this test\n"); @@ -209,102 +60,66 @@ int main(int argc, char *argv[]) } printf("Performing erasure code test\n"); - srand(TEST_SEED); - // Allocate the arrays - for (i = 0; i < TEST_SOURCES; i++) { - buffs[i] = malloc(TEST_LEN); - } + dataUnits = calloc(numDataUnits, sizeof(unsigned char*)); + parityUnits = calloc(numParityUnits, sizeof(unsigned char*)); + backupUnits = calloc(numParityUnits, sizeof(unsigned char*)); - for (i = 0; i < TEST_SOURCES; i++) { - temp_buffs[i] = malloc(TEST_LEN); + // Allocate and generate data units + srand(135); + for (i = 0; i < numDataUnits; i++) { + dataUnits[i] = calloc(chunkSize, sizeof(unsigned char)); + for (j = 0; j < chunkSize; j++) { + dataUnits[i][j] = rand(); + } } - // Test erasure code by encode and recovery - - encode_matrix = malloc(MMAX * KMAX); - decode_matrix = malloc(MMAX * KMAX); - invert_matrix = malloc(MMAX * KMAX); - g_tbls = malloc(KMAX * TEST_SOURCES * 32); - if (encode_matrix == NULL || decode_matrix == NULL - || invert_matrix == NULL || g_tbls == NULL) { - snprintf(err, err_len, "%s", "allocating test matrix buffers error"); - return -1; + // Allocate and initialize parity units + for (i = 0; i < numParityUnits; i++) { + parityUnits[i] = calloc(chunkSize, sizeof(unsigned char)); + for (j = 0; j < chunkSize; j++) { + parityUnits[i][j] = 0; + } } - m = 9; - k = 5; - if (m > MMAX || k > KMAX) - return -1; - - // Make random data - for (i = 0; i < k; i++) - for (j = 0; j < TEST_LEN; j++) - buffs[i][j] = rand(); + pEncoder = (IsalEncoder*)malloc(sizeof(IsalEncoder)); + memset(pEncoder, 0, sizeof(*pEncoder)); + initEncoder(pEncoder, numDataUnits, numParityUnits); + encode(pEncoder, dataUnits, parityUnits, chunkSize); - // The matrix generated by gf_gen_cauchy1_matrix - // is always invertable. - h_gf_gen_cauchy_matrix(encode_matrix, m, k); + pDecoder = (IsalDecoder*)malloc(sizeof(IsalDecoder)); + memset(pDecoder, 0, sizeof(*pDecoder)); + initDecoder(pDecoder, numDataUnits, numParityUnits); - // Generate g_tbls from encode matrix encode_matrix - h_ec_init_tables(k, m - k, &encode_matrix[k * k], g_tbls); + memcpy(allUnits, dataUnits, numDataUnits * (sizeof (unsigned char*))); + memcpy(allUnits + numDataUnits, parityUnits, + numParityUnits * (sizeof (unsigned char*))); - // Perform matrix dot_prod for EC encoding - // using g_tbls from encode matrix encode_matrix - h_ec_encode_data(TEST_LEN, k, m - k, g_tbls, buffs, &buffs[k]); + erasedIndexes[0] = 1; + erasedIndexes[1] = 7; - // Choose random buffers to be in erasure - memset(src_in_err, 0, TEST_SOURCES); - gen_err_list(src_err_list, src_in_err, &nerrs, &nsrcerrs, k, m); + backupUnits[0] = allUnits[1]; + backupUnits[1] = allUnits[7]; - // Generate decode matrix - re = gf_gen_decode_matrix(encode_matrix, decode_matrix, - invert_matrix, decode_index, src_err_list, src_in_err, - nerrs, nsrcerrs, k, m); - if (re != 0) { - snprintf(err, err_len, "%s", "gf_gen_decode_matrix failed"); - return -1; - } - // Pack recovery array as list of valid sources - // Its order must be the same as the order - // to generate matrix b in gf_gen_decode_matrix - for (i = 0; i < k; i++) { - recov[i] = buffs[decode_index[i]]; - } + allUnits[0] = NULL; // Not to read + allUnits[1] = NULL; + allUnits[7] = NULL; - // Recover data - h_ec_init_tables(k, nerrs, decode_matrix, g_tbls); - h_ec_encode_data(TEST_LEN, k, nerrs, g_tbls, recov, &temp_buffs[k]); - for (i = 0; i < nerrs; i++) { - if (0 != memcmp(temp_buffs[k + i], buffs[src_err_list[i]], TEST_LEN)) { - snprintf(err, err_len, "%s", "Error recovery failed"); - printf("Fail error recovery (%d, %d, %d)\n", m, k, nerrs); + decodingOutput[0] = malloc(chunkSize); + decodingOutput[1] = malloc(chunkSize); - printf(" - erase list = "); - for (j = 0; j < nerrs; j++) { - printf(" %d", src_err_list[j]); - } - - printf(" - Index = "); - for (p = 0; p < k; p++) { - printf(" %d", decode_index[p]); - } - - printf("\nencode_matrix:\n"); - dump_u8xu8((unsigned char *) encode_matrix, m, k); - printf("inv b:\n"); - dump_u8xu8((unsigned char *) invert_matrix, k, k); - printf("\ndecode_matrix:\n"); - dump_u8xu8((unsigned char *) decode_matrix, m, k); - printf("recov %d:", src_err_list[i]); - dump(temp_buffs[k + i], 25); - printf("orig :"); - dump(buffs[src_err_list[i]], 25); + decode(pDecoder, allUnits, erasedIndexes, 2, decodingOutput, chunkSize); + for (i = 0; i < pDecoder->numErased; i++) { + if (0 != memcmp(decodingOutput[i], backupUnits[i], chunkSize)) { + fprintf(stderr, "Decoding failed\n\n"); + dumpDecoder(pDecoder); return -1; } } - printf("done EC tests: Pass\n"); + dumpDecoder(pDecoder); + fprintf(stdout, "Successfully done, passed!\n\n"); + return 0; }
