HADOOP-11996. Improve and restructure native ISAL support (Kai Zheng via cmccabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/658ee95f Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/658ee95f Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/658ee95f Branch: refs/heads/HDFS-7240 Commit: 658ee95ff367eaf58d76581e75bdaba0dd0d871e Parents: 6e9a582 Author: Colin Patrick Mccabe <[email protected]> Authored: Fri Mar 11 12:56:12 2016 -0800 Committer: Colin Patrick Mccabe <[email protected]> Committed: Fri Mar 11 12:57:54 2016 -0800 ---------------------------------------------------------------------- .../hadoop-common/src/CMakeLists.txt | 22 +- .../src/main/native/native.vcxproj | 11 +- .../io/erasurecode/coder/erasure_code_native.c | 49 --- ...he_hadoop_io_erasurecode_ErasureCodeNative.h | 29 -- .../src/org/apache/hadoop/io/erasurecode/dump.c | 100 ++++++ .../src/org/apache/hadoop/io/erasurecode/dump.h | 40 +++ .../apache/hadoop/io/erasurecode/erasure_code.c | 239 +-------------- .../apache/hadoop/io/erasurecode/erasure_code.h | 107 +++++++ .../hadoop/io/erasurecode/erasure_coder.c | 229 ++++++++++++++ .../hadoop/io/erasurecode/erasure_coder.h | 88 ++++++ .../org/apache/hadoop/io/erasurecode/gf_util.c | 54 ++++ .../org/apache/hadoop/io/erasurecode/gf_util.h | 111 +++++++ .../io/erasurecode/include/erasure_code.h | 125 -------- .../hadoop/io/erasurecode/include/gf_util.h | 111 ------- .../apache/hadoop/io/erasurecode/isal_load.c | 148 +++++++++ .../apache/hadoop/io/erasurecode/isal_load.h | 149 +++++++++ .../apache/hadoop/io/erasurecode/jni_common.c | 98 ++++++ .../apache/hadoop/io/erasurecode/jni_common.h | 42 +++ .../io/erasurecode/jni_erasure_code_native.c | 45 +++ .../hadoop/io/erasurecode/jni_rs_decoder.c | 72 +++++ .../hadoop/io/erasurecode/jni_rs_encoder.c | 66 ++++ ...he_hadoop_io_erasurecode_ErasureCodeNative.h | 29 ++ ...io_erasurecode_rawcoder_NativeRSRawDecoder.h | 37 +++ ...io_erasurecode_rawcoder_NativeRSRawEncoder.h | 37 +++ .../hadoop/io/erasurecode/erasure_code_test.c | 307 ++++--------------- 25 files changed, 1545 insertions(+), 800 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/658ee95f/hadoop-common-project/hadoop-common/src/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/CMakeLists.txt b/hadoop-common-project/hadoop-common/src/CMakeLists.txt index 51fb0fe..8026eb5 100644 --- a/hadoop-common-project/hadoop-common/src/CMakeLists.txt +++ b/hadoop-common-project/hadoop-common/src/CMakeLists.txt @@ -103,13 +103,25 @@ find_library(ISAL_LIBRARY set(CMAKE_FIND_LIBRARY_SUFFIXES ${STORED_CMAKE_FIND_LIBRARY_SUFFIXES}) if (ISAL_LIBRARY) GET_FILENAME_COMPONENT(HADOOP_ISAL_LIBRARY ${ISAL_LIBRARY} NAME) - set(ISAL_INCLUDE_DIR ${SRC}/io/erasurecode/include) + set(ISAL_INCLUDE_DIR ${SRC}/io/erasurecode) set(ISAL_SOURCE_FILES - ${SRC}/io/erasurecode/coder/erasure_code_native.c - ${SRC}/io/erasurecode/erasure_code.c) + ${SRC}/io/erasurecode/isal_load.c + ${SRC}/io/erasurecode/erasure_code.c + ${SRC}/io/erasurecode/gf_util.c + ${SRC}/io/erasurecode/dump.c + ${SRC}/io/erasurecode/erasure_coder.c + ${SRC}/io/erasurecode/jni_erasure_code_native.c + ${SRC}/io/erasurecode/jni_common.c + ${SRC}/io/erasurecode/jni_rs_encoder.c + ${SRC}/io/erasurecode/jni_rs_decoder.c) + add_executable(erasure_code_test - ${SRC}/io/erasurecode/erasure_code.c - ${TST}/io/erasurecode/erasure_code_test.c + ${SRC}/io/erasurecode/isal_load.c + ${SRC}/io/erasurecode/erasure_code.c + ${SRC}/io/erasurecode/gf_util.c + ${SRC}/io/erasurecode/dump.c + ${SRC}/io/erasurecode/erasure_coder.c + ${TST}/io/erasurecode/erasure_code_test.c ) target_link_libraries(erasure_code_test ${CMAKE_DL_LIBS}) else (ISAL_LIBRARY) http://git-wip-us.apache.org/repos/asf/hadoop/blob/658ee95f/hadoop-common-project/hadoop-common/src/main/native/native.vcxproj ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/native/native.vcxproj b/hadoop-common-project/hadoop-common/src/main/native/native.vcxproj index 17149f7..2274c41 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/native.vcxproj +++ b/hadoop-common-project/hadoop-common/src/main/native/native.vcxproj @@ -162,10 +162,17 @@ <ClCompile Include="src\org\apache\hadoop\yarn\server\nodemanager\windows_secure_container_executor.c"> <AdditionalIncludeDirectories>src\org\apache\hadoop\io\nativeio;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> </ClCompile> - <ClCompile Include="src\org\apache\hadoop\io\erasurecode\erasure_code.c" Condition="'$(IsalEnabled)' == 'true'"> + <ClCompile Include="src\org\apache\hadoop\io\erasurecode\isal_load.c" Condition="'$(IsalEnabled)' == 'true'"> <AdditionalOptions>/D HADOOP_ISAL_LIBRARY=\"isa-l.dll\"</AdditionalOptions> </ClCompile> - <ClCompile Include="src\org\apache\hadoop\io\erasurecode\coder\erasure_code_native.c" Condition="'$(IsalEnabled)' == 'true'"/> + <ClCompile Include="src\org\apache\hadoop\io\erasurecode\erasure_code.c" Condition="'$(IsalEnabled)' == 'true'"/> + <ClCompile Include="src\org\apache\hadoop\io\erasurecode\gf_util.c" Condition="'$(IsalEnabled)' == 'true'"/> + <ClCompile Include="src\org\apache\hadoop\io\erasurecode\erasure_coder.c" Condition="'$(IsalEnabled)' == 'true'"/> + <ClCompile Include="src\org\apache\hadoop\io\erasurecode\dump.c" Condition="'$(IsalEnabled)' == 'true'"/> + <ClCompile Include="src\org\apache\hadoop\io\erasurecode\jni_erasure_code_native.c" Condition="'$(IsalEnabled)' == 'true'"/> + <ClCompile Include="src\org\apache\hadoop\io\erasurecode\jni_common.c" Condition="'$(IsalEnabled)' == 'true'"/> + <ClCompile Include="src\org\apache\hadoop\io\erasurecode\jni_rs_encoder.c" Condition="'$(IsalEnabled)' == 'true'"/> + <ClCompile Include="src\org\apache\hadoop\io\erasurecode\jni_rs_decoder.c" Condition="'$(IsalEnabled)' == 'true'"/> </ItemGroup> <ItemGroup> <ClInclude Include="..\src\org\apache\hadoop\util\crc32c_tables.h" /> http://git-wip-us.apache.org/repos/asf/hadoop/blob/658ee95f/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/coder/erasure_code_native.c ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/coder/erasure_code_native.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/coder/erasure_code_native.c deleted file mode 100644 index e84df9a..0000000 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/coder/erasure_code_native.c +++ /dev/null @@ -1,49 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <stdio.h> -#include <stdlib.h> -#include <string.h> - -#include "org_apache_hadoop.h" -#include "../include/erasure_code.h" -#include "org_apache_hadoop_io_erasurecode_ErasureCodeNative.h" - -#ifdef UNIX -#include "config.h" -#endif - -JNIEXPORT void JNICALL -Java_org_apache_hadoop_io_erasurecode_ErasureCodeNative_loadLibrary -(JNIEnv *env, jclass myclass) { - char errMsg[1024]; - load_erasurecode_lib(errMsg, sizeof(errMsg)); - if (strlen(errMsg) > 0) { - THROW(env, "java/lang/UnsatisfiedLinkError", errMsg); - } -} - -JNIEXPORT jstring JNICALL -Java_org_apache_hadoop_io_erasurecode_ErasureCodeNative_getLibraryName -(JNIEnv *env, jclass myclass) { - char* libName = get_library_name(); - if (libName == NULL) { - libName = "Unavailable"; - } - return (*env)->NewStringUTF(env, libName); -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/658ee95f/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/coder/org_apache_hadoop_io_erasurecode_ErasureCodeNative.h ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/coder/org_apache_hadoop_io_erasurecode_ErasureCodeNative.h b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/coder/org_apache_hadoop_io_erasurecode_ErasureCodeNative.h deleted file mode 100644 index d8ff3a0..0000000 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/coder/org_apache_hadoop_io_erasurecode_ErasureCodeNative.h +++ /dev/null @@ -1,29 +0,0 @@ -/* DO NOT EDIT THIS FILE - it is machine generated */ -#include <jni.h> -/* Header for class org_apache_hadoop_io_erasurecode_ErasureCodeNative */ - -#ifndef _Included_org_apache_hadoop_io_erasurecode_ErasureCodeNative -#define _Included_org_apache_hadoop_io_erasurecode_ErasureCodeNative -#ifdef __cplusplus -extern "C" { -#endif -/* - * Class: org_apache_hadoop_io_erasurecode_ErasureCodeNative - * Method: loadLibrary - * Signature: ()V - */ -JNIEXPORT void JNICALL Java_org_apache_hadoop_io_erasurecode_ErasureCodeNative_loadLibrary - (JNIEnv *, jclass); - -/* - * Class: org_apache_hadoop_io_erasurecode_ErasureCodeNative - * Method: getLibraryName - * Signature: ()Ljava/lang/String; - */ -JNIEXPORT jstring JNICALL Java_org_apache_hadoop_io_erasurecode_ErasureCodeNative_getLibraryName - (JNIEnv *, jclass); - -#ifdef __cplusplus -} -#endif -#endif http://git-wip-us.apache.org/repos/asf/hadoop/blob/658ee95f/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/dump.c ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/dump.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/dump.c new file mode 100644 index 0000000..20bd189 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/dump.c @@ -0,0 +1,100 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "erasure_code.h" +#include "gf_util.h" +#include "erasure_coder.h" + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +void dump(unsigned char* buf, int len) { + int i; + for (i = 0; i < len;) { + printf(" %2x", 0xff & buf[i++]); + if (i % 32 == 0) + printf("\n"); + } +} + +void dumpMatrix(unsigned char** buf, int n1, int n2) { + int i, j; + for (i = 0; i < n1; i++) { + for (j = 0; j < n2; j++) { + printf(" %2x", buf[i][j]); + } + printf("\n"); + } + printf("\n"); +} + +void dumpCodingMatrix(unsigned char* buf, int n1, int n2) { + int i, j; + for (i = 0; i < n1; i++) { + for (j = 0; j < n2; j++) { + printf(" %d", 0xff & buf[j + (i * n2)]); + } + printf("\n"); + } + printf("\n"); +} + +void dumpEncoder(IsalEncoder* pCoder) { + int numDataUnits = pCoder->coder.numDataUnits; + int numParityUnits = pCoder->coder.numDataUnits; + int numAllUnits = pCoder->coder.numAllUnits; + + printf("Encoding (numAlnumParityUnitslUnits = %d, numDataUnits = %d)\n", + numParityUnits, numDataUnits); + + printf("\n\nEncodeMatrix:\n"); + dumpCodingMatrix((unsigned char*) pCoder->encodeMatrix, + numDataUnits, numAllUnits); +} + +void dumpDecoder(IsalDecoder* pCoder) { + int i, j; + int numDataUnits = pCoder->coder.numDataUnits; + int numAllUnits = pCoder->coder.numAllUnits; + + printf("Recovering (numAllUnits = %d, numDataUnits = %d, numErased = %d)\n", + numAllUnits, numDataUnits, pCoder->numErased); + + printf(" - ErasedIndexes = "); + for (j = 0; j < pCoder->numErased; j++) { + printf(" %d", pCoder->erasedIndexes[j]); + } + printf(" - DecodeIndex = "); + for (i = 0; i < numDataUnits; i++) { + printf(" %d", pCoder->decodeIndex[i]); + } + + printf("\n\nEncodeMatrix:\n"); + dumpCodingMatrix((unsigned char*) pCoder->encodeMatrix, + numDataUnits, numAllUnits); + + printf("InvertMatrix:\n"); + dumpCodingMatrix((unsigned char*) pCoder->invertMatrix, + numDataUnits, numDataUnits); + + printf("DecodeMatrix:\n"); + dumpCodingMatrix((unsigned char*) pCoder->decodeMatrix, + numDataUnits, numAllUnits); +} + http://git-wip-us.apache.org/repos/asf/hadoop/blob/658ee95f/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/dump.h ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/dump.h b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/dump.h new file mode 100644 index 0000000..f58d960 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/dump.h @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Dump utilities for erasure coders. + */ + +#ifndef _DUMP_H_ +#define _DUMP_H_ + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +void dumpEncoder(IsalEncoder* pCoder); + +void dumpDecoder(IsalDecoder* pCoder); + +void dump(unsigned char* buf, int len); + +void dumpMatrix(unsigned char** s, int k, int m); + +void dumpCodingMatrix(unsigned char* s, int n1, int n2); + +#endif //_DUMP_H_ http://git-wip-us.apache.org/repos/asf/hadoop/blob/658ee95f/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/erasure_code.c ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/erasure_code.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/erasure_code.c index a6c099a..1250688 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/erasure_code.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/erasure_code.c @@ -20,252 +20,25 @@ #include <stdlib.h> #include <string.h> -#include "org_apache_hadoop.h" -#include "../include/gf_util.h" -#include "../include/erasure_code.h" - -#ifdef UNIX -#include <sys/time.h> -#include <sys/types.h> -#include <sys/stat.h> -#include <dlfcn.h> - -#include "config.h" -#endif - -#ifdef WINDOWS -#include <Windows.h> -#endif +#include "isal_load.h" +#include "erasure_code.h" /** * erasure_code.c - * Implementation erasure code utilities based on lib of erasure_code.so. - * Building of this codes won't rely on any ISA-L source codes, but running - * into this will rely on successfully loading of the dynamic library. + * Implementation erasure code utilities based on ISA-L library. * */ -/** - * The loaded library handle. - */ -static void* libec = NULL; - -/** - * A helper function to dlsym a 'symbol' from a given library-handle. - */ - -#ifdef UNIX - -static __attribute__ ((unused)) -void *my_dlsym(void *handle, const char *symbol) { - void *func_ptr = dlsym(handle, symbol); - return func_ptr; -} - -/* A helper macro to dlsym the requisite dynamic symbol in NON-JNI env. */ -#define EC_LOAD_DYNAMIC_SYMBOL(func_ptr, handle, symbol) \ - if ((func_ptr = my_dlsym(handle, symbol)) == NULL) { \ - return "Failed to load symbol" symbol; \ - } - -#endif - -#ifdef WINDOWS - - - -static FARPROC WINAPI my_dlsym(HMODULE handle, LPCSTR symbol) { - FARPROC func_ptr = GetProcAddress(handle, symbol); - return func_ptr; -} - -/* A helper macro to dlsym the requisite dynamic symbol in NON-JNI env. */ -#define EC_LOAD_DYNAMIC_SYMBOL(func_type, func_ptr, handle, symbol) \ - if ((func_ptr = (func_type)my_dlsym(handle, symbol)) == NULL) { \ - return "Failed to load symbol" symbol; \ - } - -#endif - - -#ifdef UNIX -// For gf_util.h -static unsigned char (*d_gf_mul)(unsigned char, unsigned char); -static unsigned char (*d_gf_inv)(unsigned char); -static void (*d_gf_gen_rs_matrix)(unsigned char *, int, int); -static void (*d_gf_gen_cauchy_matrix)(unsigned char *, int, int); -static int (*d_gf_invert_matrix)(unsigned char *, unsigned char *, const int); -static int (*d_gf_vect_mul)(int, unsigned char *, void *, void *); - -// For erasure_code.h -static void (*d_ec_init_tables)(int, int, unsigned char*, unsigned char*); -static void (*d_ec_encode_data)(int, int, int, unsigned char*, - unsigned char**, unsigned char**); -static void (*d_ec_encode_data_update)(int, int, int, int, unsigned char*, - unsigned char*, unsigned char**); -#endif - -#ifdef WINDOWS -// For erasure_code.h -typedef unsigned char (__cdecl *__d_gf_mul)(unsigned char, unsigned char); -static __d_gf_mul d_gf_mul; -typedef unsigned char (__cdecl *__d_gf_inv)(unsigned char); -static __d_gf_inv d_gf_inv; -typedef void (__cdecl *__d_gf_gen_rs_matrix)(unsigned char *, int, int); -static __d_gf_gen_rs_matrix d_gf_gen_rs_matrix; -typedef void (__cdecl *__d_gf_gen_cauchy_matrix)(unsigned char *, int, int); -static __d_gf_gen_cauchy_matrix d_gf_gen_cauchy_matrix; -typedef int (__cdecl *__d_gf_invert_matrix)(unsigned char *, - unsigned char *, const int); -static __d_gf_invert_matrix d_gf_invert_matrix; -typedef int (__cdecl *__d_gf_vect_mul)(int, unsigned char *, void *, void *); -static __d_gf_vect_mul d_gf_vect_mul; - -// For erasure_code.h -typedef void (__cdecl *__d_ec_init_tables)(int, int, - unsigned char*, unsigned char*); -static __d_ec_init_tables d_ec_init_tables; -typedef void (__cdecl *__d_ec_encode_data)(int, int, int, unsigned char*, - unsigned char**, unsigned char**); -static __d_ec_encode_data d_ec_encode_data; -typedef void (__cdecl *__d_ec_encode_data_update)(int, int, int, int, unsigned char*, - unsigned char*, unsigned char**); -static __d_ec_encode_data_update d_ec_encode_data_update; -#endif - -static const char* load_functions(void* libec) { -#ifdef UNIX - EC_LOAD_DYNAMIC_SYMBOL(d_gf_mul, libec, "gf_mul"); - EC_LOAD_DYNAMIC_SYMBOL(d_gf_inv, libec, "gf_inv"); - EC_LOAD_DYNAMIC_SYMBOL(d_gf_gen_rs_matrix, libec, "gf_gen_rs_matrix"); - EC_LOAD_DYNAMIC_SYMBOL(d_gf_gen_cauchy_matrix, libec, "gf_gen_cauchy1_matrix"); - EC_LOAD_DYNAMIC_SYMBOL(d_gf_invert_matrix, libec, "gf_invert_matrix"); - EC_LOAD_DYNAMIC_SYMBOL(d_gf_vect_mul, libec, "gf_vect_mul"); - - EC_LOAD_DYNAMIC_SYMBOL(d_ec_init_tables, libec, "ec_init_tables"); - EC_LOAD_DYNAMIC_SYMBOL(d_ec_encode_data, libec, "ec_encode_data"); - EC_LOAD_DYNAMIC_SYMBOL(d_ec_encode_data_update, libec, "ec_encode_data_update"); -#endif - -#ifdef WINDOWS - EC_LOAD_DYNAMIC_SYMBOL(__d_gf_mul, d_gf_mul, libec, "gf_mul"); - EC_LOAD_DYNAMIC_SYMBOL(__d_gf_inv, d_gf_inv, libec, "gf_inv"); - EC_LOAD_DYNAMIC_SYMBOL(__d_gf_gen_rs_matrix, d_gf_gen_rs_matrix, libec, "gf_gen_rs_matrix"); - EC_LOAD_DYNAMIC_SYMBOL(__d_gf_gen_cauchy_matrix, d_gf_gen_cauchy_matrix, libec, "gf_gen_cauchy1_matrix"); - EC_LOAD_DYNAMIC_SYMBOL(__d_gf_invert_matrix, d_gf_invert_matrix, libec, "gf_invert_matrix"); - EC_LOAD_DYNAMIC_SYMBOL(__d_gf_vect_mul, d_gf_vect_mul, libec, "gf_vect_mul"); - - EC_LOAD_DYNAMIC_SYMBOL(__d_ec_init_tables, d_ec_init_tables, libec, "ec_init_tables"); - EC_LOAD_DYNAMIC_SYMBOL(__d_ec_encode_data, d_ec_encode_data, libec, "ec_encode_data"); - EC_LOAD_DYNAMIC_SYMBOL(__d_ec_encode_data_update, d_ec_encode_data_update, libec, "ec_encode_data_update"); -#endif - - return NULL; -} - -void load_erasurecode_lib(char* err, size_t err_len) { - const char* errMsg; - - err[0] = '\0'; - - if (libec != NULL) { - return; - } - - // Load Intel ISA-L - #ifdef UNIX - libec = dlopen(HADOOP_ISAL_LIBRARY, RTLD_LAZY | RTLD_GLOBAL); - if (libec == NULL) { - snprintf(err, err_len, "Failed to load %s (%s)", - HADOOP_ISAL_LIBRARY, dlerror()); - return; - } - // Clear any existing error - dlerror(); - #endif - - #ifdef WINDOWS - libec = LoadLibrary(HADOOP_ISAL_LIBRARY); - if (libec == NULL) { - snprintf(err, err_len, "Failed to load %s", HADOOP_ISAL_LIBRARY); - return; - } - #endif - - errMsg = load_functions(libec); - if (errMsg != NULL) { - snprintf(err, err_len, "Loading functions from ISA-L failed: %s", errMsg); - } -} - -int build_support_erasurecode() { -#ifdef HADOOP_ISAL_LIBRARY - return 1; -#else - return 0; -#endif -} - -const char* get_library_name() { -#ifdef UNIX - Dl_info dl_info; - - if (d_ec_encode_data == NULL) { - return HADOOP_ISAL_LIBRARY; - } - - if(dladdr(d_ec_encode_data, &dl_info)) { - return dl_info.dli_fname; - } -#else - LPTSTR filename = NULL; - - if (libec == NULL) { - return HADOOP_ISAL_LIBRARY; - } - - if (GetModuleFileName(libec, filename, 256) > 0) { - return filename; - } -#endif - - return NULL; -} - -unsigned char h_gf_mul(unsigned char a, unsigned char b) { - return d_gf_mul(a, b); -} - -unsigned char h_gf_inv(unsigned char a) { - return d_gf_inv(a); -} - -void h_gf_gen_rs_matrix(unsigned char *a, int m, int k) { - d_gf_gen_rs_matrix(a, m, k); -} - -void h_gf_gen_cauchy_matrix(unsigned char *a, int m, int k) { - d_gf_gen_cauchy_matrix(a, m, k); -} - -int h_gf_invert_matrix(unsigned char *in, unsigned char *out, const int n) { - return d_gf_invert_matrix(in, out, n); -} - -int h_gf_vect_mul(int len, unsigned char *gftbl, void *src, void *dest) { - return d_gf_vect_mul(len, gftbl, src, dest); -} - void h_ec_init_tables(int k, int rows, unsigned char* a, unsigned char* gftbls) { - d_ec_init_tables(k, rows, a, gftbls); + isaLoader->ec_init_tables(k, rows, a, gftbls); } void h_ec_encode_data(int len, int k, int rows, unsigned char *gftbls, unsigned char **data, unsigned char **coding) { - d_ec_encode_data(len, k, rows, gftbls, data, coding); + isaLoader->ec_encode_data(len, k, rows, gftbls, data, coding); } void h_ec_encode_data_update(int len, int k, int rows, int vec_i, unsigned char *gftbls, unsigned char *data, unsigned char **coding) { - d_ec_encode_data_update(len, k, rows, vec_i, gftbls, data, coding); + isaLoader->ec_encode_data_update(len, k, rows, vec_i, gftbls, data, coding); } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/658ee95f/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/erasure_code.h ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/erasure_code.h b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/erasure_code.h new file mode 100644 index 0000000..a6cfd58 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/erasure_code.h @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef _ERASURE_CODE_H_ +#define _ERASURE_CODE_H_ + +#include <stddef.h> + +/** + * Interface to functions supporting erasure code encode and decode. + * + * This file defines the interface to optimized functions used in erasure + * codes. Encode and decode of erasures in GF(2^8) are made by calculating the + * dot product of the symbols (bytes in GF(2^8)) across a set of buffers and a + * set of coefficients. Values for the coefficients are determined by the type + * of erasure code. Using a general dot product means that any sequence of + * coefficients may be used including erasure codes based on random + * coefficients. + * Multiple versions of dot product are supplied to calculate 1-6 output + * vectors in one pass. + * Base GF multiply and divide functions can be sped up by defining + * GF_LARGE_TABLES at the expense of memory size. + * + */ + +/** + * Initialize tables for fast Erasure Code encode and decode. + * + * Generates the expanded tables needed for fast encode or decode for erasure + * codes on blocks of data. 32bytes is generated for each input coefficient. + * + * @param k The number of vector sources or rows in the generator matrix + * for coding. + * @param rows The number of output vectors to concurrently encode/decode. + * @param a Pointer to sets of arrays of input coefficients used to encode + * or decode data. + * @param gftbls Pointer to start of space for concatenated output tables + * generated from input coefficients. Must be of size 32*k*rows. + * @returns none + */ +void h_ec_init_tables(int k, int rows, unsigned char* a, unsigned char* gftbls); + +/** + * Generate or decode erasure codes on blocks of data, runs appropriate version. + * + * Given a list of source data blocks, generate one or multiple blocks of + * encoded data as specified by a matrix of GF(2^8) coefficients. When given a + * suitable set of coefficients, this function will perform the fast generation + * or decoding of Reed-Solomon type erasure codes. + * + * This function determines what instruction sets are enabled and + * selects the appropriate version at runtime. + * + * @param len Length of each block of data (vector) of source or dest data. + * @param k The number of vector sources or rows in the generator matrix + * for coding. + * @param rows The number of output vectors to concurrently encode/decode. + * @param gftbls Pointer to array of input tables generated from coding + * coefficients in ec_init_tables(). Must be of size 32*k*rows + * @param data Array of pointers to source input buffers. + * @param coding Array of pointers to coded output buffers. + * @returns none + */ +void h_ec_encode_data(int len, int k, int rows, unsigned char *gftbls, + unsigned char **data, unsigned char **coding); + +/** + * @brief Generate update for encode or decode of erasure codes from single + * source, runs appropriate version. + * + * Given one source data block, update one or multiple blocks of encoded data as + * specified by a matrix of GF(2^8) coefficients. When given a suitable set of + * coefficients, this function will perform the fast generation or decoding of + * Reed-Solomon type erasure codes from one input source at a time. + * + * This function determines what instruction sets are enabled and selects the + * appropriate version at runtime. + * + * @param len Length of each block of data (vector) of source or dest data. + * @param k The number of vector sources or rows in the generator matrix + * for coding. + * @param rows The number of output vectors to concurrently encode/decode. + * @param vec_i The vector index corresponding to the single input source. + * @param gftbls Pointer to array of input tables generated from coding + * coefficients in ec_init_tables(). Must be of size 32*k*rows + * @param data Pointer to single input source used to update output parity. + * @param coding Array of pointers to coded output buffers. + * @returns none + */ +void h_ec_encode_data_update(int len, int k, int rows, int vec_i, + unsigned char *gftbls, unsigned char *data, unsigned char **coding); + +#endif //_ERASURE_CODE_H_ http://git-wip-us.apache.org/repos/asf/hadoop/blob/658ee95f/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/erasure_coder.c ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/erasure_coder.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/erasure_coder.c new file mode 100644 index 0000000..b3479bb --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/erasure_coder.c @@ -0,0 +1,229 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "erasure_code.h" +#include "gf_util.h" +#include "erasure_coder.h" + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +void initCoder(IsalCoder* pCoder, int numDataUnits, int numParityUnits) { + pCoder->verbose = 0; + pCoder->numParityUnits = numParityUnits; + pCoder->numDataUnits = numDataUnits; + pCoder->numAllUnits = numDataUnits + numParityUnits; +} + +// 0 not to verbose, 1 to verbose +void allowVerbose(IsalCoder* pCoder, int flag) { + pCoder->verbose = flag; +} + +static void initEncodeMatrix(int numDataUnits, int numParityUnits, + unsigned char* encodeMatrix) { + // Generate encode matrix, always invertible + h_gf_gen_cauchy_matrix(encodeMatrix, + numDataUnits + numParityUnits, numDataUnits); +} + +void initEncoder(IsalEncoder* pCoder, int numDataUnits, + int numParityUnits) { + initCoder(&pCoder->coder, numDataUnits, numParityUnits); + + initEncodeMatrix(numDataUnits, numParityUnits, pCoder->encodeMatrix); + + // Generate gftbls from encode matrix + h_ec_init_tables(numDataUnits, numParityUnits, + &pCoder->encodeMatrix[numDataUnits * numDataUnits], + pCoder->gftbls); + + if (pCoder->coder.verbose > 0) { + dumpEncoder(pCoder); + } +} + +void initDecoder(IsalDecoder* pCoder, int numDataUnits, + int numParityUnits) { + initCoder(&pCoder->coder, numDataUnits, numParityUnits); + + initEncodeMatrix(numDataUnits, numParityUnits, pCoder->encodeMatrix); +} + +int encode(IsalEncoder* pCoder, unsigned char** dataUnits, + unsigned char** parityUnits, int chunkSize) { + int numDataUnits = pCoder->coder.numDataUnits; + int numParityUnits = pCoder->coder.numParityUnits; + int i; + + for (i = 0; i < numParityUnits; i++) { + memset(parityUnits[i], 0, chunkSize); + } + + h_ec_encode_data(chunkSize, numDataUnits, numParityUnits, + pCoder->gftbls, dataUnits, parityUnits); + + return 0; +} + +// Return 1 when diff, 0 otherwise +static int compare(int* arr1, int len1, int* arr2, int len2) { + int i; + + if (len1 == len2) { + for (i = 0; i < len1; i++) { + if (arr1[i] != arr2[i]) { + return 1; + } + } + return 0; + } + + return 1; +} + +static int processErasures(IsalDecoder* pCoder, unsigned char** inputs, + int* erasedIndexes, int numErased) { + int i, r, ret, index; + int numDataUnits = pCoder->coder.numDataUnits; + int isChanged = 0; + + for (i = 0, r = 0; i < numDataUnits; i++, r++) { + while (inputs[r] == NULL) { + r++; + } + + if (pCoder->decodeIndex[i] != r) { + pCoder->decodeIndex[i] = r; + isChanged = 1; + } + } + + for (i = 0; i < numDataUnits; i++) { + pCoder->realInputs[i] = inputs[pCoder->decodeIndex[i]]; + } + + if (isChanged == 0 && + compare(pCoder->erasedIndexes, pCoder->numErased, + erasedIndexes, numErased) == 0) { + return 0; // Optimization, nothing to do + } + + clearDecoder(pCoder); + + for (i = 0; i < numErased; i++) { + index = erasedIndexes[i]; + pCoder->erasedIndexes[i] = index; + pCoder->erasureFlags[index] = 1; + if (index < numDataUnits) { + pCoder->numErasedDataUnits++; + } + } + + pCoder->numErased = numErased; + + ret = generateDecodeMatrix(pCoder); + if (ret != 0) { + printf("Failed to generate decode matrix\n"); + return -1; + } + + h_ec_init_tables(numDataUnits, pCoder->numErased, + pCoder->decodeMatrix, pCoder->gftbls); + + if (pCoder->coder.verbose > 0) { + dumpDecoder(pCoder); + } + + return 0; +} + +int decode(IsalDecoder* pCoder, unsigned char** inputs, + int* erasedIndexes, int numErased, + unsigned char** outputs, int chunkSize) { + int numDataUnits = pCoder->coder.numDataUnits; + int i; + + processErasures(pCoder, inputs, erasedIndexes, numErased); + + for (i = 0; i < numErased; i++) { + memset(outputs[i], 0, chunkSize); + } + + h_ec_encode_data(chunkSize, numDataUnits, pCoder->numErased, + pCoder->gftbls, pCoder->realInputs, outputs); + + return 0; +} + +// Clear variables used per decode call +void clearDecoder(IsalDecoder* decoder) { + decoder->numErasedDataUnits = 0; + decoder->numErased = 0; + memset(decoder->gftbls, 0, sizeof(decoder->gftbls)); + memset(decoder->decodeMatrix, 0, sizeof(decoder->decodeMatrix)); + memset(decoder->tmpMatrix, 0, sizeof(decoder->tmpMatrix)); + memset(decoder->invertMatrix, 0, sizeof(decoder->invertMatrix)); + memset(decoder->erasureFlags, 0, sizeof(decoder->erasureFlags)); + memset(decoder->erasedIndexes, 0, sizeof(decoder->erasedIndexes)); +} + +// Generate decode matrix from encode matrix +int generateDecodeMatrix(IsalDecoder* pCoder) { + int i, j, r, p; + unsigned char s; + int numDataUnits; + + numDataUnits = pCoder->coder.numDataUnits; + + // Construct matrix b by removing error rows + for (i = 0; i < numDataUnits; i++) { + r = pCoder->decodeIndex[i]; + for (j = 0; j < numDataUnits; j++) { + pCoder->tmpMatrix[numDataUnits * i + j] = + pCoder->encodeMatrix[numDataUnits * r + j]; + } + } + + h_gf_invert_matrix(pCoder->tmpMatrix, + pCoder->invertMatrix, numDataUnits); + + for (i = 0; i < pCoder->numErasedDataUnits; i++) { + for (j = 0; j < numDataUnits; j++) { + pCoder->decodeMatrix[numDataUnits * i + j] = + pCoder->invertMatrix[numDataUnits * + pCoder->erasedIndexes[i] + j]; + } + } + + for (p = pCoder->numErasedDataUnits; p < pCoder->numErased; p++) { + for (i = 0; i < numDataUnits; i++) { + s = 0; + for (j = 0; j < numDataUnits; j++) { + s ^= h_gf_mul(pCoder->invertMatrix[j * numDataUnits + i], + pCoder->encodeMatrix[numDataUnits * + pCoder->erasedIndexes[p] + j]); + } + + pCoder->decodeMatrix[numDataUnits * p + i] = s; + } + } + + return 0; +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/658ee95f/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/erasure_coder.h ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/erasure_coder.h b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/erasure_coder.h new file mode 100644 index 0000000..8f5bf8a --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/erasure_coder.h @@ -0,0 +1,88 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * This is a sample program illustrating how to use the Intel ISA-L library. + * Note it's adapted from erasure_code_test.c test program, but trying to use + * variable names and styles we're more familiar with already similar to Java + * coders. + */ + +#ifndef _ERASURE_CODER_H_ +#define _ERASURE_CODER_H_ + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#define MMAX 14 +#define KMAX 10 + +typedef struct _IsalCoder { + int verbose; + int numParityUnits; + int numDataUnits; + int numAllUnits; +} IsalCoder; + +typedef struct _IsalEncoder { + IsalCoder coder; + + unsigned char gftbls[MMAX * KMAX * 32]; + + unsigned char encodeMatrix[MMAX * KMAX]; +} IsalEncoder; + +typedef struct _IsalDecoder { + IsalCoder coder; + + unsigned char encodeMatrix[MMAX * KMAX]; + + // Below are per decode call + unsigned char gftbls[MMAX * KMAX * 32]; + unsigned int decodeIndex[MMAX]; + unsigned char tmpMatrix[MMAX * KMAX]; + unsigned char invertMatrix[MMAX * KMAX]; + unsigned char decodeMatrix[MMAX * KMAX]; + unsigned char erasureFlags[MMAX]; + int erasedIndexes[MMAX]; + int numErased; + int numErasedDataUnits; + unsigned char* realInputs[MMAX]; +} IsalDecoder; + +void initCoder(IsalCoder* pCoder, int numDataUnits, int numParityUnits); + +void allowVerbose(IsalCoder* pCoder, int flag); + +void initEncoder(IsalEncoder* encoder, int numDataUnits, int numParityUnits); + +void initDecoder(IsalDecoder* decoder, int numDataUnits, int numParityUnits); + +void clearDecoder(IsalDecoder* decoder); + +int encode(IsalEncoder* encoder, unsigned char** dataUnits, + unsigned char** parityUnits, int chunkSize); + +int decode(IsalDecoder* decoder, unsigned char** allUnits, + int* erasedIndexes, int numErased, + unsigned char** recoveredUnits, int chunkSize); + +int generateDecodeMatrix(IsalDecoder* pCoder); + +#endif //_ERASURE_CODER_H_ http://git-wip-us.apache.org/repos/asf/hadoop/blob/658ee95f/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/gf_util.c ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/gf_util.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/gf_util.c new file mode 100644 index 0000000..804dcec --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/gf_util.c @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <errno.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include "isal_load.h" +#include "gf_util.h" + +/** + * gf_util.c + * Implementation GF utilities based on ISA-L library. + * + */ + +unsigned char h_gf_mul(unsigned char a, unsigned char b) { + return isaLoader->gf_mul(a, b); +} + +unsigned char h_gf_inv(unsigned char a) { + return isaLoader->gf_inv(a); +} + +void h_gf_gen_rs_matrix(unsigned char *a, int m, int k) { + isaLoader->gf_gen_rs_matrix(a, m, k); +} + +void h_gf_gen_cauchy_matrix(unsigned char *a, int m, int k) { + isaLoader->gf_gen_cauchy_matrix(a, m, k); +} + +int h_gf_invert_matrix(unsigned char *in, unsigned char *out, const int n) { + return isaLoader->gf_invert_matrix(in, out, n); +} + +int h_gf_vect_mul(int len, unsigned char *gftbl, void *src, void *dest) { + return isaLoader->gf_vect_mul(len, gftbl, src, dest); +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/658ee95f/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/gf_util.h ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/gf_util.h b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/gf_util.h new file mode 100644 index 0000000..2be8328 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/gf_util.h @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef _GF_UTIL_H +#define _GF_UTIL_H + +/** + * gf_util.h + * Interface to functions for vector (block) multiplication in GF(2^8). + * + * This file defines the interface to routines used in fast RAID rebuild and + * erasure codes. + */ + + +/** + * Single element GF(2^8) multiply. + * + * @param a Multiplicand a + * @param b Multiplicand b + * @returns Product of a and b in GF(2^8) + */ +unsigned char h_gf_mul(unsigned char a, unsigned char b); + +/** + * Single element GF(2^8) inverse. + * + * @param a Input element + * @returns Field element b such that a x b = {1} + */ +unsigned char h_gf_inv(unsigned char a); + +/** + * Generate a matrix of coefficients to be used for encoding. + * + * Vandermonde matrix example of encoding coefficients where high portion of + * matrix is identity matrix I and lower portion is constructed as 2^{i*(j-k+1)} + * i:{0,k-1} j:{k,m-1}. Commonly used method for choosing coefficients in + * erasure encoding but does not guarantee invertable for every sub matrix. For + * large k it is possible to find cases where the decode matrix chosen from + * sources and parity not in erasure are not invertable. Users may want to + * adjust for k > 5. + * + * @param a [mxk] array to hold coefficients + * @param m number of rows in matrix corresponding to srcs + parity. + * @param k number of columns in matrix corresponding to srcs. + * @returns none + */ +void h_gf_gen_rs_matrix(unsigned char *a, int m, int k); + +/** + * Generate a Cauchy matrix of coefficients to be used for encoding. + * + * Cauchy matrix example of encoding coefficients where high portion of matrix + * is identity matrix I and lower portion is constructed as 1/(i + j) | i != j, + * i:{0,k-1} j:{k,m-1}. Any sub-matrix of a Cauchy matrix should be invertable. + * + * @param a [mxk] array to hold coefficients + * @param m number of rows in matrix corresponding to srcs + parity. + * @param k number of columns in matrix corresponding to srcs. + * @returns none + */ +void h_gf_gen_cauchy_matrix(unsigned char *a, int m, int k); + +/** + * Invert a matrix in GF(2^8) + * + * @param in input matrix + * @param out output matrix such that [in] x [out] = [I] - identity matrix + * @param n size of matrix [nxn] + * @returns 0 successful, other fail on singular input matrix + */ +int h_gf_invert_matrix(unsigned char *in, unsigned char *out, const int n); + +/** + * GF(2^8) vector multiply by constant, runs appropriate version. + * + * Does a GF(2^8) vector multiply b = Ca where a and b are arrays and C + * is a single field element in GF(2^8). Can be used for RAID6 rebuild + * and partial write functions. Function requires pre-calculation of a + * 32-element constant array based on constant C. gftbl(C) = {C{00}, + * C{01}, C{02}, ... , C{0f} }, {C{00}, C{10}, C{20}, ... , C{f0} }. + * Len and src must be aligned to 32B. + * + * This function determines what instruction sets are enabled + * and selects the appropriate version at runtime. + * + * @param len Length of vector in bytes. Must be aligned to 32B. + * @param gftbl Pointer to 32-byte array of pre-calculated constants based on C. + * @param src Pointer to src data array. Must be aligned to 32B. + * @param dest Pointer to destination data array. Must be aligned to 32B. + * @returns 0 pass, other fail + */ +int h_gf_vect_mul(int len, unsigned char *gftbl, void *src, void *dest); + + +#endif //_GF_UTIL_H http://git-wip-us.apache.org/repos/asf/hadoop/blob/658ee95f/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/include/erasure_code.h ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/include/erasure_code.h b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/include/erasure_code.h deleted file mode 100644 index 123085e..0000000 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/include/erasure_code.h +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef _ERASURE_CODE_H_ -#define _ERASURE_CODE_H_ - -#include <stddef.h> - -/** - * Interface to functions supporting erasure code encode and decode. - * - * This file defines the interface to optimized functions used in erasure - * codes. Encode and decode of erasures in GF(2^8) are made by calculating the - * dot product of the symbols (bytes in GF(2^8)) across a set of buffers and a - * set of coefficients. Values for the coefficients are determined by the type - * of erasure code. Using a general dot product means that any sequence of - * coefficients may be used including erasure codes based on random - * coefficients. - * Multiple versions of dot product are supplied to calculate 1-6 output - * vectors in one pass. - * Base GF multiply and divide functions can be sped up by defining - * GF_LARGE_TABLES at the expense of memory size. - * - */ - -/** - * Return 0 if not support, 1 otherwise. - */ -int build_support_erasurecode(); - -/** - * Get the library name possibly of full path. - */ -const char* get_library_name(); - -/** - * Initialize and load erasure code library, returning error message if any. - * - * @param err The err message buffer. - * @param err_len The length of the message buffer. - */ -void load_erasurecode_lib(char* err, size_t err_len); - -/** - * Initialize tables for fast Erasure Code encode and decode. - * - * Generates the expanded tables needed for fast encode or decode for erasure - * codes on blocks of data. 32bytes is generated for each input coefficient. - * - * @param k The number of vector sources or rows in the generator matrix - * for coding. - * @param rows The number of output vectors to concurrently encode/decode. - * @param a Pointer to sets of arrays of input coefficients used to encode - * or decode data. - * @param gftbls Pointer to start of space for concatenated output tables - * generated from input coefficients. Must be of size 32*k*rows. - * @returns none - */ -void h_ec_init_tables(int k, int rows, unsigned char* a, unsigned char* gftbls); - -/** - * Generate or decode erasure codes on blocks of data, runs appropriate version. - * - * Given a list of source data blocks, generate one or multiple blocks of - * encoded data as specified by a matrix of GF(2^8) coefficients. When given a - * suitable set of coefficients, this function will perform the fast generation - * or decoding of Reed-Solomon type erasure codes. - * - * This function determines what instruction sets are enabled and - * selects the appropriate version at runtime. - * - * @param len Length of each block of data (vector) of source or dest data. - * @param k The number of vector sources or rows in the generator matrix - * for coding. - * @param rows The number of output vectors to concurrently encode/decode. - * @param gftbls Pointer to array of input tables generated from coding - * coefficients in ec_init_tables(). Must be of size 32*k*rows - * @param data Array of pointers to source input buffers. - * @param coding Array of pointers to coded output buffers. - * @returns none - */ -void h_ec_encode_data(int len, int k, int rows, unsigned char *gftbls, - unsigned char **data, unsigned char **coding); - -/** - * @brief Generate update for encode or decode of erasure codes from single - * source, runs appropriate version. - * - * Given one source data block, update one or multiple blocks of encoded data as - * specified by a matrix of GF(2^8) coefficients. When given a suitable set of - * coefficients, this function will perform the fast generation or decoding of - * Reed-Solomon type erasure codes from one input source at a time. - * - * This function determines what instruction sets are enabled and selects the - * appropriate version at runtime. - * - * @param len Length of each block of data (vector) of source or dest data. - * @param k The number of vector sources or rows in the generator matrix - * for coding. - * @param rows The number of output vectors to concurrently encode/decode. - * @param vec_i The vector index corresponding to the single input source. - * @param gftbls Pointer to array of input tables generated from coding - * coefficients in ec_init_tables(). Must be of size 32*k*rows - * @param data Pointer to single input source used to update output parity. - * @param coding Array of pointers to coded output buffers. - * @returns none - */ -void h_ec_encode_data_update(int len, int k, int rows, int vec_i, - unsigned char *gftbls, unsigned char *data, unsigned char **coding); - -#endif //_ERASURE_CODE_H_ http://git-wip-us.apache.org/repos/asf/hadoop/blob/658ee95f/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/include/gf_util.h ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/include/gf_util.h b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/include/gf_util.h deleted file mode 100644 index 2be8328..0000000 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/include/gf_util.h +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef _GF_UTIL_H -#define _GF_UTIL_H - -/** - * gf_util.h - * Interface to functions for vector (block) multiplication in GF(2^8). - * - * This file defines the interface to routines used in fast RAID rebuild and - * erasure codes. - */ - - -/** - * Single element GF(2^8) multiply. - * - * @param a Multiplicand a - * @param b Multiplicand b - * @returns Product of a and b in GF(2^8) - */ -unsigned char h_gf_mul(unsigned char a, unsigned char b); - -/** - * Single element GF(2^8) inverse. - * - * @param a Input element - * @returns Field element b such that a x b = {1} - */ -unsigned char h_gf_inv(unsigned char a); - -/** - * Generate a matrix of coefficients to be used for encoding. - * - * Vandermonde matrix example of encoding coefficients where high portion of - * matrix is identity matrix I and lower portion is constructed as 2^{i*(j-k+1)} - * i:{0,k-1} j:{k,m-1}. Commonly used method for choosing coefficients in - * erasure encoding but does not guarantee invertable for every sub matrix. For - * large k it is possible to find cases where the decode matrix chosen from - * sources and parity not in erasure are not invertable. Users may want to - * adjust for k > 5. - * - * @param a [mxk] array to hold coefficients - * @param m number of rows in matrix corresponding to srcs + parity. - * @param k number of columns in matrix corresponding to srcs. - * @returns none - */ -void h_gf_gen_rs_matrix(unsigned char *a, int m, int k); - -/** - * Generate a Cauchy matrix of coefficients to be used for encoding. - * - * Cauchy matrix example of encoding coefficients where high portion of matrix - * is identity matrix I and lower portion is constructed as 1/(i + j) | i != j, - * i:{0,k-1} j:{k,m-1}. Any sub-matrix of a Cauchy matrix should be invertable. - * - * @param a [mxk] array to hold coefficients - * @param m number of rows in matrix corresponding to srcs + parity. - * @param k number of columns in matrix corresponding to srcs. - * @returns none - */ -void h_gf_gen_cauchy_matrix(unsigned char *a, int m, int k); - -/** - * Invert a matrix in GF(2^8) - * - * @param in input matrix - * @param out output matrix such that [in] x [out] = [I] - identity matrix - * @param n size of matrix [nxn] - * @returns 0 successful, other fail on singular input matrix - */ -int h_gf_invert_matrix(unsigned char *in, unsigned char *out, const int n); - -/** - * GF(2^8) vector multiply by constant, runs appropriate version. - * - * Does a GF(2^8) vector multiply b = Ca where a and b are arrays and C - * is a single field element in GF(2^8). Can be used for RAID6 rebuild - * and partial write functions. Function requires pre-calculation of a - * 32-element constant array based on constant C. gftbl(C) = {C{00}, - * C{01}, C{02}, ... , C{0f} }, {C{00}, C{10}, C{20}, ... , C{f0} }. - * Len and src must be aligned to 32B. - * - * This function determines what instruction sets are enabled - * and selects the appropriate version at runtime. - * - * @param len Length of vector in bytes. Must be aligned to 32B. - * @param gftbl Pointer to 32-byte array of pre-calculated constants based on C. - * @param src Pointer to src data array. Must be aligned to 32B. - * @param dest Pointer to destination data array. Must be aligned to 32B. - * @returns 0 pass, other fail - */ -int h_gf_vect_mul(int len, unsigned char *gftbl, void *src, void *dest); - - -#endif //_GF_UTIL_H http://git-wip-us.apache.org/repos/asf/hadoop/blob/658ee95f/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/isal_load.c ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/isal_load.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/isal_load.c new file mode 100644 index 0000000..55e8efd --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/isal_load.c @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <errno.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include "org_apache_hadoop.h" +#include "isal_load.h" + +#ifdef UNIX +#include <sys/time.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <dlfcn.h> + +#include "config.h" +#endif + +#ifdef WINDOWS +#include <Windows.h> +#endif + +IsaLibLoader* isaLoader; + +/** + * isal_load.c + * Utility of loading the ISA-L library and the required functions. + * Building of this codes won't rely on any ISA-L source codes, but running + * into this will rely on successfully loading of the dynamic library. + * + */ + +static const char* load_functions() { +#ifdef UNIX + EC_LOAD_DYNAMIC_SYMBOL((isaLoader->gf_mul), "gf_mul"); + EC_LOAD_DYNAMIC_SYMBOL((isaLoader->gf_inv), "gf_inv"); + EC_LOAD_DYNAMIC_SYMBOL((isaLoader->gf_gen_rs_matrix), "gf_gen_rs_matrix"); + EC_LOAD_DYNAMIC_SYMBOL((isaLoader->gf_gen_cauchy_matrix), "gf_gen_cauchy1_matrix"); + EC_LOAD_DYNAMIC_SYMBOL((isaLoader->gf_invert_matrix), "gf_invert_matrix"); + EC_LOAD_DYNAMIC_SYMBOL((isaLoader->gf_vect_mul), "gf_vect_mul"); + + EC_LOAD_DYNAMIC_SYMBOL((isaLoader->ec_init_tables), "ec_init_tables"); + EC_LOAD_DYNAMIC_SYMBOL((isaLoader->ec_encode_data), "ec_encode_data"); + EC_LOAD_DYNAMIC_SYMBOL((isaLoader->ec_encode_data_update), "ec_encode_data_update"); +#endif + +#ifdef WINDOWS + EC_LOAD_DYNAMIC_SYMBOL(__d_gf_mul, (isaLoader->gf_mul), "gf_mul"); + EC_LOAD_DYNAMIC_SYMBOL(__d_gf_inv, (isaLoader->gf_inv), "gf_inv"); + EC_LOAD_DYNAMIC_SYMBOL(__d_gf_gen_rs_matrix, (isaLoader->gf_gen_rs_matrix), "gf_gen_rs_matrix"); + EC_LOAD_DYNAMIC_SYMBOL(__d_gf_gen_cauchy_matrix, (isaLoader->gf_gen_cauchy_matrix), "gf_gen_cauchy1_matrix"); + EC_LOAD_DYNAMIC_SYMBOL(__d_gf_invert_matrix, (isaLoader->gf_invert_matrix), "gf_invert_matrix"); + EC_LOAD_DYNAMIC_SYMBOL(__d_gf_vect_mul, (isaLoader->gf_vect_mul), "gf_vect_mul"); + + EC_LOAD_DYNAMIC_SYMBOL(__d_ec_init_tables, (isaLoader->ec_init_tables), "ec_init_tables"); + EC_LOAD_DYNAMIC_SYMBOL(__d_ec_encode_data, (isaLoader->ec_encode_data), "ec_encode_data"); + EC_LOAD_DYNAMIC_SYMBOL(__d_ec_encode_data_update, (isaLoader->ec_encode_data_update), "ec_encode_data_update"); +#endif + + return NULL; +} + +void load_erasurecode_lib(char* err, size_t err_len) { + const char* errMsg; + + err[0] = '\0'; + + if (isaLoader != NULL) { + return; + } + isaLoader = calloc(1, sizeof(IsaLibLoader)); + memset(isaLoader, 0, sizeof(IsaLibLoader)); + + // Load Intel ISA-L + #ifdef UNIX + isaLoader->libec = dlopen(HADOOP_ISAL_LIBRARY, RTLD_LAZY | RTLD_GLOBAL); + if (isaLoader->libec == NULL) { + snprintf(err, err_len, "Failed to load %s (%s)", + HADOOP_ISAL_LIBRARY, dlerror()); + return; + } + // Clear any existing error + dlerror(); + #endif + + #ifdef WINDOWS + isaLoader->libec = LoadLibrary(HADOOP_ISAL_LIBRARY); + if (isaLoader->libec == NULL) { + snprintf(err, err_len, "Failed to load %s", HADOOP_ISAL_LIBRARY); + return; + } + #endif + + errMsg = load_functions(isaLoader->libec); + if (errMsg != NULL) { + snprintf(err, err_len, "Loading functions from ISA-L failed: %s", errMsg); + } +} + +int build_support_erasurecode() { +#ifdef HADOOP_ISAL_LIBRARY + return 1; +#else + return 0; +#endif +} + +const char* get_library_name() { +#ifdef UNIX + Dl_info dl_info; + + if (isaLoader->ec_encode_data == NULL) { + return HADOOP_ISAL_LIBRARY; + } + + if(dladdr(isaLoader->ec_encode_data, &dl_info)) { + return dl_info.dli_fname; + } +#else + LPTSTR filename = NULL; + + if (isaLoader->libec == NULL) { + return HADOOP_ISAL_LIBRARY; + } + + if (GetModuleFileName(isaLoader->libec, filename, 256) > 0) { + return filename; + } +#endif + + return NULL; +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/658ee95f/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/isal_load.h ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/isal_load.h b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/isal_load.h new file mode 100644 index 0000000..635706d --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/isal_load.h @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <errno.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include "org_apache_hadoop.h" + +#ifdef UNIX +#include <sys/time.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <dlfcn.h> + +#include "config.h" +#endif + +#ifdef WINDOWS +#include <Windows.h> +#endif + +#ifndef _ISAL_LOAD_H_ +#define _ISAL_LOAD_H_ + + +#ifdef UNIX +// For gf_util.h +typedef unsigned char (*__d_gf_mul)(unsigned char, unsigned char); +typedef unsigned char (*__d_gf_inv)(unsigned char); +typedef void (*__d_gf_gen_rs_matrix)(unsigned char *, int, int); +typedef void (*__d_gf_gen_cauchy_matrix)(unsigned char *, int, int); +typedef int (*__d_gf_invert_matrix)(unsigned char *, unsigned char *, const int); +typedef int (*__d_gf_vect_mul)(int, unsigned char *, void *, void *); + +// For erasure_code.h +typedef void (*__d_ec_init_tables)(int, int, unsigned char*, unsigned char*); +typedef void (*__d_ec_encode_data)(int, int, int, unsigned char*, + unsigned char**, unsigned char**); +typedef void (*__d_ec_encode_data_update)(int, int, int, int, unsigned char*, + unsigned char*, unsigned char**); +#endif + +#ifdef WINDOWS +// For erasure_code.h +typedef unsigned char (__cdecl *__d_gf_mul)(unsigned char, unsigned char); +typedef unsigned char (__cdecl *__d_gf_inv)(unsigned char); +typedef void (__cdecl *__d_gf_gen_rs_matrix)(unsigned char *, int, int); +typedef void (__cdecl *__d_gf_gen_cauchy_matrix)(unsigned char *, int, int); +typedef int (__cdecl *__d_gf_invert_matrix)(unsigned char *, + unsigned char *, const int); +typedef int (__cdecl *__d_gf_vect_mul)(int, unsigned char *, void *, void *); + +// For erasure_code.h +typedef void (__cdecl *__d_ec_init_tables)(int, int, + unsigned char*, unsigned char*); +typedef void (__cdecl *__d_ec_encode_data)(int, int, int, unsigned char*, + unsigned char**, unsigned char**); +typedef void (__cdecl *__d_ec_encode_data_update)(int, int, int, int, unsigned char*, + unsigned char*, unsigned char**); +#endif + +typedef struct __IsaLibLoader { + // The loaded library handle + void* libec; + + __d_gf_mul gf_mul; + __d_gf_inv gf_inv; + __d_gf_gen_rs_matrix gf_gen_rs_matrix; + __d_gf_gen_cauchy_matrix gf_gen_cauchy_matrix; + __d_gf_invert_matrix gf_invert_matrix; + __d_gf_vect_mul gf_vect_mul; + __d_ec_init_tables ec_init_tables; + __d_ec_encode_data ec_encode_data; + __d_ec_encode_data_update ec_encode_data_update; +} IsaLibLoader; + +extern IsaLibLoader* isaLoader; + +/** + * A helper function to dlsym a 'symbol' from a given library-handle. + */ + +#ifdef UNIX + +static __attribute__ ((unused)) +void *myDlsym(void *handle, const char *symbol) { + void *func_ptr = dlsym(handle, symbol); + return func_ptr; +} + +/* A helper macro to dlsym the requisite dynamic symbol in NON-JNI env. */ +#define EC_LOAD_DYNAMIC_SYMBOL(func_ptr, symbol) \ + if ((func_ptr = myDlsym(isaLoader->libec, symbol)) == NULL) { \ + return "Failed to load symbol" symbol; \ + } + +#endif + +#ifdef WINDOWS + + +static FARPROC WINAPI myDlsym(HMODULE handle, LPCSTR symbol) { + FARPROC func_ptr = GetProcAddress(handle, symbol); + return func_ptr; +} + +/* A helper macro to dlsym the requisite dynamic symbol in NON-JNI env. */ +#define EC_LOAD_DYNAMIC_SYMBOL(func_type, func_ptr, symbol) \ + if ((func_ptr = (func_type)myDlsym(isaLoader->libec, symbol)) == NULL) { \ + return "Failed to load symbol" symbol; \ + } + +#endif + +/** + * Return 0 if not support, 1 otherwise. + */ +int build_support_erasurecode(); + +/** + * Get the library name possibly of full path. + */ +const char* get_library_name(); + +/** + * Initialize and load erasure code library, returning error message if any. + * + * @param err The err message buffer. + * @param err_len The length of the message buffer. + */ +void load_erasurecode_lib(char* err, size_t err_len); + +#endif //_ISAL_LOAD_H_ \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/658ee95f/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_common.c ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_common.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_common.c new file mode 100644 index 0000000..8126e9a --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_common.c @@ -0,0 +1,98 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include "org_apache_hadoop.h" +#include "isal_load.h" +#include "erasure_code.h" +#include "jni_common.h" + +void loadLib(JNIEnv *env) { + char errMsg[1024]; + load_erasurecode_lib(errMsg, sizeof(errMsg)); + if (strlen(errMsg) > 0) { + THROW(env, "java/lang/UnsatisfiedLinkError", errMsg); + } +} + +void setCoder(JNIEnv* env, jobject thiz, IsalCoder* pCoder) { + jclass clazz = (*env)->GetObjectClass(env, thiz); + jfieldID __coderState = (*env)->GetFieldID(env, clazz, "__native_coder", "J"); + (*env)->SetLongField(env, thiz, __coderState, (jlong) pCoder); +} + +IsalCoder* getCoder(JNIEnv* env, jobject thiz) { + jclass clazz = (*env)->GetObjectClass(env, thiz); + + jfieldID __verbose = (*env)->GetFieldID(env, clazz, "__native_verbose", "J"); + int verbose = (int)(*env)->GetIntField(env, thiz, __verbose); + + jfieldID __coderState = (*env)->GetFieldID(env, clazz, "__native_coder", "J"); + IsalCoder* pCoder = (IsalCoder*)(*env)->GetLongField(env, + thiz, __coderState); + pCoder->verbose = verbose; + + return pCoder; +} + +void getInputs(JNIEnv *env, jobjectArray inputs, jintArray inputOffsets, + unsigned char** destInputs, int num) { + int numInputs = (*env)->GetArrayLength(env, inputs); + int* tmpInputOffsets; + int i; + jobject byteBuffer; + + if (numInputs != num) { + THROW(env, "java/lang/InternalError", "Invalid inputs"); + } + + tmpInputOffsets = (int*)(*env)->GetIntArrayElements(env, + inputOffsets, NULL); + for (i = 0; i < numInputs; i++) { + byteBuffer = (*env)->GetObjectArrayElement(env, inputs, i); + if (byteBuffer != NULL) { + destInputs[i] = (unsigned char *)((*env)->GetDirectBufferAddress(env, + byteBuffer)); + destInputs[i] += tmpInputOffsets[i]; + } else { + destInputs[i] = NULL; + } + } +} + +void getOutputs(JNIEnv *env, jobjectArray outputs, jintArray outputOffsets, + unsigned char** destOutputs, int num) { + int numOutputs = (*env)->GetArrayLength(env, outputs); + int i, *tmpOutputOffsets; + jobject byteBuffer; + + if (numOutputs != num) { + THROW(env, "java/lang/InternalError", "Invalid outputs"); + } + + tmpOutputOffsets = (int*)(*env)->GetIntArrayElements(env, + outputOffsets, NULL); + for (i = 0; i < numOutputs; i++) { + byteBuffer = (*env)->GetObjectArrayElement(env, outputs, i); + destOutputs[i] = (unsigned char *)((*env)->GetDirectBufferAddress(env, + byteBuffer)); + destOutputs[i] += tmpOutputOffsets[i]; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/658ee95f/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_common.h ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_common.h b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_common.h new file mode 100644 index 0000000..a9f62a0 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_common.h @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef _JNI_CODER_COMMON_H_ +#define _JNI_CODER_COMMON_H_ + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include <jni.h> + +#include "erasure_coder.h" + +void loadLib(JNIEnv *env); + +void setCoder(JNIEnv* env, jobject thiz, IsalCoder* coder); + +IsalCoder* getCoder(JNIEnv* env, jobject thiz); + +void getInputs(JNIEnv *env, jobjectArray inputs, jintArray inputOffsets, + unsigned char** destInputs, int num); + +void getOutputs(JNIEnv *env, jobjectArray outputs, jintArray outputOffsets, + unsigned char** destOutputs, int num); + +#endif //_JNI_CODER_COMMON_H_ \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/658ee95f/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_erasure_code_native.c ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_erasure_code_native.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_erasure_code_native.c new file mode 100644 index 0000000..eb09e73 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_erasure_code_native.c @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include "org_apache_hadoop.h" +#include "jni_common.h" +#include "org_apache_hadoop_io_erasurecode_ErasureCodeNative.h" + +#ifdef UNIX +#include "config.h" +#endif + +JNIEXPORT void JNICALL +Java_org_apache_hadoop_io_erasurecode_ErasureCodeNative_loadLibrary +(JNIEnv *env, jclass myclass) { + loadLib(env); +} + +JNIEXPORT jstring JNICALL +Java_org_apache_hadoop_io_erasurecode_ErasureCodeNative_getLibraryName +(JNIEnv *env, jclass myclass) { + char* libName = get_library_name(); + if (libName == NULL) { + libName = "Unavailable"; + } + return (*env)->NewStringUTF(env, libName); +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/658ee95f/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_rs_decoder.c ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_rs_decoder.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_rs_decoder.c new file mode 100644 index 0000000..eb4b903 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_rs_decoder.c @@ -0,0 +1,72 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include "org_apache_hadoop.h" +#include "erasure_code.h" +#include "gf_util.h" +#include "jni_common.h" +#include "org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawDecoder.h" + +typedef struct _RSDecoder { + IsalDecoder decoder; + unsigned char* inputs[MMAX]; + unsigned char* outputs[MMAX]; +} RSDecoder; + +JNIEXPORT void JNICALL +Java_org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawDecoder_initImpl( +JNIEnv *env, jobject thiz, jint numDataUnits, jint numParityUnits) { + RSDecoder* rsDecoder = (RSDecoder*)malloc(sizeof(RSDecoder)); + memset(rsDecoder, 0, sizeof(*rsDecoder)); + initDecoder(&rsDecoder->decoder, (int)numDataUnits, (int)numParityUnits); + + setCoder(env, thiz, &rsDecoder->decoder.coder); +} + +JNIEXPORT void JNICALL +Java_org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawDecoder_decodeImpl( +JNIEnv *env, jobject thiz, jobjectArray inputs, jintArray inputOffsets, +jint dataLen, jintArray erasedIndexes, jobjectArray outputs, +jintArray outputOffsets) { + RSDecoder* rsDecoder = (RSDecoder*)getCoder(env, thiz); + + int numDataUnits = rsDecoder->decoder.coder.numDataUnits; + int numParityUnits = rsDecoder->decoder.coder.numParityUnits; + int chunkSize = (int)dataLen; + + int* tmpErasedIndexes = (int*)(*env)->GetIntArrayElements(env, + erasedIndexes, NULL); + int numErased = (*env)->GetArrayLength(env, erasedIndexes); + getInputs(env, inputs, inputOffsets, rsDecoder->inputs, + numDataUnits + numParityUnits); + getOutputs(env, outputs, outputOffsets, rsDecoder->outputs, numErased); + + decode(&rsDecoder->decoder, rsDecoder->inputs, tmpErasedIndexes, + numErased, rsDecoder->outputs, chunkSize); +} + +JNIEXPORT void JNICALL +Java_org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawDecoder_destroyImpl( +JNIEnv *env, jobject thiz) { + RSDecoder* rsDecoder = (RSDecoder*)getCoder(env, thiz); + free(rsDecoder); +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/658ee95f/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_rs_encoder.c ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_rs_encoder.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_rs_encoder.c new file mode 100644 index 0000000..6c477ed --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_rs_encoder.c @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include "org_apache_hadoop.h" +#include "erasure_code.h" +#include "gf_util.h" +#include "jni_common.h" +#include "org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawEncoder.h" + +typedef struct _RSEncoder { + IsalEncoder encoder; + unsigned char* inputs[MMAX]; + unsigned char* outputs[MMAX]; +} RSEncoder; + +JNIEXPORT void JNICALL +Java_org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawEncoder_initImpl( +JNIEnv *env, jobject thiz, jint numDataUnits, jint numParityUnits) { + RSEncoder* rsEncoder = (RSEncoder*)malloc(sizeof(RSEncoder)); + memset(rsEncoder, 0, sizeof(*rsEncoder)); + initEncoder(&rsEncoder->encoder, (int)numDataUnits, (int)numParityUnits); + + setCoder(env, thiz, &rsEncoder->encoder.coder); +} + +JNIEXPORT void JNICALL +Java_org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawEncoder_encodeImpl( +JNIEnv *env, jobject thiz, jobjectArray inputs, jintArray inputOffsets, +jint dataLen, jobjectArray outputs, jintArray outputOffsets) { + RSEncoder* rsEncoder = (RSEncoder*)getCoder(env, thiz); + + int numDataUnits = rsEncoder->encoder.coder.numDataUnits; + int numParityUnits = rsEncoder->encoder.coder.numParityUnits; + int chunkSize = (int)dataLen; + + getInputs(env, inputs, inputOffsets, rsEncoder->inputs, numDataUnits); + getOutputs(env, outputs, outputOffsets, rsEncoder->outputs, numParityUnits); + + encode(&rsEncoder->encoder, rsEncoder->inputs, rsEncoder->outputs, chunkSize); +} + +JNIEXPORT void JNICALL +Java_org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawEncoder_destroyImpl( +JNIEnv *env, jobject thiz) { + RSEncoder* rsEncoder = (RSEncoder*)getCoder(env, thiz); + free(rsEncoder); +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/658ee95f/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/org_apache_hadoop_io_erasurecode_ErasureCodeNative.h ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/org_apache_hadoop_io_erasurecode_ErasureCodeNative.h b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/org_apache_hadoop_io_erasurecode_ErasureCodeNative.h new file mode 100644 index 0000000..d8ff3a0 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/org_apache_hadoop_io_erasurecode_ErasureCodeNative.h @@ -0,0 +1,29 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include <jni.h> +/* Header for class org_apache_hadoop_io_erasurecode_ErasureCodeNative */ + +#ifndef _Included_org_apache_hadoop_io_erasurecode_ErasureCodeNative +#define _Included_org_apache_hadoop_io_erasurecode_ErasureCodeNative +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_apache_hadoop_io_erasurecode_ErasureCodeNative + * Method: loadLibrary + * Signature: ()V + */ +JNIEXPORT void JNICALL Java_org_apache_hadoop_io_erasurecode_ErasureCodeNative_loadLibrary + (JNIEnv *, jclass); + +/* + * Class: org_apache_hadoop_io_erasurecode_ErasureCodeNative + * Method: getLibraryName + * Signature: ()Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_org_apache_hadoop_io_erasurecode_ErasureCodeNative_getLibraryName + (JNIEnv *, jclass); + +#ifdef __cplusplus +} +#endif +#endif
