Repository: hadoop Updated Branches: refs/heads/branch-2 489c920c9 -> f7e6a5dfb
Revert "HADOOP-11887. Introduce Intel ISA-L erasure coding library for native erasure encoding support (Kai Zheng via Colin P. McCabe)" This reverts commit bb3b2977cd3c1ccbf8d861cf264a06c249223dc8. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f7e6a5df Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f7e6a5df Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f7e6a5df Branch: refs/heads/branch-2 Commit: f7e6a5dfb1a2d91e825a0184c9172bd36290b927 Parents: 489c920 Author: Colin Patrick Mccabe <cmcc...@cloudera.com> Authored: Thu Nov 5 11:11:27 2015 -0800 Committer: Colin Patrick Mccabe <cmcc...@cloudera.com> Committed: Thu Nov 5 11:11:27 2015 -0800 ---------------------------------------------------------------------- BUILDING.txt | 25 -- hadoop-common-project/hadoop-common/CHANGES.txt | 3 - hadoop-common-project/hadoop-common/pom.xml | 21 +- .../hadoop-common/src/CMakeLists.txt | 25 -- .../hadoop-common/src/config.h.cmake | 1 - .../io/erasurecode/ErasureCodeNative.java | 86 ----- .../apache/hadoop/util/NativeCodeLoader.java | 11 +- .../hadoop/util/NativeLibraryChecker.java | 20 +- .../src/main/native/native.vcxproj | 22 +- .../io/erasurecode/coder/erasure_code_native.c | 49 --- ...he_hadoop_io_erasurecode_ErasureCodeNative.h | 29 -- .../apache/hadoop/io/erasurecode/erasure_code.c | 271 ---------------- .../io/erasurecode/include/erasure_code.h | 125 -------- .../hadoop/io/erasurecode/include/gf_util.h | 111 ------- .../org/apache/hadoop/util/NativeCodeLoader.c | 10 - .../hadoop/io/erasurecode/erasure_code_test.c | 310 ------------------- hadoop-project-dist/pom.xml | 13 +- hadoop-project/pom.xml | 1 - 18 files changed, 11 insertions(+), 1122 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7e6a5df/BUILDING.txt ---------------------------------------------------------------------- diff --git a/BUILDING.txt b/BUILDING.txt index d5bea32..a18853d 100644 --- a/BUILDING.txt +++ b/BUILDING.txt @@ -72,8 +72,6 @@ Optional packages: * Snappy compression $ sudo apt-get install snappy libsnappy-dev -* Intel ISA-L library for erasure coding - Please refer to https://01.org/intel%C2%AE-storage-acceleration-library-open-source-version * Bzip2 $ sudo apt-get install bzip2 libbz2-dev * Jansson (C Library for JSON) @@ -179,29 +177,6 @@ Maven build goals: * -Dtest.exclude=<TESTCLASSNAME> * -Dtest.exclude.pattern=**/<TESTCLASSNAME1>.java,**/<TESTCLASSNAME2>.java - Intel ISA-L build options: - - Intel ISA-L is a erasure coding library that can be utilized by the native code. - It is currently an optional component, meaning that Hadoop can be built with - or without this dependency. Note the library is used via dynamic module. Please - reference the official site for the library details. - https://01.org/intel%C2%AE-storage-acceleration-library-open-source-version - - * Use -Drequire.isal to fail the build if libisal.so is not found. - If this option is not specified and the isal library is missing, - we silently build a version of libhadoop.so that cannot make use of ISA-L and - the native raw erasure coders. - This option is recommended if you plan on making use of native raw erasure - coders and want to get more repeatable builds. - * Use -Disal.prefix to specify a nonstandard location for the libisal - library files. You do not need this option if you have installed ISA-L to the - system library path. - * Use -Disal.lib to specify a nonstandard location for the libisal library - files. - * Use -Dbundle.isal to copy the contents of the isal.lib directory into - the final tar file. This option requires that -Disal.lib is also given, - and it ignores the -Disal.prefix option. - ---------------------------------------------------------------------------------- Building components separately http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7e6a5df/hadoop-common-project/hadoop-common/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 30ec480..1d6cd57 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -329,9 +329,6 @@ Release 2.8.0 - UNRELEASED HADOOP-11685. StorageException complaining " no lease ID" during HBase distributed log splitting (Duo Xu via cnauroth) - HADOOP-11887. Introduce Intel ISA-L erasure coding library for native - erasure encoding support (Kai Zheng via Colin P. McCabe) - OPTIMIZATIONS HADOOP-11785. Reduce the number of listStatus operation in distcp http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7e6a5df/hadoop-common-project/hadoop-common/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml index d99076a..ccbe0d6 100644 --- a/hadoop-common-project/hadoop-common/pom.xml +++ b/hadoop-common-project/hadoop-common/pom.xml @@ -516,9 +516,6 @@ <openssl.prefix></openssl.prefix> <openssl.lib></openssl.lib> <openssl.include></openssl.include> - <require.isal>false</require.isal> - <isal.prefix></isal.prefix> - <isal.lib></isal.lib> <require.openssl>false</require.openssl> <runningWithNative>true</runningWithNative> <bundle.openssl.in.bin>false</bundle.openssl.in.bin> @@ -572,7 +569,6 @@ <javahClassName>org.apache.hadoop.io.compress.snappy.SnappyDecompressor</javahClassName> <javahClassName>org.apache.hadoop.io.compress.lz4.Lz4Compressor</javahClassName> <javahClassName>org.apache.hadoop.io.compress.lz4.Lz4Decompressor</javahClassName> - <javahClassName>org.apache.hadoop.io.erasurecode.ErasureCodeNative</javahClassName> <javahClassName>org.apache.hadoop.crypto.OpensslCipher</javahClassName> <javahClassName>org.apache.hadoop.crypto.random.OpensslSecureRandom</javahClassName> <javahClassName>org.apache.hadoop.util.NativeCrc32</javahClassName> @@ -595,7 +591,7 @@ <configuration> <target> <exec executable="cmake" dir="${project.build.directory}/native" failonerror="true"> - <arg line="${basedir}/src/ -DGENERATED_JAVAH=${project.build.directory}/native/javah -DJVM_ARCH_DATA_MODEL=${sun.arch.data.model} -DREQUIRE_BZIP2=${require.bzip2} -DREQUIRE_SNAPPY=${require.snappy} -DCUSTOM_SNAPPY_PREFIX=${snappy.prefix} -DCUSTOM_SNAPPY_LIB=${snappy.lib} -DCUSTOM_SNAPPY_INCLUDE=${snappy.include} -DREQUIRE_ISAL=${require.isal} -DCUSTOM_ISAL_PREFIX=${isal.prefix} -DCUSTOM_ISAL_LIB=${isal.lib} -DREQUIRE_OPENSSL=${require.openssl} -DCUSTOM_OPENSSL_PREFIX=${openssl.prefix} -DCUSTOM_OPENSSL_LIB=${openssl.lib} -DCUSTOM_OPENSSL_INCLUDE=${openssl.include} -DEXTRA_LIBHADOOP_RPATH=${extra.libhadoop.rpath}"/> + <arg line="${basedir}/src/ -DGENERATED_JAVAH=${project.build.directory}/native/javah -DJVM_ARCH_DATA_MODEL=${sun.arch.data.model} -DREQUIRE_BZIP2=${require.bzip2} -DREQUIRE_SNAPPY=${require.snappy} -DCUSTOM_SNAPPY_PREFIX=${snappy.prefix} -DCUSTOM_SNAPPY_LIB=${snappy.lib} -DCUSTOM_SNAPPY_INCLUDE=${snappy.include} -DREQUIRE_OPENSSL=${require.openssl} -DCUSTOM_OPENSSL_PREFIX=${openssl.prefix} -DCUSTOM_OPENSSL_LIB=${openssl.lib} -DCUSTOM_OPENSSL_INCLUDE=${openssl.include} -DEXTRA_LIBHADOOP_RPATH=${extra.libhadoop.rpath}"/> </exec> <exec executable="make" dir="${project.build.directory}/native" failonerror="true"> <arg line="VERBOSE=1"/> @@ -617,13 +613,7 @@ <arg value="[ x$SKIPTESTS = xtrue ] || ${project.build.directory}/native/test_bulk_crc32"/> <env key="SKIPTESTS" value="${skipTests}"/> </exec> - <exec executable="${shell-executable}" failonerror="true" dir="${project.build.directory}/native"> - <arg value="-c"/> - <arg value="[ ! -f ${project.build.directory}/native/erasure_code_test ] || ${project.build.directory}/native/erasure_code_test"/> - <env key="SKIPTESTS" value="${skipTests}"/> - <env key="LD_LIBRARY_PATH" value="${LD_LIBRARY_PATH}:${isal.lib}:${isal.prefix}"/> - </exec> - </target> + </target> </configuration> </execution> </executions> @@ -643,9 +633,6 @@ <snappy.prefix></snappy.prefix> <snappy.lib></snappy.lib> <snappy.include></snappy.include> - <require.isal>false</require.isal> - <isal.prefix></isal.prefix> - <isal.lib></isal.lib> <require.snappy>false</require.snappy> <bundle.snappy.in.bin>true</bundle.snappy.in.bin> <openssl.prefix></openssl.prefix> @@ -699,7 +686,6 @@ <javahClassName>org.apache.hadoop.io.compress.snappy.SnappyDecompressor</javahClassName> <javahClassName>org.apache.hadoop.io.compress.lz4.Lz4Compressor</javahClassName> <javahClassName>org.apache.hadoop.io.compress.lz4.Lz4Decompressor</javahClassName> - <javahClassName>org.apache.hadoop.io.erasurecode.ErasureCodeNative</javahClassName> <javahClassName>org.apache.hadoop.crypto.OpensslCipher</javahClassName> <javahClassName>org.apache.hadoop.crypto.random.OpensslSecureRandom</javahClassName> <javahClassName>org.apache.hadoop.util.NativeCrc32</javahClassName> @@ -753,9 +739,6 @@ <argument>/p:CustomOpensslLib=${openssl.lib}</argument> <argument>/p:CustomOpensslInclude=${openssl.include}</argument> <argument>/p:RequireOpenssl=${require.openssl}</argument> - <argument>/p:RequireIsal=${require.isal}</argument> - <argument>/p:CustomIsalPrefix=${isal.prefix}</argument> - <argument>/p:CustomIsalLib=${isal.lib}</argument> </arguments> </configuration> </execution> http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7e6a5df/hadoop-common-project/hadoop-common/src/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/CMakeLists.txt b/hadoop-common-project/hadoop-common/src/CMakeLists.txt index 63bb773..c93bfe7 100644 --- a/hadoop-common-project/hadoop-common/src/CMakeLists.txt +++ b/hadoop-common-project/hadoop-common/src/CMakeLists.txt @@ -94,29 +94,6 @@ else() endif() endif() -set(STORED_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES}) -hadoop_set_find_shared_library_version("2") -find_library(ISAL_LIBRARY - NAMES isal - PATHS ${CUSTOM_ISAL_PREFIX} ${CUSTOM_ISAL_PREFIX}/lib - ${CUSTOM_ISAL_PREFIX}/lib64 ${CUSTOM_ISAL_LIB}) -set(CMAKE_FIND_LIBRARY_SUFFIXES ${STORED_CMAKE_FIND_LIBRARY_SUFFIXES}) -if (ISAL_LIBRARY) - GET_FILENAME_COMPONENT(HADOOP_ISAL_LIBRARY ${ISAL_LIBRARY} NAME) - set(ISAL_INCLUDE_DIR ${SRC}/io/erasurecode/include) - set(ISAL_SOURCE_FILES - ${SRC}/io/erasurecode/erasure_code.c) - add_executable(erasure_code_test - ${SRC}/io/erasurecode/erasure_code.c - ${TST}/io/erasurecode/erasure_code_test.c - ) - target_link_libraries(erasure_code_test ${CMAKE_DL_LIBS}) -else (ISAL_LIBRARY) - IF(REQUIRE_ISAL) - MESSAGE(FATAL_ERROR "Required ISA-L library could not be found. ISAL_LIBRARY=${ISAL_LIBRARY}, CUSTOM_ISAL_PREFIX=${CUSTOM_ISAL_PREFIX}") - ENDIF(REQUIRE_ISAL) -endif (ISAL_LIBRARY) - # Build hardware CRC32 acceleration, if supported on the platform. if(CMAKE_SYSTEM_PROCESSOR MATCHES "^i.86$" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "amd64") set(BULK_CRC_ARCH_SOURCE_FIlE "${SRC}/util/bulk_crc32_x86.c") @@ -192,7 +169,6 @@ include_directories( ${ZLIB_INCLUDE_DIRS} ${BZIP2_INCLUDE_DIR} ${SNAPPY_INCLUDE_DIR} - ${ISAL_INCLUDE_DIR} ${OPENSSL_INCLUDE_DIR} ${SRC}/util ) @@ -205,7 +181,6 @@ hadoop_add_dual_library(hadoop ${SRC}/io/compress/lz4/Lz4Decompressor.c ${SRC}/io/compress/lz4/lz4.c ${SRC}/io/compress/lz4/lz4hc.c - ${ISAL_SOURCE_FILES} ${SNAPPY_SOURCE_FILES} ${OPENSSL_SOURCE_FILES} ${SRC}/io/compress/zlib/ZlibCompressor.c http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7e6a5df/hadoop-common-project/hadoop-common/src/config.h.cmake ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/config.h.cmake b/hadoop-common-project/hadoop-common/src/config.h.cmake index 445cc33..d71271d 100644 --- a/hadoop-common-project/hadoop-common/src/config.h.cmake +++ b/hadoop-common-project/hadoop-common/src/config.h.cmake @@ -22,7 +22,6 @@ #cmakedefine HADOOP_BZIP2_LIBRARY "@HADOOP_BZIP2_LIBRARY@" #cmakedefine HADOOP_SNAPPY_LIBRARY "@HADOOP_SNAPPY_LIBRARY@" #cmakedefine HADOOP_OPENSSL_LIBRARY "@HADOOP_OPENSSL_LIBRARY@" -#cmakedefine HADOOP_ISAL_LIBRARY "@HADOOP_ISAL_LIBRARY@" #cmakedefine HAVE_SYNC_FILE_RANGE #cmakedefine HAVE_POSIX_FADVISE http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7e6a5df/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeNative.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeNative.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeNative.java deleted file mode 100644 index 1c98f3c..0000000 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeNative.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.io.erasurecode; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.util.NativeCodeLoader; - -/** - * Erasure code native libraries (for now, Intel ISA-L) related utilities. - */ -public final class ErasureCodeNative { - - private static final Log LOG = - LogFactory.getLog(ErasureCodeNative.class.getName()); - - /** - * The reason why ISA-L library is not available, or null if it is available. - */ - private static final String LOADING_FAILURE_REASON; - - static { - if (!NativeCodeLoader.isNativeCodeLoaded()) { - LOADING_FAILURE_REASON = "hadoop native library cannot be loaded."; - } else if (!NativeCodeLoader.buildSupportsIsal()) { - LOADING_FAILURE_REASON = "libhadoop was built without ISA-L support"; - } else { - String problem = null; - try { - loadLibrary(); - } catch (Throwable t) { - problem = "Loading ISA-L failed: " + t.getMessage(); - LOG.error("Loading ISA-L failed", t); - } - LOADING_FAILURE_REASON = problem; - } - } - - private ErasureCodeNative() {} - - /** - * Are native libraries loaded? - */ - public static boolean isNativeCodeLoaded() { - return LOADING_FAILURE_REASON == null; - } - - /** - * Is the native ISA-L library loaded and initialized? Throw exception if not. - */ - public static void checkNativeCodeLoaded() { - if (LOADING_FAILURE_REASON != null) { - throw new RuntimeException(LOADING_FAILURE_REASON); - } - } - - /** - * Load native library available or supported. - */ - public static native void loadLibrary(); - - /** - * Get the native library name that's available or supported. - */ - public static native String getLibraryName(); - - public static String getLoadingFailureReason() { - return LOADING_FAILURE_REASON; - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7e6a5df/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCodeLoader.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCodeLoader.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCodeLoader.java index 6f502e4..533fc07 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCodeLoader.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCodeLoader.java @@ -33,7 +33,7 @@ import org.apache.hadoop.fs.CommonConfigurationKeys; */ @InterfaceAudience.Private @InterfaceStability.Unstable -public final class NativeCodeLoader { +public class NativeCodeLoader { private static final Log LOG = LogFactory.getLog(NativeCodeLoader.class); @@ -64,8 +64,6 @@ public final class NativeCodeLoader { } } - private NativeCodeLoader() {} - /** * Check if native-hadoop code is loaded for this platform. * @@ -80,12 +78,7 @@ public final class NativeCodeLoader { * Returns true only if this build was compiled with support for snappy. */ public static native boolean buildSupportsSnappy(); - - /** - * Returns true only if this build was compiled with support for ISA-L. - */ - public static native boolean buildSupportsIsal(); - + /** * Returns true only if this build was compiled with support for openssl. */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7e6a5df/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java index c31f85d..d8c6899 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java @@ -18,7 +18,6 @@ package org.apache.hadoop.util; -import org.apache.hadoop.io.erasurecode.ErasureCodeNative; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.crypto.OpensslCipher; import org.apache.hadoop.io.compress.Lz4Codec; @@ -66,7 +65,6 @@ public class NativeLibraryChecker { boolean nativeHadoopLoaded = NativeCodeLoader.isNativeCodeLoaded(); boolean zlibLoaded = false; boolean snappyLoaded = false; - boolean isalLoaded = false; // lz4 is linked within libhadoop boolean lz4Loaded = nativeHadoopLoaded; boolean bzip2Loaded = Bzip2Factory.isNativeBzip2Loaded(conf); @@ -77,7 +75,6 @@ public class NativeLibraryChecker { String hadoopLibraryName = ""; String zlibLibraryName = ""; String snappyLibraryName = ""; - String isalDetail = ""; String lz4LibraryName = ""; String bzip2LibraryName = ""; String winutilsPath = null; @@ -88,29 +85,18 @@ public class NativeLibraryChecker { if (zlibLoaded) { zlibLibraryName = ZlibFactory.getLibraryName(); } - snappyLoaded = NativeCodeLoader.buildSupportsSnappy() && SnappyCodec.isNativeCodeLoaded(); if (snappyLoaded && NativeCodeLoader.buildSupportsSnappy()) { snappyLibraryName = SnappyCodec.getLibraryName(); } - - isalDetail = ErasureCodeNative.getLoadingFailureReason(); - if (isalDetail != null) { - isalLoaded = false; - } else { - isalDetail = ErasureCodeNative.getLibraryName(); - isalLoaded = true; - } - - openSslDetail = OpensslCipher.getLoadingFailureReason(); - if (openSslDetail != null) { + if (OpensslCipher.getLoadingFailureReason() != null) { + openSslDetail = OpensslCipher.getLoadingFailureReason(); openSslLoaded = false; } else { openSslDetail = OpensslCipher.getLibraryName(); openSslLoaded = true; } - if (lz4Loaded) { lz4LibraryName = Lz4Codec.getLibraryName(); } @@ -139,8 +125,6 @@ public class NativeLibraryChecker { System.out.printf("lz4: %b %s%n", lz4Loaded, lz4LibraryName); System.out.printf("bzip2: %b %s%n", bzip2Loaded, bzip2LibraryName); System.out.printf("openssl: %b %s%n", openSslLoaded, openSslDetail); - System.out.printf("ISA-L: %b %s%n", isalLoaded, isalDetail); - if (Shell.WINDOWS) { System.out.printf("winutils: %b %s%n", winutilsExists, winutilsPath); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7e6a5df/hadoop-common-project/hadoop-common/src/main/native/native.vcxproj ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/native/native.vcxproj b/hadoop-common-project/hadoop-common/src/main/native/native.vcxproj index 17149f7..0912c6a 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/native.vcxproj +++ b/hadoop-common-project/hadoop-common/src/main/native/native.vcxproj @@ -17,7 +17,7 @@ limitations under the License. --> -<Project DefaultTargets="CheckRequireSnappy;CheckRequireIsal;Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003";> +<Project DefaultTargets="CheckRequireSnappy;Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003";> <ItemGroup Label="ProjectConfigurations"> <ProjectConfiguration Include="Release|Win32"> <Configuration>Release</Configuration> @@ -79,22 +79,11 @@ <IncludePath Condition="'$(SnappyEnabled)' == 'true'">$(SnappyInclude);$(IncludePath)</IncludePath> <IncludePath Condition="Exists('$(ZLIB_HOME)')">$(ZLIB_HOME);$(IncludePath)</IncludePath> </PropertyGroup> - <PropertyGroup> - <IsalLib Condition="Exists('$(CustomIsalPrefix)\isa-l.dll')">$(CustomIsalPrefix)</IsalLib> - <IsalLib Condition="Exists('$(CustomIsalPrefix)\lib\isa-l.dll') And '$(IsalLib)' == ''">$(CustomIsalPrefix)\lib</IsalLib> - <IsalLib Condition="Exists('$(CustomIsalLib)') And '$(IsalLib)' == ''">$(CustomIsalLib)</IsalLib> - <IsalEnabled Condition="'$(IsalLib)' != ''">true</IsalEnabled> - </PropertyGroup> <Target Name="CheckRequireSnappy"> <Error Text="Required snappy library could not be found. SnappyLibrary=$(SnappyLibrary), SnappyInclude=$(SnappyInclude), CustomSnappyLib=$(CustomSnappyLib), CustomSnappyInclude=$(CustomSnappyInclude), CustomSnappyPrefix=$(CustomSnappyPrefix)" Condition="'$(RequireSnappy)' == 'true' And '$(SnappyEnabled)' != 'true'" /> </Target> - <Target Name="CheckRequireIsal"> - <Error - Text="Required ISA-L library could not be found. CustomIsalLib=$(CustomIsalLib), CustomIsalPrefix=$(CustomIsalPrefix)" - Condition="'$(RequireIsal)' == 'true' And '$(IsalEnabled)' != 'true'" /> - </Target> <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'"> <ClCompile> <WarningLevel>Level3</WarningLevel> @@ -103,7 +92,7 @@ <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> <PreprocessorDefinitions>WIN32;NDEBUG;_WINDOWS;_USRDLL;NATIVE_EXPORTS;%(PreprocessorDefinitions)</PreprocessorDefinitions> - <AdditionalIncludeDirectories>..\winutils\include;..\native\src\org\apache\hadoop\io\erasurecode\include;..\..\..\target\native\javah;%JAVA_HOME%\include;%JAVA_HOME%\include\win32;.\src;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> + <AdditionalIncludeDirectories>..\winutils\include;..\..\..\target\native\javah;%JAVA_HOME%\include;%JAVA_HOME%\include\win32;.\src;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> <CompileAs>CompileAsC</CompileAs> <DisableSpecificWarnings>4244</DisableSpecificWarnings> </ClCompile> @@ -124,7 +113,7 @@ <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> <PreprocessorDefinitions>WIN32;NDEBUG;_WINDOWS;_USRDLL;NATIVE_EXPORTS;%(PreprocessorDefinitions)</PreprocessorDefinitions> - <AdditionalIncludeDirectories>..\winutils\include;..\native\src\org\apache\hadoop\io\erasurecode\include;..\..\..\target\native\javah;%JAVA_HOME%\include;%JAVA_HOME%\include\win32;.\src;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> + <AdditionalIncludeDirectories>..\winutils\include;..\..\..\target\native\javah;%JAVA_HOME%\include;%JAVA_HOME%\include\win32;.\src;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> <CompileAs>CompileAsC</CompileAs> <DisableSpecificWarnings>4244</DisableSpecificWarnings> </ClCompile> @@ -156,16 +145,11 @@ <ClCompile Include="src\org\apache\hadoop\util\bulk_crc32.c" /> <ClCompile Include="src\org\apache\hadoop\util\NativeCodeLoader.c"> <AdditionalOptions Condition="'$(SnappyEnabled)' == 'true'">/D HADOOP_SNAPPY_LIBRARY=L\"snappy.dll\"</AdditionalOptions> - <AdditionalOptions Condition="'$(IsalEnabled)' == 'true'">/D HADOOP_ISAL_LIBRARY=\"isa-l.dll\"</AdditionalOptions> </ClCompile> <ClCompile Include="src\org\apache\hadoop\util\NativeCrc32.c" /> <ClCompile Include="src\org\apache\hadoop\yarn\server\nodemanager\windows_secure_container_executor.c"> <AdditionalIncludeDirectories>src\org\apache\hadoop\io\nativeio;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> </ClCompile> - <ClCompile Include="src\org\apache\hadoop\io\erasurecode\erasure_code.c" Condition="'$(IsalEnabled)' == 'true'"> - <AdditionalOptions>/D HADOOP_ISAL_LIBRARY=\"isa-l.dll\"</AdditionalOptions> - </ClCompile> - <ClCompile Include="src\org\apache\hadoop\io\erasurecode\coder\erasure_code_native.c" Condition="'$(IsalEnabled)' == 'true'"/> </ItemGroup> <ItemGroup> <ClInclude Include="..\src\org\apache\hadoop\util\crc32c_tables.h" /> http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7e6a5df/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/coder/erasure_code_native.c ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/coder/erasure_code_native.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/coder/erasure_code_native.c deleted file mode 100644 index e84df9a..0000000 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/coder/erasure_code_native.c +++ /dev/null @@ -1,49 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <stdio.h> -#include <stdlib.h> -#include <string.h> - -#include "org_apache_hadoop.h" -#include "../include/erasure_code.h" -#include "org_apache_hadoop_io_erasurecode_ErasureCodeNative.h" - -#ifdef UNIX -#include "config.h" -#endif - -JNIEXPORT void JNICALL -Java_org_apache_hadoop_io_erasurecode_ErasureCodeNative_loadLibrary -(JNIEnv *env, jclass myclass) { - char errMsg[1024]; - load_erasurecode_lib(errMsg, sizeof(errMsg)); - if (strlen(errMsg) > 0) { - THROW(env, "java/lang/UnsatisfiedLinkError", errMsg); - } -} - -JNIEXPORT jstring JNICALL -Java_org_apache_hadoop_io_erasurecode_ErasureCodeNative_getLibraryName -(JNIEnv *env, jclass myclass) { - char* libName = get_library_name(); - if (libName == NULL) { - libName = "Unavailable"; - } - return (*env)->NewStringUTF(env, libName); -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7e6a5df/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/coder/org_apache_hadoop_io_erasurecode_ErasureCodeNative.h ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/coder/org_apache_hadoop_io_erasurecode_ErasureCodeNative.h b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/coder/org_apache_hadoop_io_erasurecode_ErasureCodeNative.h deleted file mode 100644 index d8ff3a0..0000000 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/coder/org_apache_hadoop_io_erasurecode_ErasureCodeNative.h +++ /dev/null @@ -1,29 +0,0 @@ -/* DO NOT EDIT THIS FILE - it is machine generated */ -#include <jni.h> -/* Header for class org_apache_hadoop_io_erasurecode_ErasureCodeNative */ - -#ifndef _Included_org_apache_hadoop_io_erasurecode_ErasureCodeNative -#define _Included_org_apache_hadoop_io_erasurecode_ErasureCodeNative -#ifdef __cplusplus -extern "C" { -#endif -/* - * Class: org_apache_hadoop_io_erasurecode_ErasureCodeNative - * Method: loadLibrary - * Signature: ()V - */ -JNIEXPORT void JNICALL Java_org_apache_hadoop_io_erasurecode_ErasureCodeNative_loadLibrary - (JNIEnv *, jclass); - -/* - * Class: org_apache_hadoop_io_erasurecode_ErasureCodeNative - * Method: getLibraryName - * Signature: ()Ljava/lang/String; - */ -JNIEXPORT jstring JNICALL Java_org_apache_hadoop_io_erasurecode_ErasureCodeNative_getLibraryName - (JNIEnv *, jclass); - -#ifdef __cplusplus -} -#endif -#endif http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7e6a5df/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/erasure_code.c ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/erasure_code.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/erasure_code.c deleted file mode 100644 index a6c099a..0000000 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/erasure_code.c +++ /dev/null @@ -1,271 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <errno.h> -#include <stdio.h> -#include <stdlib.h> -#include <string.h> - -#include "org_apache_hadoop.h" -#include "../include/gf_util.h" -#include "../include/erasure_code.h" - -#ifdef UNIX -#include <sys/time.h> -#include <sys/types.h> -#include <sys/stat.h> -#include <dlfcn.h> - -#include "config.h" -#endif - -#ifdef WINDOWS -#include <Windows.h> -#endif - -/** - * erasure_code.c - * Implementation erasure code utilities based on lib of erasure_code.so. - * Building of this codes won't rely on any ISA-L source codes, but running - * into this will rely on successfully loading of the dynamic library. - * - */ - -/** - * The loaded library handle. - */ -static void* libec = NULL; - -/** - * A helper function to dlsym a 'symbol' from a given library-handle. - */ - -#ifdef UNIX - -static __attribute__ ((unused)) -void *my_dlsym(void *handle, const char *symbol) { - void *func_ptr = dlsym(handle, symbol); - return func_ptr; -} - -/* A helper macro to dlsym the requisite dynamic symbol in NON-JNI env. */ -#define EC_LOAD_DYNAMIC_SYMBOL(func_ptr, handle, symbol) \ - if ((func_ptr = my_dlsym(handle, symbol)) == NULL) { \ - return "Failed to load symbol" symbol; \ - } - -#endif - -#ifdef WINDOWS - - - -static FARPROC WINAPI my_dlsym(HMODULE handle, LPCSTR symbol) { - FARPROC func_ptr = GetProcAddress(handle, symbol); - return func_ptr; -} - -/* A helper macro to dlsym the requisite dynamic symbol in NON-JNI env. */ -#define EC_LOAD_DYNAMIC_SYMBOL(func_type, func_ptr, handle, symbol) \ - if ((func_ptr = (func_type)my_dlsym(handle, symbol)) == NULL) { \ - return "Failed to load symbol" symbol; \ - } - -#endif - - -#ifdef UNIX -// For gf_util.h -static unsigned char (*d_gf_mul)(unsigned char, unsigned char); -static unsigned char (*d_gf_inv)(unsigned char); -static void (*d_gf_gen_rs_matrix)(unsigned char *, int, int); -static void (*d_gf_gen_cauchy_matrix)(unsigned char *, int, int); -static int (*d_gf_invert_matrix)(unsigned char *, unsigned char *, const int); -static int (*d_gf_vect_mul)(int, unsigned char *, void *, void *); - -// For erasure_code.h -static void (*d_ec_init_tables)(int, int, unsigned char*, unsigned char*); -static void (*d_ec_encode_data)(int, int, int, unsigned char*, - unsigned char**, unsigned char**); -static void (*d_ec_encode_data_update)(int, int, int, int, unsigned char*, - unsigned char*, unsigned char**); -#endif - -#ifdef WINDOWS -// For erasure_code.h -typedef unsigned char (__cdecl *__d_gf_mul)(unsigned char, unsigned char); -static __d_gf_mul d_gf_mul; -typedef unsigned char (__cdecl *__d_gf_inv)(unsigned char); -static __d_gf_inv d_gf_inv; -typedef void (__cdecl *__d_gf_gen_rs_matrix)(unsigned char *, int, int); -static __d_gf_gen_rs_matrix d_gf_gen_rs_matrix; -typedef void (__cdecl *__d_gf_gen_cauchy_matrix)(unsigned char *, int, int); -static __d_gf_gen_cauchy_matrix d_gf_gen_cauchy_matrix; -typedef int (__cdecl *__d_gf_invert_matrix)(unsigned char *, - unsigned char *, const int); -static __d_gf_invert_matrix d_gf_invert_matrix; -typedef int (__cdecl *__d_gf_vect_mul)(int, unsigned char *, void *, void *); -static __d_gf_vect_mul d_gf_vect_mul; - -// For erasure_code.h -typedef void (__cdecl *__d_ec_init_tables)(int, int, - unsigned char*, unsigned char*); -static __d_ec_init_tables d_ec_init_tables; -typedef void (__cdecl *__d_ec_encode_data)(int, int, int, unsigned char*, - unsigned char**, unsigned char**); -static __d_ec_encode_data d_ec_encode_data; -typedef void (__cdecl *__d_ec_encode_data_update)(int, int, int, int, unsigned char*, - unsigned char*, unsigned char**); -static __d_ec_encode_data_update d_ec_encode_data_update; -#endif - -static const char* load_functions(void* libec) { -#ifdef UNIX - EC_LOAD_DYNAMIC_SYMBOL(d_gf_mul, libec, "gf_mul"); - EC_LOAD_DYNAMIC_SYMBOL(d_gf_inv, libec, "gf_inv"); - EC_LOAD_DYNAMIC_SYMBOL(d_gf_gen_rs_matrix, libec, "gf_gen_rs_matrix"); - EC_LOAD_DYNAMIC_SYMBOL(d_gf_gen_cauchy_matrix, libec, "gf_gen_cauchy1_matrix"); - EC_LOAD_DYNAMIC_SYMBOL(d_gf_invert_matrix, libec, "gf_invert_matrix"); - EC_LOAD_DYNAMIC_SYMBOL(d_gf_vect_mul, libec, "gf_vect_mul"); - - EC_LOAD_DYNAMIC_SYMBOL(d_ec_init_tables, libec, "ec_init_tables"); - EC_LOAD_DYNAMIC_SYMBOL(d_ec_encode_data, libec, "ec_encode_data"); - EC_LOAD_DYNAMIC_SYMBOL(d_ec_encode_data_update, libec, "ec_encode_data_update"); -#endif - -#ifdef WINDOWS - EC_LOAD_DYNAMIC_SYMBOL(__d_gf_mul, d_gf_mul, libec, "gf_mul"); - EC_LOAD_DYNAMIC_SYMBOL(__d_gf_inv, d_gf_inv, libec, "gf_inv"); - EC_LOAD_DYNAMIC_SYMBOL(__d_gf_gen_rs_matrix, d_gf_gen_rs_matrix, libec, "gf_gen_rs_matrix"); - EC_LOAD_DYNAMIC_SYMBOL(__d_gf_gen_cauchy_matrix, d_gf_gen_cauchy_matrix, libec, "gf_gen_cauchy1_matrix"); - EC_LOAD_DYNAMIC_SYMBOL(__d_gf_invert_matrix, d_gf_invert_matrix, libec, "gf_invert_matrix"); - EC_LOAD_DYNAMIC_SYMBOL(__d_gf_vect_mul, d_gf_vect_mul, libec, "gf_vect_mul"); - - EC_LOAD_DYNAMIC_SYMBOL(__d_ec_init_tables, d_ec_init_tables, libec, "ec_init_tables"); - EC_LOAD_DYNAMIC_SYMBOL(__d_ec_encode_data, d_ec_encode_data, libec, "ec_encode_data"); - EC_LOAD_DYNAMIC_SYMBOL(__d_ec_encode_data_update, d_ec_encode_data_update, libec, "ec_encode_data_update"); -#endif - - return NULL; -} - -void load_erasurecode_lib(char* err, size_t err_len) { - const char* errMsg; - - err[0] = '\0'; - - if (libec != NULL) { - return; - } - - // Load Intel ISA-L - #ifdef UNIX - libec = dlopen(HADOOP_ISAL_LIBRARY, RTLD_LAZY | RTLD_GLOBAL); - if (libec == NULL) { - snprintf(err, err_len, "Failed to load %s (%s)", - HADOOP_ISAL_LIBRARY, dlerror()); - return; - } - // Clear any existing error - dlerror(); - #endif - - #ifdef WINDOWS - libec = LoadLibrary(HADOOP_ISAL_LIBRARY); - if (libec == NULL) { - snprintf(err, err_len, "Failed to load %s", HADOOP_ISAL_LIBRARY); - return; - } - #endif - - errMsg = load_functions(libec); - if (errMsg != NULL) { - snprintf(err, err_len, "Loading functions from ISA-L failed: %s", errMsg); - } -} - -int build_support_erasurecode() { -#ifdef HADOOP_ISAL_LIBRARY - return 1; -#else - return 0; -#endif -} - -const char* get_library_name() { -#ifdef UNIX - Dl_info dl_info; - - if (d_ec_encode_data == NULL) { - return HADOOP_ISAL_LIBRARY; - } - - if(dladdr(d_ec_encode_data, &dl_info)) { - return dl_info.dli_fname; - } -#else - LPTSTR filename = NULL; - - if (libec == NULL) { - return HADOOP_ISAL_LIBRARY; - } - - if (GetModuleFileName(libec, filename, 256) > 0) { - return filename; - } -#endif - - return NULL; -} - -unsigned char h_gf_mul(unsigned char a, unsigned char b) { - return d_gf_mul(a, b); -} - -unsigned char h_gf_inv(unsigned char a) { - return d_gf_inv(a); -} - -void h_gf_gen_rs_matrix(unsigned char *a, int m, int k) { - d_gf_gen_rs_matrix(a, m, k); -} - -void h_gf_gen_cauchy_matrix(unsigned char *a, int m, int k) { - d_gf_gen_cauchy_matrix(a, m, k); -} - -int h_gf_invert_matrix(unsigned char *in, unsigned char *out, const int n) { - return d_gf_invert_matrix(in, out, n); -} - -int h_gf_vect_mul(int len, unsigned char *gftbl, void *src, void *dest) { - return d_gf_vect_mul(len, gftbl, src, dest); -} - -void h_ec_init_tables(int k, int rows, unsigned char* a, unsigned char* gftbls) { - d_ec_init_tables(k, rows, a, gftbls); -} - -void h_ec_encode_data(int len, int k, int rows, unsigned char *gftbls, - unsigned char **data, unsigned char **coding) { - d_ec_encode_data(len, k, rows, gftbls, data, coding); -} - -void h_ec_encode_data_update(int len, int k, int rows, int vec_i, - unsigned char *gftbls, unsigned char *data, unsigned char **coding) { - d_ec_encode_data_update(len, k, rows, vec_i, gftbls, data, coding); -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7e6a5df/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/include/erasure_code.h ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/include/erasure_code.h b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/include/erasure_code.h deleted file mode 100644 index 123085e..0000000 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/include/erasure_code.h +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef _ERASURE_CODE_H_ -#define _ERASURE_CODE_H_ - -#include <stddef.h> - -/** - * Interface to functions supporting erasure code encode and decode. - * - * This file defines the interface to optimized functions used in erasure - * codes. Encode and decode of erasures in GF(2^8) are made by calculating the - * dot product of the symbols (bytes in GF(2^8)) across a set of buffers and a - * set of coefficients. Values for the coefficients are determined by the type - * of erasure code. Using a general dot product means that any sequence of - * coefficients may be used including erasure codes based on random - * coefficients. - * Multiple versions of dot product are supplied to calculate 1-6 output - * vectors in one pass. - * Base GF multiply and divide functions can be sped up by defining - * GF_LARGE_TABLES at the expense of memory size. - * - */ - -/** - * Return 0 if not support, 1 otherwise. - */ -int build_support_erasurecode(); - -/** - * Get the library name possibly of full path. - */ -const char* get_library_name(); - -/** - * Initialize and load erasure code library, returning error message if any. - * - * @param err The err message buffer. - * @param err_len The length of the message buffer. - */ -void load_erasurecode_lib(char* err, size_t err_len); - -/** - * Initialize tables for fast Erasure Code encode and decode. - * - * Generates the expanded tables needed for fast encode or decode for erasure - * codes on blocks of data. 32bytes is generated for each input coefficient. - * - * @param k The number of vector sources or rows in the generator matrix - * for coding. - * @param rows The number of output vectors to concurrently encode/decode. - * @param a Pointer to sets of arrays of input coefficients used to encode - * or decode data. - * @param gftbls Pointer to start of space for concatenated output tables - * generated from input coefficients. Must be of size 32*k*rows. - * @returns none - */ -void h_ec_init_tables(int k, int rows, unsigned char* a, unsigned char* gftbls); - -/** - * Generate or decode erasure codes on blocks of data, runs appropriate version. - * - * Given a list of source data blocks, generate one or multiple blocks of - * encoded data as specified by a matrix of GF(2^8) coefficients. When given a - * suitable set of coefficients, this function will perform the fast generation - * or decoding of Reed-Solomon type erasure codes. - * - * This function determines what instruction sets are enabled and - * selects the appropriate version at runtime. - * - * @param len Length of each block of data (vector) of source or dest data. - * @param k The number of vector sources or rows in the generator matrix - * for coding. - * @param rows The number of output vectors to concurrently encode/decode. - * @param gftbls Pointer to array of input tables generated from coding - * coefficients in ec_init_tables(). Must be of size 32*k*rows - * @param data Array of pointers to source input buffers. - * @param coding Array of pointers to coded output buffers. - * @returns none - */ -void h_ec_encode_data(int len, int k, int rows, unsigned char *gftbls, - unsigned char **data, unsigned char **coding); - -/** - * @brief Generate update for encode or decode of erasure codes from single - * source, runs appropriate version. - * - * Given one source data block, update one or multiple blocks of encoded data as - * specified by a matrix of GF(2^8) coefficients. When given a suitable set of - * coefficients, this function will perform the fast generation or decoding of - * Reed-Solomon type erasure codes from one input source at a time. - * - * This function determines what instruction sets are enabled and selects the - * appropriate version at runtime. - * - * @param len Length of each block of data (vector) of source or dest data. - * @param k The number of vector sources or rows in the generator matrix - * for coding. - * @param rows The number of output vectors to concurrently encode/decode. - * @param vec_i The vector index corresponding to the single input source. - * @param gftbls Pointer to array of input tables generated from coding - * coefficients in ec_init_tables(). Must be of size 32*k*rows - * @param data Pointer to single input source used to update output parity. - * @param coding Array of pointers to coded output buffers. - * @returns none - */ -void h_ec_encode_data_update(int len, int k, int rows, int vec_i, - unsigned char *gftbls, unsigned char *data, unsigned char **coding); - -#endif //_ERASURE_CODE_H_ http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7e6a5df/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/include/gf_util.h ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/include/gf_util.h b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/include/gf_util.h deleted file mode 100644 index 2be8328..0000000 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/include/gf_util.h +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef _GF_UTIL_H -#define _GF_UTIL_H - -/** - * gf_util.h - * Interface to functions for vector (block) multiplication in GF(2^8). - * - * This file defines the interface to routines used in fast RAID rebuild and - * erasure codes. - */ - - -/** - * Single element GF(2^8) multiply. - * - * @param a Multiplicand a - * @param b Multiplicand b - * @returns Product of a and b in GF(2^8) - */ -unsigned char h_gf_mul(unsigned char a, unsigned char b); - -/** - * Single element GF(2^8) inverse. - * - * @param a Input element - * @returns Field element b such that a x b = {1} - */ -unsigned char h_gf_inv(unsigned char a); - -/** - * Generate a matrix of coefficients to be used for encoding. - * - * Vandermonde matrix example of encoding coefficients where high portion of - * matrix is identity matrix I and lower portion is constructed as 2^{i*(j-k+1)} - * i:{0,k-1} j:{k,m-1}. Commonly used method for choosing coefficients in - * erasure encoding but does not guarantee invertable for every sub matrix. For - * large k it is possible to find cases where the decode matrix chosen from - * sources and parity not in erasure are not invertable. Users may want to - * adjust for k > 5. - * - * @param a [mxk] array to hold coefficients - * @param m number of rows in matrix corresponding to srcs + parity. - * @param k number of columns in matrix corresponding to srcs. - * @returns none - */ -void h_gf_gen_rs_matrix(unsigned char *a, int m, int k); - -/** - * Generate a Cauchy matrix of coefficients to be used for encoding. - * - * Cauchy matrix example of encoding coefficients where high portion of matrix - * is identity matrix I and lower portion is constructed as 1/(i + j) | i != j, - * i:{0,k-1} j:{k,m-1}. Any sub-matrix of a Cauchy matrix should be invertable. - * - * @param a [mxk] array to hold coefficients - * @param m number of rows in matrix corresponding to srcs + parity. - * @param k number of columns in matrix corresponding to srcs. - * @returns none - */ -void h_gf_gen_cauchy_matrix(unsigned char *a, int m, int k); - -/** - * Invert a matrix in GF(2^8) - * - * @param in input matrix - * @param out output matrix such that [in] x [out] = [I] - identity matrix - * @param n size of matrix [nxn] - * @returns 0 successful, other fail on singular input matrix - */ -int h_gf_invert_matrix(unsigned char *in, unsigned char *out, const int n); - -/** - * GF(2^8) vector multiply by constant, runs appropriate version. - * - * Does a GF(2^8) vector multiply b = Ca where a and b are arrays and C - * is a single field element in GF(2^8). Can be used for RAID6 rebuild - * and partial write functions. Function requires pre-calculation of a - * 32-element constant array based on constant C. gftbl(C) = {C{00}, - * C{01}, C{02}, ... , C{0f} }, {C{00}, C{10}, C{20}, ... , C{f0} }. - * Len and src must be aligned to 32B. - * - * This function determines what instruction sets are enabled - * and selects the appropriate version at runtime. - * - * @param len Length of vector in bytes. Must be aligned to 32B. - * @param gftbl Pointer to 32-byte array of pre-calculated constants based on C. - * @param src Pointer to src data array. Must be aligned to 32B. - * @param dest Pointer to destination data array. Must be aligned to 32B. - * @returns 0 pass, other fail - */ -int h_gf_vect_mul(int len, unsigned char *gftbl, void *src, void *dest); - - -#endif //_GF_UTIL_H http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7e6a5df/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCodeLoader.c ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCodeLoader.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCodeLoader.c index ae8263a..3625112 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCodeLoader.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCodeLoader.c @@ -49,16 +49,6 @@ JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_util_NativeCodeLoader_buildSup #endif } -JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_util_NativeCodeLoader_buildSupportsIsal - (JNIEnv *env, jclass clazz) -{ -#ifdef HADOOP_ISAL_LIBRARY - return JNI_TRUE; -#else - return JNI_FALSE; -#endif -} - JNIEXPORT jstring JNICALL Java_org_apache_hadoop_util_NativeCodeLoader_getLibraryName (JNIEnv *env, jclass clazz) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7e6a5df/hadoop-common-project/hadoop-common/src/main/native/src/test/org/apache/hadoop/io/erasurecode/erasure_code_test.c ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/test/org/apache/hadoop/io/erasurecode/erasure_code_test.c b/hadoop-common-project/hadoop-common/src/main/native/src/test/org/apache/hadoop/io/erasurecode/erasure_code_test.c deleted file mode 100644 index 9817a76..0000000 --- a/hadoop-common-project/hadoop-common/src/main/native/src/test/org/apache/hadoop/io/erasurecode/erasure_code_test.c +++ /dev/null @@ -1,310 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * This is a lightweight version of the same file in Intel ISA-L library to test - * and verify the basic functions of ISA-L integration. Note it's not serving as - * a complete ISA-L library test nor as any sample to write an erasure coder - * using the library. A sample is to be written and provided separately. - */ - -#include "org_apache_hadoop.h" -#include "erasure_code.h" -#include "gf_util.h" - -#include <stdio.h> -#include <stdlib.h> -#include <string.h> - -#define TEST_LEN 8192 -#define TEST_SOURCES 127 -#define MMAX TEST_SOURCES -#define KMAX TEST_SOURCES -#define TEST_SEED 11 - -static void dump(unsigned char *buf, int len) -{ - int i; - for (i = 0; i < len;) { - printf(" %2x", 0xff & buf[i++]); - if (i % 32 == 0) - printf("\n"); - } - printf("\n"); -} - -static void dump_matrix(unsigned char **s, int k, int m) -{ - int i, j; - for (i = 0; i < k; i++) { - for (j = 0; j < m; j++) { - printf(" %2x", s[i][j]); - } - printf("\n"); - } - printf("\n"); -} - -static void dump_u8xu8(unsigned char *s, int k, int m) -{ - int i, j; - for (i = 0; i < k; i++) { - for (j = 0; j < m; j++) { - printf(" %2x", 0xff & s[j + (i * m)]); - } - printf("\n"); - } - printf("\n"); -} - -// Generate Random errors -static void gen_err_list(unsigned char *src_err_list, - unsigned char *src_in_err, int *pnerrs, int *pnsrcerrs, int k, int m) -{ - int i, err; - int nerrs = 0, nsrcerrs = 0; - - for (i = 0, nerrs = 0, nsrcerrs = 0; i < m && nerrs < m - k; i++) { - err = 1 & rand(); - src_in_err[i] = err; - if (err) { - src_err_list[nerrs++] = i; - if (i < k) { - nsrcerrs++; - } - } - } - if (nerrs == 0) { // should have at least one error - while ((err = (rand() % KMAX)) >= m) ; - src_err_list[nerrs++] = err; - src_in_err[err] = 1; - if (err < k) - nsrcerrs = 1; - } - *pnerrs = nerrs; - *pnsrcerrs = nsrcerrs; - return; -} - -#define NO_INVERT_MATRIX -2 -// Generate decode matrix from encode matrix -static int gf_gen_decode_matrix(unsigned char *encode_matrix, - unsigned char *decode_matrix, - unsigned char *invert_matrix, - unsigned int *decode_index, - unsigned char *src_err_list, - unsigned char *src_in_err, - int nerrs, int nsrcerrs, int k, int m) -{ - int i, j, p; - int r; - unsigned char *backup, *b, s; - int incr = 0; - - b = malloc(MMAX * KMAX); - backup = malloc(MMAX * KMAX); - - if (b == NULL || backup == NULL) { - printf("Test failure! Error with malloc\n"); - free(b); - free(backup); - return -1; - } - // Construct matrix b by removing error rows - for (i = 0, r = 0; i < k; i++, r++) { - while (src_in_err[r]) - r++; - for (j = 0; j < k; j++) { - b[k * i + j] = encode_matrix[k * r + j]; - backup[k * i + j] = encode_matrix[k * r + j]; - } - decode_index[i] = r; - } - incr = 0; - while (h_gf_invert_matrix(b, invert_matrix, k) < 0) { - if (nerrs == (m - k)) { - free(b); - free(backup); - printf("BAD MATRIX\n"); - return NO_INVERT_MATRIX; - } - incr++; - memcpy(b, backup, MMAX * KMAX); - for (i = nsrcerrs; i < nerrs - nsrcerrs; i++) { - if (src_err_list[i] == (decode_index[k - 1] + incr)) { - // skip the erased parity line - incr++; - continue; - } - } - if (decode_index[k - 1] + incr >= m) { - free(b); - free(backup); - printf("BAD MATRIX\n"); - return NO_INVERT_MATRIX; - } - decode_index[k - 1] += incr; - for (j = 0; j < k; j++) - b[k * (k - 1) + j] = encode_matrix[k * decode_index[k - 1] + j]; - - }; - - for (i = 0; i < nsrcerrs; i++) { - for (j = 0; j < k; j++) { - decode_matrix[k * i + j] = invert_matrix[k * src_err_list[i] + j]; - } - } - /* src_err_list from encode_matrix * invert of b for parity decoding */ - for (p = nsrcerrs; p < nerrs; p++) { - for (i = 0; i < k; i++) { - s = 0; - for (j = 0; j < k; j++) - s ^= h_gf_mul(invert_matrix[j * k + i], - encode_matrix[k * src_err_list[p] + j]); - - decode_matrix[k * p + i] = s; - } - } - free(b); - free(backup); - return 0; -} - -int main(int argc, char *argv[]) -{ - char err[256]; - size_t err_len = sizeof(err); - int re, i, j, p, m, k; - int nerrs, nsrcerrs; - unsigned int decode_index[MMAX]; - unsigned char *temp_buffs[TEST_SOURCES], *buffs[TEST_SOURCES]; - unsigned char *encode_matrix, *decode_matrix, *invert_matrix, *g_tbls; - unsigned char src_in_err[TEST_SOURCES], src_err_list[TEST_SOURCES]; - unsigned char *recov[TEST_SOURCES]; - - if (0 == build_support_erasurecode()) { - printf("The native library isn't available, skipping this test\n"); - return 0; // Normal, not an error - } - - load_erasurecode_lib(err, err_len); - if (strlen(err) > 0) { - printf("Loading erasurecode library failed: %s\n", err); - return -1; - } - - printf("Performing erasure code test\n"); - srand(TEST_SEED); - - // Allocate the arrays - for (i = 0; i < TEST_SOURCES; i++) { - buffs[i] = malloc(TEST_LEN); - } - - for (i = 0; i < TEST_SOURCES; i++) { - temp_buffs[i] = malloc(TEST_LEN); - } - - // Test erasure code by encode and recovery - - encode_matrix = malloc(MMAX * KMAX); - decode_matrix = malloc(MMAX * KMAX); - invert_matrix = malloc(MMAX * KMAX); - g_tbls = malloc(KMAX * TEST_SOURCES * 32); - if (encode_matrix == NULL || decode_matrix == NULL - || invert_matrix == NULL || g_tbls == NULL) { - snprintf(err, err_len, "%s", "allocating test matrix buffers error"); - return -1; - } - - m = 9; - k = 5; - if (m > MMAX || k > KMAX) - return -1; - - // Make random data - for (i = 0; i < k; i++) - for (j = 0; j < TEST_LEN; j++) - buffs[i][j] = rand(); - - // The matrix generated by gf_gen_cauchy1_matrix - // is always invertable. - h_gf_gen_cauchy_matrix(encode_matrix, m, k); - - // Generate g_tbls from encode matrix encode_matrix - h_ec_init_tables(k, m - k, &encode_matrix[k * k], g_tbls); - - // Perform matrix dot_prod for EC encoding - // using g_tbls from encode matrix encode_matrix - h_ec_encode_data(TEST_LEN, k, m - k, g_tbls, buffs, &buffs[k]); - - // Choose random buffers to be in erasure - memset(src_in_err, 0, TEST_SOURCES); - gen_err_list(src_err_list, src_in_err, &nerrs, &nsrcerrs, k, m); - - // Generate decode matrix - re = gf_gen_decode_matrix(encode_matrix, decode_matrix, - invert_matrix, decode_index, src_err_list, src_in_err, - nerrs, nsrcerrs, k, m); - if (re != 0) { - snprintf(err, err_len, "%s", "gf_gen_decode_matrix failed"); - return -1; - } - // Pack recovery array as list of valid sources - // Its order must be the same as the order - // to generate matrix b in gf_gen_decode_matrix - for (i = 0; i < k; i++) { - recov[i] = buffs[decode_index[i]]; - } - - // Recover data - h_ec_init_tables(k, nerrs, decode_matrix, g_tbls); - h_ec_encode_data(TEST_LEN, k, nerrs, g_tbls, recov, &temp_buffs[k]); - for (i = 0; i < nerrs; i++) { - if (0 != memcmp(temp_buffs[k + i], buffs[src_err_list[i]], TEST_LEN)) { - snprintf(err, err_len, "%s", "Error recovery failed"); - printf("Fail error recovery (%d, %d, %d)\n", m, k, nerrs); - - printf(" - erase list = "); - for (j = 0; j < nerrs; j++) { - printf(" %d", src_err_list[j]); - } - - printf(" - Index = "); - for (p = 0; p < k; p++) { - printf(" %d", decode_index[p]); - } - - printf("\nencode_matrix:\n"); - dump_u8xu8((unsigned char *) encode_matrix, m, k); - printf("inv b:\n"); - dump_u8xu8((unsigned char *) invert_matrix, k, k); - printf("\ndecode_matrix:\n"); - dump_u8xu8((unsigned char *) decode_matrix, m, k); - printf("recov %d:", src_err_list[i]); - dump(temp_buffs[k + i], 25); - printf("orig :"); - dump(buffs[src_err_list[i]], 25); - - return -1; - } - } - - printf("done EC tests: Pass\n"); - return 0; -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7e6a5df/hadoop-project-dist/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-project-dist/pom.xml b/hadoop-project-dist/pom.xml index f1676b6..5a48bcd 100644 --- a/hadoop-project-dist/pom.xml +++ b/hadoop-project-dist/pom.xml @@ -41,7 +41,6 @@ <hadoop.component>UNDEF</hadoop.component> <bundle.snappy>false</bundle.snappy> <bundle.snappy.in.bin>false</bundle.snappy.in.bin> - <bundle.isal>true</bundle.isal> <bundle.openssl>false</bundle.openssl> <bundle.openssl.in.bin>false</bundle.openssl.in.bin> </properties> @@ -349,22 +348,14 @@ mkdir -p $${TARGET_DIR} cd $${LIB_DIR} $$TAR lib* | (cd $${TARGET_DIR}/; $$UNTAR) - if [ "X${bundle.snappy}" = "Xtrue" ] ; then + if [ "${bundle.snappy}" = "true" ] ; then cd "${snappy.lib}" $$TAR *snappy* | (cd $${TARGET_DIR}/; $$UNTAR) fi - if [ "X${bundle.openssl}" = "Xtrue" ] ; then + if [ "${bundle.openssl}" = "true" ] ; then cd "${openssl.lib}" $$TAR *crypto* | (cd $${TARGET_DIR}/; $$UNTAR) fi - if [ "X${bundle.isal}" = "Xtrue" ] ; then - if [ "X${isal.lib}" != "X" ]; then - cd "${isal.lib}" - $$TAR *isa* | (cd $${TARGET_DIR}/; $$UNTAR) - else - echo "The required option isal.lib isn't given, bundling ISA-L skipped" - fi - fi fi BIN_DIR="${BUILD_DIR}/bin" if [ -d $${BIN_DIR} ] ; then http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7e6a5df/hadoop-project/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index a6ecb02..cf58a1d 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -1272,7 +1272,6 @@ <!-- Specify where to look for the native DLL on Windows --> <PATH>${env.PATH};${hadoop.common.build.dir}/bin;${snappy.lib}</PATH> <PATH>${env.PATH};${hadoop.common.build.dir}/bin;${openssl.lib}</PATH> - <PATH>${env.PATH};${hadoop.common.build.dir}/bin;${isal.lib}</PATH> </environmentVariables> </configuration> </plugin>
