Converted KenLM into a submodule
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/6da3961b Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/6da3961b Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/6da3961b Branch: refs/heads/master Commit: 6da3961bee52d10d17d9218ba267a505d1fb70ea Parents: 76bb8fd Author: Matt Post <[email protected]> Authored: Tue Apr 19 15:23:42 2016 -0400 Committer: Matt Post <[email protected]> Committed: Tue Apr 19 15:23:42 2016 -0400 ---------------------------------------------------------------------- .gitmodules | 4 + build.xml | 2 +- ext/kenlm | 1 + ext/kenlm/.gitignore | 32 - ext/kenlm/BUILDING | 10 - ext/kenlm/CMakeLists.txt | 57 - ext/kenlm/COPYING | 502 -- ext/kenlm/COPYING.3 | 674 -- ext/kenlm/COPYING.LESSER.3 | 165 - ext/kenlm/Doxyfile | 1519 ---- ext/kenlm/Jamroot | 68 - ext/kenlm/LICENSE | 26 - ext/kenlm/README.md | 96 - ext/kenlm/bjam | 23 - ext/kenlm/clean_query_only.sh | 2 - ext/kenlm/cmake/KenLMFunctions.cmake | 82 - ext/kenlm/cmake/modules/FindEigen3.cmake | 90 - ext/kenlm/compile_query_only.sh | 34 - ext/kenlm/jam-files/LICENSE_1_0.txt | 23 - ext/kenlm/jam-files/boost-build/boost-build.jam | 8 - ext/kenlm/jam-files/boost-build/bootstrap.jam | 18 - .../jam-files/boost-build/build-system.jam | 981 --- ext/kenlm/jam-files/boost-build/build/ac.jam | 303 - ext/kenlm/jam-files/boost-build/build/alias.jam | 74 - .../boost-build/build/build-request.jam | 322 - .../boost-build/build/config-cache.jam | 64 - .../jam-files/boost-build/build/configure.jam | 281 - .../jam-files/boost-build/build/feature.jam | 1350 ---- .../jam-files/boost-build/build/generators.jam | 1420 ---- .../jam-files/boost-build/build/project.jam | 1228 --- .../boost-build/build/property-set.jam | 517 -- .../jam-files/boost-build/build/property.jam | 905 --- .../jam-files/boost-build/build/readme.txt | 11 - .../jam-files/boost-build/build/scanner.jam | 163 - .../jam-files/boost-build/build/targets.jam | 1694 ---- .../jam-files/boost-build/build/toolset.jam | 582 -- ext/kenlm/jam-files/boost-build/build/type.jam | 401 - .../jam-files/boost-build/build/version.jam | 165 - .../boost-build/build/virtual-target.jam | 1339 ---- .../boost-build/kernel/boost-build.jam | 5 - .../jam-files/boost-build/kernel/bootstrap.jam | 266 - .../jam-files/boost-build/kernel/class.jam | 420 - .../jam-files/boost-build/kernel/errors.jam | 287 - .../jam-files/boost-build/kernel/modules.jam | 359 - .../jam-files/boost-build/options/help.jam | 212 - ext/kenlm/jam-files/boost-build/site-config.jam | 11 - ext/kenlm/jam-files/boost-build/tools/acc.jam | 118 - .../jam-files/boost-build/tools/auto-index.jam | 204 - ext/kenlm/jam-files/boost-build/tools/bison.jam | 32 - .../boost-build/tools/boostbook-config.jam | 13 - .../jam-files/boost-build/tools/boostbook.jam | 773 -- .../jam-files/boost-build/tools/borland.jam | 220 - .../jam-files/boost-build/tools/builtin.jam | 974 --- ext/kenlm/jam-files/boost-build/tools/cast.jam | 91 - .../boost-build/tools/clang-darwin.jam | 170 - .../jam-files/boost-build/tools/clang-linux.jam | 196 - ext/kenlm/jam-files/boost-build/tools/clang.jam | 27 - .../jam-files/boost-build/tools/common.jam | 988 --- .../jam-files/boost-build/tools/como-linux.jam | 103 - .../jam-files/boost-build/tools/como-win.jam | 117 - ext/kenlm/jam-files/boost-build/tools/como.jam | 29 - .../jam-files/boost-build/tools/convert.jam | 62 - ext/kenlm/jam-files/boost-build/tools/cray.jam | 115 - .../jam-files/boost-build/tools/cw-config.jam | 34 - ext/kenlm/jam-files/boost-build/tools/cw.jam | 246 - .../jam-files/boost-build/tools/darwin.jam | 592 -- ext/kenlm/jam-files/boost-build/tools/dmc.jam | 134 - .../jam-files/boost-build/tools/docutils.jam | 85 - .../boost-build/tools/doxygen-config.jam | 11 - .../jam-files/boost-build/tools/doxygen.jam | 775 -- .../tools/doxygen/windows-paths-check.doxyfile | 3 - .../tools/doxygen/windows-paths-check.hpp | 0 ext/kenlm/jam-files/boost-build/tools/fop.jam | 69 - .../jam-files/boost-build/tools/fortran.jam | 55 - ext/kenlm/jam-files/boost-build/tools/gcc.jam | 1184 --- .../jam-files/boost-build/tools/generate.jam | 108 - .../jam-files/boost-build/tools/gettext.jam | 230 - .../jam-files/boost-build/tools/gfortran.jam | 39 - .../jam-files/boost-build/tools/hp_cxx.jam | 181 - .../jam-files/boost-build/tools/hpfortran.jam | 35 - ext/kenlm/jam-files/boost-build/tools/ifort.jam | 44 - .../boost-build/tools/intel-darwin.jam | 227 - .../jam-files/boost-build/tools/intel-linux.jam | 250 - .../jam-files/boost-build/tools/intel-win.jam | 184 - ext/kenlm/jam-files/boost-build/tools/intel.jam | 34 - ext/kenlm/jam-files/boost-build/tools/jpeg.jam | 233 - ext/kenlm/jam-files/boost-build/tools/lex.jam | 33 - ext/kenlm/jam-files/boost-build/tools/make.jam | 63 - ext/kenlm/jam-files/boost-build/tools/mc.jam | 44 - .../jam-files/boost-build/tools/message.jam | 62 - ext/kenlm/jam-files/boost-build/tools/midl.jam | 142 - .../jam-files/boost-build/tools/mipspro.jam | 145 - ext/kenlm/jam-files/boost-build/tools/mpi.jam | 583 -- .../jam-files/boost-build/tools/msvc-config.jam | 12 - ext/kenlm/jam-files/boost-build/tools/msvc.jam | 1435 ---- .../jam-files/boost-build/tools/notfile.jam | 65 - .../jam-files/boost-build/tools/package.jam | 165 - .../jam-files/boost-build/tools/pathscale.jam | 178 - ext/kenlm/jam-files/boost-build/tools/pch.jam | 95 - ext/kenlm/jam-files/boost-build/tools/pgi.jam | 147 - ext/kenlm/jam-files/boost-build/tools/png.jam | 226 - .../boost-build/tools/python-config.jam | 27 - .../jam-files/boost-build/tools/python.jam | 1258 --- ext/kenlm/jam-files/boost-build/tools/qcc.jam | 238 - ext/kenlm/jam-files/boost-build/tools/qt.jam | 17 - ext/kenlm/jam-files/boost-build/tools/qt3.jam | 209 - ext/kenlm/jam-files/boost-build/tools/qt4.jam | 755 -- ext/kenlm/jam-files/boost-build/tools/qt5.jam | 728 -- .../boost-build/tools/quickbook-config.jam | 44 - .../jam-files/boost-build/tools/quickbook.jam | 361 - ext/kenlm/jam-files/boost-build/tools/rc.jam | 155 - ext/kenlm/jam-files/boost-build/tools/stage.jam | 519 -- .../jam-files/boost-build/tools/stlport.jam | 309 - ext/kenlm/jam-files/boost-build/tools/sun.jam | 142 - .../jam-files/boost-build/tools/symlink.jam | 140 - .../jam-files/boost-build/tools/testing-aux.jam | 220 - .../jam-files/boost-build/tools/testing.jam | 594 -- ext/kenlm/jam-files/boost-build/tools/tiff.jam | 230 - .../jam-files/boost-build/tools/types/asm.jam | 4 - .../jam-files/boost-build/tools/types/cpp.jam | 90 - .../jam-files/boost-build/tools/types/exe.jam | 9 - .../jam-files/boost-build/tools/types/html.jam | 4 - .../jam-files/boost-build/tools/types/lib.jam | 74 - .../jam-files/boost-build/tools/types/obj.jam | 9 - .../jam-files/boost-build/tools/types/objc.jam | 26 - .../boost-build/tools/types/preprocessed.jam | 9 - .../jam-files/boost-build/tools/types/qt.jam | 12 - .../boost-build/tools/types/register.jam | 39 - .../jam-files/boost-build/tools/types/rsp.jam | 4 - ext/kenlm/jam-files/boost-build/tools/unix.jam | 224 - ext/kenlm/jam-files/boost-build/tools/vacpp.jam | 150 - ext/kenlm/jam-files/boost-build/tools/whale.jam | 116 - ext/kenlm/jam-files/boost-build/tools/xlf.jam | 39 - .../boost-build/tools/xsltproc-config.jam | 36 - .../jam-files/boost-build/tools/xsltproc.jam | 205 - .../boost-build/tools/xsltproc/included.xsl | 11 - .../boost-build/tools/xsltproc/test.xml | 2 - .../boost-build/tools/xsltproc/test.xsl | 12 - ext/kenlm/jam-files/boost-build/tools/zlib.jam | 227 - ext/kenlm/jam-files/boost-build/user-config.jam | 92 - ext/kenlm/jam-files/boost-build/util/assert.jam | 346 - .../jam-files/boost-build/util/container.jam | 339 - ext/kenlm/jam-files/boost-build/util/doc.jam | 1002 --- .../jam-files/boost-build/util/indirect.jam | 117 - .../jam-files/boost-build/util/numbers.jam | 218 - ext/kenlm/jam-files/boost-build/util/option.jam | 109 - ext/kenlm/jam-files/boost-build/util/order.jam | 169 - ext/kenlm/jam-files/boost-build/util/os.jam | 171 - ext/kenlm/jam-files/boost-build/util/path.jam | 910 --- ext/kenlm/jam-files/boost-build/util/print.jam | 488 -- ext/kenlm/jam-files/boost-build/util/regex.jam | 203 - .../jam-files/boost-build/util/sequence.jam | 339 - ext/kenlm/jam-files/boost-build/util/set.jam | 93 - ext/kenlm/jam-files/boost-build/util/string.jam | 189 - .../jam-files/boost-build/util/utility.jam | 235 - ext/kenlm/jam-files/empty_test_main.cc | 8 - ext/kenlm/jam-files/engine/Jambase | 2473 ------ ext/kenlm/jam-files/engine/boost-jam.spec | 64 - ext/kenlm/jam-files/engine/boost-no-inspect | 1 - ext/kenlm/jam-files/engine/build.bat | 590 -- ext/kenlm/jam-files/engine/build.jam | 1022 --- ext/kenlm/jam-files/engine/build.sh | 303 - ext/kenlm/jam-files/engine/builtins.c | 2354 ------ ext/kenlm/jam-files/engine/builtins.h | 70 - ext/kenlm/jam-files/engine/bump_version.py | 98 - ext/kenlm/jam-files/engine/class.c | 191 - ext/kenlm/jam-files/engine/class.h | 14 - ext/kenlm/jam-files/engine/command.c | 84 - ext/kenlm/jam-files/engine/command.h | 73 - ext/kenlm/jam-files/engine/compile.c | 271 - ext/kenlm/jam-files/engine/compile.h | 59 - ext/kenlm/jam-files/engine/constants.c | 186 - ext/kenlm/jam-files/engine/constants.h | 73 - ext/kenlm/jam-files/engine/cwd.c | 83 - ext/kenlm/jam-files/engine/cwd.h | 35 - ext/kenlm/jam-files/engine/debian/changelog | 72 - ext/kenlm/jam-files/engine/debian/control | 16 - ext/kenlm/jam-files/engine/debian/copyright | 25 - ext/kenlm/jam-files/engine/debian/jam.man.sgml | 236 - ext/kenlm/jam-files/engine/debian/rules | 73 - ext/kenlm/jam-files/engine/debug.c | 145 - ext/kenlm/jam-files/engine/debug.h | 55 - ext/kenlm/jam-files/engine/execcmd.c | 121 - ext/kenlm/jam-files/engine/execcmd.h | 102 - ext/kenlm/jam-files/engine/execnt.c | 1232 --- ext/kenlm/jam-files/engine/execunix.c | 559 -- ext/kenlm/jam-files/engine/filent.c | 435 -- ext/kenlm/jam-files/engine/filesys.c | 326 - ext/kenlm/jam-files/engine/filesys.h | 57 - ext/kenlm/jam-files/engine/fileunix.c | 464 -- ext/kenlm/jam-files/engine/frames.c | 29 - ext/kenlm/jam-files/engine/frames.h | 42 - ext/kenlm/jam-files/engine/function.c | 4870 ------------ ext/kenlm/jam-files/engine/function.h | 46 - ext/kenlm/jam-files/engine/glob.c | 152 - ext/kenlm/jam-files/engine/hash.c | 387 - ext/kenlm/jam-files/engine/hash.h | 79 - ext/kenlm/jam-files/engine/hcache.c | 519 -- ext/kenlm/jam-files/engine/hcache.h | 19 - ext/kenlm/jam-files/engine/hdrmacro.c | 139 - ext/kenlm/jam-files/engine/hdrmacro.h | 21 - ext/kenlm/jam-files/engine/headers.c | 197 - ext/kenlm/jam-files/engine/headers.h | 25 - ext/kenlm/jam-files/engine/jam.c | 656 -- ext/kenlm/jam-files/engine/jam.h | 475 -- ext/kenlm/jam-files/engine/jambase.c | 1691 ---- ext/kenlm/jam-files/engine/jambase.h | 15 - ext/kenlm/jam-files/engine/jamgram.c | 2327 ------ ext/kenlm/jam-files/engine/jamgram.h | 149 - ext/kenlm/jam-files/engine/jamgram.y | 373 - ext/kenlm/jam-files/engine/jamgram.yy | 329 - ext/kenlm/jam-files/engine/jamgramtab.h | 44 - ext/kenlm/jam-files/engine/lists.c | 475 -- ext/kenlm/jam-files/engine/lists.h | 113 - ext/kenlm/jam-files/engine/make.c | 908 --- ext/kenlm/jam-files/engine/make.h | 44 - ext/kenlm/jam-files/engine/make1.c | 1283 --- ext/kenlm/jam-files/engine/md5.c | 381 - ext/kenlm/jam-files/engine/md5.h | 91 - ext/kenlm/jam-files/engine/mem.c | 75 - ext/kenlm/jam-files/engine/mem.h | 133 - ext/kenlm/jam-files/engine/mkjambase.c | 123 - ext/kenlm/jam-files/engine/modules.c | 431 - ext/kenlm/jam-files/engine/modules.h | 52 - ext/kenlm/jam-files/engine/modules/order.c | 160 - ext/kenlm/jam-files/engine/modules/path.c | 25 - .../jam-files/engine/modules/property-set.c | 330 - ext/kenlm/jam-files/engine/modules/readme.txt | 3 - ext/kenlm/jam-files/engine/modules/regex.c | 220 - ext/kenlm/jam-files/engine/modules/sequence.c | 97 - ext/kenlm/jam-files/engine/modules/set.c | 43 - ext/kenlm/jam-files/engine/native.c | 34 - ext/kenlm/jam-files/engine/native.h | 34 - ext/kenlm/jam-files/engine/object.c | 394 - ext/kenlm/jam-files/engine/object.h | 44 - ext/kenlm/jam-files/engine/option.c | 94 - ext/kenlm/jam-files/engine/option.h | 23 - ext/kenlm/jam-files/engine/output.c | 98 - ext/kenlm/jam-files/engine/output.h | 30 - ext/kenlm/jam-files/engine/parse.c | 132 - ext/kenlm/jam-files/engine/parse.h | 76 - ext/kenlm/jam-files/engine/patchlevel.h | 17 - ext/kenlm/jam-files/engine/pathnt.c | 301 - ext/kenlm/jam-files/engine/pathsys.c | 285 - ext/kenlm/jam-files/engine/pathsys.h | 85 - ext/kenlm/jam-files/engine/pathunix.c | 71 - ext/kenlm/jam-files/engine/regexp.c | 1329 ---- ext/kenlm/jam-files/engine/regexp.h | 34 - ext/kenlm/jam-files/engine/rules.c | 740 -- ext/kenlm/jam-files/engine/rules.h | 270 - ext/kenlm/jam-files/engine/scan.c | 404 - ext/kenlm/jam-files/engine/scan.h | 61 - ext/kenlm/jam-files/engine/search.c | 274 - ext/kenlm/jam-files/engine/search.h | 22 - ext/kenlm/jam-files/engine/strings.c | 223 - ext/kenlm/jam-files/engine/strings.h | 36 - ext/kenlm/jam-files/engine/subst.c | 116 - ext/kenlm/jam-files/engine/subst.h | 14 - ext/kenlm/jam-files/engine/timestamp.c | 262 - ext/kenlm/jam-files/engine/timestamp.h | 47 - ext/kenlm/jam-files/engine/variable.c | 345 - ext/kenlm/jam-files/engine/variable.h | 34 - ext/kenlm/jam-files/engine/w32_getreg.c | 201 - ext/kenlm/jam-files/engine/yyacc.c | 268 - ext/kenlm/jam-files/fail/Jamroot | 4 - ext/kenlm/jam-files/sanity.jam | 344 - ext/kenlm/lm/CMakeLists.txt | 90 - ext/kenlm/lm/Jamfile | 40 - ext/kenlm/lm/bhiksha.cc | 94 - ext/kenlm/lm/bhiksha.hh | 122 - ext/kenlm/lm/binary_format.cc | 302 - ext/kenlm/lm/binary_format.hh | 106 - ext/kenlm/lm/blank.hh | 42 - ext/kenlm/lm/build_binary_main.cc | 234 - ext/kenlm/lm/builder/CMakeLists.txt | 67 - ext/kenlm/lm/builder/Jamfile | 13 - ext/kenlm/lm/builder/README.md | 47 - ext/kenlm/lm/builder/TODO | 5 - ext/kenlm/lm/builder/adjust_counts.cc | 353 - ext/kenlm/lm/builder/adjust_counts.hh | 72 - ext/kenlm/lm/builder/adjust_counts_test.cc | 112 - ext/kenlm/lm/builder/combine_counts.hh | 31 - ext/kenlm/lm/builder/corpus_count.cc | 239 - ext/kenlm/lm/builder/corpus_count.hh | 53 - ext/kenlm/lm/builder/corpus_count_test.cc | 79 - ext/kenlm/lm/builder/debug_print.hh | 70 - ext/kenlm/lm/builder/discount.hh | 26 - ext/kenlm/lm/builder/dump_counts_main.cc | 36 - ext/kenlm/lm/builder/hash_gamma.hh | 19 - ext/kenlm/lm/builder/header_info.hh | 28 - ext/kenlm/lm/builder/initial_probabilities.cc | 306 - ext/kenlm/lm/builder/initial_probabilities.hh | 45 - ext/kenlm/lm/builder/interpolate.cc | 166 - ext/kenlm/lm/builder/interpolate.hh | 37 - ext/kenlm/lm/builder/lmplz_main.cc | 220 - ext/kenlm/lm/builder/output.cc | 52 - ext/kenlm/lm/builder/output.hh | 85 - ext/kenlm/lm/builder/payload.hh | 48 - ext/kenlm/lm/builder/pipeline.cc | 385 - ext/kenlm/lm/builder/pipeline.hh | 76 - ext/kenlm/lm/common/CMakeLists.txt | 40 - ext/kenlm/lm/common/Jamfile | 2 - ext/kenlm/lm/common/compare.hh | 174 - ext/kenlm/lm/common/joint_order.hh | 71 - ext/kenlm/lm/common/model_buffer.cc | 91 - ext/kenlm/lm/common/model_buffer.hh | 63 - ext/kenlm/lm/common/ngram.hh | 77 - ext/kenlm/lm/common/ngram_stream.hh | 65 - ext/kenlm/lm/common/print.cc | 62 - ext/kenlm/lm/common/print.hh | 58 - ext/kenlm/lm/common/renumber.cc | 17 - ext/kenlm/lm/common/renumber.hh | 30 - ext/kenlm/lm/common/size_option.cc | 24 - ext/kenlm/lm/common/size_option.hh | 11 - ext/kenlm/lm/common/special.hh | 27 - ext/kenlm/lm/config.cc | 30 - ext/kenlm/lm/config.hh | 124 - ext/kenlm/lm/enumerate_vocab.hh | 28 - ext/kenlm/lm/facade.hh | 73 - ext/kenlm/lm/filter/CMakeLists.txt | 62 - ext/kenlm/lm/filter/Jamfile | 7 - ext/kenlm/lm/filter/arpa_io.cc | 77 - ext/kenlm/lm/filter/arpa_io.hh | 99 - ext/kenlm/lm/filter/count_io.hh | 89 - ext/kenlm/lm/filter/filter_main.cc | 253 - ext/kenlm/lm/filter/format.hh | 250 - ext/kenlm/lm/filter/phrase.cc | 292 - ext/kenlm/lm/filter/phrase.hh | 168 - ext/kenlm/lm/filter/phrase_table_vocab_main.cc | 165 - ext/kenlm/lm/filter/thread.hh | 167 - ext/kenlm/lm/filter/vocab.cc | 53 - ext/kenlm/lm/filter/vocab.hh | 133 - ext/kenlm/lm/filter/wrapper.hh | 56 - ext/kenlm/lm/fragment_main.cc | 37 - ext/kenlm/lm/interpolate/CMakeLists.txt | 54 - ext/kenlm/lm/interpolate/Jamfile | 22 - ext/kenlm/lm/interpolate/backoff_matrix.hh | 29 - .../lm/interpolate/backoff_reunification.cc | 57 - .../lm/interpolate/backoff_reunification.hh | 27 - .../interpolate/backoff_reunification_test.cc | 159 - .../lm/interpolate/bounded_sequence_encoding.cc | 36 - .../lm/interpolate/bounded_sequence_encoding.hh | 76 - .../bounded_sequence_encoding_test.cc | 86 - .../lm/interpolate/enumerate_global_vocab.cc | 48 - .../lm/interpolate/enumerate_global_vocab.hh | 38 - ext/kenlm/lm/interpolate/interpolate_info.hh | 35 - ext/kenlm/lm/interpolate/interpolate_main.cc | 37 - ext/kenlm/lm/interpolate/merge_probabilities.cc | 285 - ext/kenlm/lm/interpolate/merge_probabilities.hh | 89 - ext/kenlm/lm/interpolate/merge_test/test1 | Bin 26 -> 0 bytes ext/kenlm/lm/interpolate/merge_test/test2 | Bin 39 -> 0 bytes ext/kenlm/lm/interpolate/merge_test/test3 | Bin 16 -> 0 bytes .../lm/interpolate/merge_test/test_bad_order | Bin 14 -> 0 bytes ext/kenlm/lm/interpolate/merge_test/test_no_unk | 1 - ext/kenlm/lm/interpolate/merge_vocab.cc | 131 - ext/kenlm/lm/interpolate/merge_vocab.hh | 23 - ext/kenlm/lm/interpolate/merge_vocab_test.cc | 126 - ext/kenlm/lm/interpolate/normalize.cc | 384 - ext/kenlm/lm/interpolate/normalize.hh | 35 - ext/kenlm/lm/interpolate/normalize_test.cc | 86 - ext/kenlm/lm/interpolate/perf_enum_gv_main.cc | 215 - ext/kenlm/lm/interpolate/pipeline.cc | 159 - ext/kenlm/lm/interpolate/pipeline.hh | 22 - ext/kenlm/lm/interpolate/split_worker.cc | 40 - ext/kenlm/lm/interpolate/split_worker.hh | 44 - .../lm/interpolate/streaming_example_main.cc | 195 - ...ear_interpolation.lambda1_0.4.lambda2_0.6.lm | 23 - ...ear_interpolation.lambda1_0.4.lambda2_0.6.lm | 23 - ext/kenlm/lm/interpolate/toy_data/toy1.lm | 23 - ext/kenlm/lm/interpolate/toy_data/toy2.lm | 23 - .../lm/interpolate/train_params_grant_main.cc | 561 -- ext/kenlm/lm/interpolate/train_params_main.cc | 349 - ext/kenlm/lm/interpolate/tune_derivatives.cc | 91 - ext/kenlm/lm/interpolate/tune_derivatives.hh | 30 - .../lm/interpolate/tune_derivatives_test.cc | 89 - ext/kenlm/lm/interpolate/tune_instance.cc | 354 - ext/kenlm/lm/interpolate/tune_instance.hh | 86 - .../interpolate/tune_instance_data/generate.sh | 9 - .../lm/interpolate/tune_instance_data/toy0.1 | Bin 60 -> 0 bytes .../lm/interpolate/tune_instance_data/toy0.2 | Bin 112 -> 0 bytes .../lm/interpolate/tune_instance_data/toy0.3 | Bin 140 -> 0 bytes .../tune_instance_data/toy0.kenlm_intermediate | 3 - .../interpolate/tune_instance_data/toy0.vocab | Bin 19 -> 0 bytes .../lm/interpolate/tune_instance_data/toy1.1 | Bin 72 -> 0 bytes .../lm/interpolate/tune_instance_data/toy1.2 | Bin 112 -> 0 bytes .../lm/interpolate/tune_instance_data/toy1.3 | Bin 120 -> 0 bytes .../tune_instance_data/toy1.kenlm_intermediate | 3 - .../interpolate/tune_instance_data/toy1.vocab | Bin 21 -> 0 bytes ext/kenlm/lm/interpolate/tune_instance_test.cc | 93 - ext/kenlm/lm/interpolate/tune_main.cc | 59 - ext/kenlm/lm/interpolate/tune_matrix.hh | 17 - ext/kenlm/lm/interpolate/universal_vocab.cc | 13 - ext/kenlm/lm/interpolate/universal_vocab.hh | 38 - ext/kenlm/lm/kenlm_benchmark_main.cc | 142 - ext/kenlm/lm/left.hh | 216 - ext/kenlm/lm/left_test.cc | 397 - ext/kenlm/lm/lm_exception.cc | 23 - ext/kenlm/lm/lm_exception.hh | 50 - ext/kenlm/lm/max_order.hh | 13 - ext/kenlm/lm/model.cc | 349 - ext/kenlm/lm/model.hh | 155 - ext/kenlm/lm/model_test.cc | 448 -- ext/kenlm/lm/model_type.hh | 23 - ext/kenlm/lm/neural/Jamfile | 6 - ext/kenlm/lm/neural/wordvecs.hh | 38 - ext/kenlm/lm/ngram_query.hh | 113 - ext/kenlm/lm/partial.hh | 166 - ext/kenlm/lm/partial_test.cc | 199 - ext/kenlm/lm/quantize.cc | 93 - ext/kenlm/lm/quantize.hh | 233 - ext/kenlm/lm/query_main.cc | 115 - ext/kenlm/lm/read_arpa.cc | 161 - ext/kenlm/lm/read_arpa.hh | 95 - ext/kenlm/lm/return.hh | 42 - ext/kenlm/lm/search_hashed.cc | 298 - ext/kenlm/lm/search_hashed.hh | 192 - ext/kenlm/lm/search_trie.cc | 600 -- ext/kenlm/lm/search_trie.hh | 129 - ext/kenlm/lm/sizes.cc | 63 - ext/kenlm/lm/sizes.hh | 17 - ext/kenlm/lm/state.hh | 125 - ext/kenlm/lm/test.arpa | 124 - ext/kenlm/lm/test_nounk.arpa | 120 - ext/kenlm/lm/trie.cc | 131 - ext/kenlm/lm/trie.hh | 146 - ext/kenlm/lm/trie_sort.cc | 304 - ext/kenlm/lm/trie_sort.hh | 114 - ext/kenlm/lm/value.hh | 158 - ext/kenlm/lm/value_build.cc | 59 - ext/kenlm/lm/value_build.hh | 97 - ext/kenlm/lm/virtual_interface.cc | 19 - ext/kenlm/lm/virtual_interface.hh | 160 - ext/kenlm/lm/vocab.cc | 329 - ext/kenlm/lm/vocab.hh | 279 - ext/kenlm/lm/weights.hh | 22 - ext/kenlm/lm/word_index.hh | 15 - ext/kenlm/lm/wrappers/README | 3 - ext/kenlm/lm/wrappers/nplm.cc | 116 - ext/kenlm/lm/wrappers/nplm.hh | 85 - ext/kenlm/python/_kenlm.pxd | 33 - ext/kenlm/python/example.py | 28 - ext/kenlm/python/kenlm.cpp | 7379 ------------------ ext/kenlm/python/kenlm.pyx | 231 - ext/kenlm/setup.py | 48 - ext/kenlm/util/CMakeLists.txt | 81 - ext/kenlm/util/Jamfile | 41 - ext/kenlm/util/bit_packing.cc | 40 - ext/kenlm/util/bit_packing.hh | 185 - ext/kenlm/util/bit_packing_test.cc | 59 - ext/kenlm/util/cat_compressed_main.cc | 47 - ext/kenlm/util/double-conversion/CMakeLists.txt | 39 - ext/kenlm/util/double-conversion/Jamfile | 1 - ext/kenlm/util/double-conversion/LICENSE | 26 - ext/kenlm/util/double-conversion/bignum-dtoa.cc | 640 -- ext/kenlm/util/double-conversion/bignum-dtoa.h | 84 - ext/kenlm/util/double-conversion/bignum.cc | 764 -- ext/kenlm/util/double-conversion/bignum.h | 145 - .../util/double-conversion/cached-powers.cc | 175 - .../util/double-conversion/cached-powers.h | 64 - ext/kenlm/util/double-conversion/diy-fp.cc | 57 - ext/kenlm/util/double-conversion/diy-fp.h | 118 - .../util/double-conversion/double-conversion.cc | 889 --- .../util/double-conversion/double-conversion.h | 536 -- ext/kenlm/util/double-conversion/fast-dtoa.cc | 664 -- ext/kenlm/util/double-conversion/fast-dtoa.h | 88 - ext/kenlm/util/double-conversion/fixed-dtoa.cc | 402 - ext/kenlm/util/double-conversion/fixed-dtoa.h | 56 - ext/kenlm/util/double-conversion/ieee.h | 398 - ext/kenlm/util/double-conversion/strtod.cc | 558 -- ext/kenlm/util/double-conversion/strtod.h | 45 - ext/kenlm/util/double-conversion/utils.h | 320 - ext/kenlm/util/ersatz_progress.cc | 47 - ext/kenlm/util/ersatz_progress.hh | 57 - ext/kenlm/util/exception.cc | 105 - ext/kenlm/util/exception.hh | 159 - ext/kenlm/util/fake_ostream.hh | 111 - ext/kenlm/util/file.cc | 574 -- ext/kenlm/util/file.hh | 154 - ext/kenlm/util/file_piece.cc | 337 - ext/kenlm/util/file_piece.hh | 175 - ext/kenlm/util/file_piece_test.cc | 154 - ext/kenlm/util/file_stream.hh | 89 - ext/kenlm/util/fixed_array.hh | 176 - ext/kenlm/util/float_to_string.cc | 23 - ext/kenlm/util/float_to_string.hh | 25 - ext/kenlm/util/getopt.c | 78 - ext/kenlm/util/getopt.hh | 33 - ext/kenlm/util/have.hh | 13 - ext/kenlm/util/integer_to_string.cc | 667 -- ext/kenlm/util/integer_to_string.hh | 66 - ext/kenlm/util/integer_to_string_test.cc | 81 - ext/kenlm/util/joint_sort.hh | 146 - ext/kenlm/util/joint_sort_test.cc | 62 - ext/kenlm/util/mmap.cc | 402 - ext/kenlm/util/mmap.hh | 225 - ext/kenlm/util/multi_intersection.hh | 80 - ext/kenlm/util/multi_intersection_test.cc | 63 - ext/kenlm/util/murmur_hash.cc | 175 - ext/kenlm/util/murmur_hash.hh | 18 - ext/kenlm/util/parallel_read.cc | 69 - ext/kenlm/util/parallel_read.hh | 16 - ext/kenlm/util/pcqueue.hh | 156 - ext/kenlm/util/pcqueue_test.cc | 20 - ext/kenlm/util/pool.cc | 38 - ext/kenlm/util/pool.hh | 44 - ext/kenlm/util/probing_hash_table.hh | 421 - .../util/probing_hash_table_benchmark_main.cc | 181 - ext/kenlm/util/probing_hash_table_test.cc | 102 - ext/kenlm/util/proxy_iterator.hh | 101 - ext/kenlm/util/read_compressed.cc | 448 -- ext/kenlm/util/read_compressed.hh | 84 - ext/kenlm/util/read_compressed_test.cc | 130 - ext/kenlm/util/scoped.cc | 43 - ext/kenlm/util/scoped.hh | 111 - ext/kenlm/util/sized_iterator.hh | 120 - ext/kenlm/util/sized_iterator_test.cc | 16 - ext/kenlm/util/sorted_uniform.hh | 105 - ext/kenlm/util/sorted_uniform_test.cc | 127 - ext/kenlm/util/stream/CMakeLists.txt | 50 - ext/kenlm/util/stream/Jamfile | 7 - ext/kenlm/util/stream/block.hh | 93 - ext/kenlm/util/stream/chain.cc | 163 - ext/kenlm/util/stream/chain.hh | 347 - ext/kenlm/util/stream/config.hh | 63 - ext/kenlm/util/stream/count_records.cc | 12 - ext/kenlm/util/stream/count_records.hh | 20 - ext/kenlm/util/stream/io.cc | 78 - ext/kenlm/util/stream/io.hh | 87 - ext/kenlm/util/stream/io_test.cc | 38 - ext/kenlm/util/stream/line_input.cc | 52 - ext/kenlm/util/stream/line_input.hh | 22 - ext/kenlm/util/stream/multi_progress.cc | 86 - ext/kenlm/util/stream/multi_progress.hh | 89 - ext/kenlm/util/stream/multi_stream.hh | 124 - ext/kenlm/util/stream/rewindable_stream.cc | 134 - ext/kenlm/util/stream/rewindable_stream.hh | 132 - ext/kenlm/util/stream/rewindable_stream_test.cc | 41 - ext/kenlm/util/stream/sort.hh | 599 -- ext/kenlm/util/stream/sort_test.cc | 62 - ext/kenlm/util/stream/stream.hh | 77 - ext/kenlm/util/stream/stream_test.cc | 35 - ext/kenlm/util/stream/timer.hh | 16 - ext/kenlm/util/string_piece.cc | 192 - ext/kenlm/util/string_piece.hh | 270 - ext/kenlm/util/string_piece_hash.hh | 43 - ext/kenlm/util/string_stream.hh | 44 - ext/kenlm/util/string_stream_test.cc | 80 - ext/kenlm/util/thread_pool.hh | 94 - ext/kenlm/util/tokenize_piece.hh | 150 - ext/kenlm/util/tokenize_piece_test.cc | 48 - ext/kenlm/util/usage.cc | 305 - ext/kenlm/util/usage.hh | 27 - ext/kenlm/windows/build_binary.vcxproj | 174 - ext/kenlm/windows/kenlm.sln | 64 - ext/kenlm/windows/kenlm.vcxproj | 291 - ext/kenlm/windows/lmplz.vcxproj | 214 - ext/kenlm/windows/ngram_query.vcxproj | 177 - jni/build_kenlm.sh | 2 +- 558 files changed, 7 insertions(+), 121077 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/.gitmodules ---------------------------------------------------------------------- diff --git a/.gitmodules b/.gitmodules index 65e0ea5..b14568b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,7 @@ [submodule "berkeleylm"] path = ext/berkeleylm url = https://github.com/joshua-decoder/berkeleylm.git +[submodule "ext/kenlm"] + path = ext/kenlm + url = https://github.com/kpu/kenlm.git + branch = 56fdb5c44fca34d5a2e07d96139c28fb163983c5 http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/build.xml ---------------------------------------------------------------------- diff --git a/build.xml b/build.xml index 7095ca2..c0c6132 100644 --- a/build.xml +++ b/build.xml @@ -63,7 +63,7 @@ <!-- TODO: add suggestion to use /System/Library/Frameworks/JavaVM.framework/Home/ iff on OSX --> </target> - <target name="kenlm" depends="check-joshua-home" description="--> Build KenLM"> + <target name="kenlm" depends="check-joshua-home,download-submodules" description="--> Build KenLM"> <exec dir="${JOSHUA}/jni" executable="/bin/bash"> <env key="JOSHUA" path="${JOSHUA}" /> <arg line="build_kenlm.sh" />
