http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/make1.c ---------------------------------------------------------------------- diff --git a/ext/kenlm b/ext/kenlm new file mode 160000 index 0000000..56fdb5c --- /dev/null +++ b/ext/kenlm @@ -0,0 +1 @@ +Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5 diff --git a/ext/kenlm/jam-files/engine/make1.c b/ext/kenlm/jam-files/engine/make1.c deleted file mode 100644 index 71eee12..0000000 --- a/ext/kenlm/jam-files/engine/make1.c +++ /dev/null @@ -1,1283 +0,0 @@ -/* - * Copyright 1993-2002 Christopher Seiwald and Perforce Software, Inc. - * - * This file is part of Jam - see jam.c for Copyright information. - */ - -/* This file is ALSO: - * Copyright 2001-2004 David Abrahams. - * Distributed under the Boost Software License, Version 1.0. - * (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt) - */ - -/* - * make1.c - execute commands to bring targets up to date - * - * This module contains make1(), the entry point called by make() to recursively - * descend the dependency graph executing update actions as marked by make0(). - * - * External routines: - * make1() - execute commands to update a TARGET and all of its dependencies - * - * Internal routines, the recursive/asynchronous command executors: - * make1a() - recursively schedules dependency builds and then goes to - * MAKE1B - * make1b() - if nothing is blocking this target's build, proceed to - * MAKE1C - * make1c() - launch target's next command, or go to parents' MAKE1B - * if none - * make1c_closure() - handle command execution completion and go to MAKE1C - * - * Internal support routines: - * make1cmds() - turn ACTIONS into CMDs, grouping, splitting, etc. - * make1list() - turn a list of targets into a LIST, for $(<) and $(>) - * make1settings() - for vars with bound values, build up replacement lists - * make1bind() - bind targets that weren't bound in dependency analysis - */ - -#include "jam.h" -#include "make.h" - -#include "command.h" -#include "compile.h" -#include "execcmd.h" -#include "headers.h" -#include "lists.h" -#include "object.h" -#include "output.h" -#include "parse.h" -#include "rules.h" -#include "search.h" -#include "variable.h" - -#include <assert.h> -#include <stdlib.h> - -#if !defined( NT ) || defined( __GNUC__ ) - #include <unistd.h> /* for unlink */ -#endif - -static CMD * make1cmds ( TARGET * ); -static LIST * make1list ( LIST *, TARGETS *, int flags ); -static SETTINGS * make1settings ( struct module_t *, LIST * vars ); -static void make1bind ( TARGET * ); -static TARGET * make1findcycle ( TARGET * ); -static void make1breakcycle( TARGET *, TARGET * cycle_root ); - -/* Ugly static - it is too hard to carry it through the callbacks. */ - -static struct -{ - int failed; - int skipped; - int total; - int made; -} counts[ 1 ]; - -/* Target state. */ -#define T_STATE_MAKE1A 0 /* make1a() should be called */ -#define T_STATE_MAKE1B 1 /* make1b() should be called */ -#define T_STATE_MAKE1C 2 /* make1c() should be called */ - -typedef struct _state state; -struct _state -{ - state * prev; /* previous state on stack */ - TARGET * t; /* current target */ - TARGET * parent; /* parent argument necessary for MAKE1A */ - int curstate; /* current state */ -}; - -static void make1a( state * const ); -static void make1b( state * const ); -static void make1c( state const * const ); - -static void make1c_closure( void * const closure, int status, - timing_info const * const, char const * const cmd_stdout, - char const * const cmd_stderr, int const cmd_exit_reason ); - -typedef struct _stack -{ - state * stack; -} stack; - -static stack state_stack = { NULL }; - -static state * state_freelist = NULL; - -/* Currently running command counter. */ -static int cmdsrunning; - - -static state * alloc_state() -{ - if ( state_freelist ) - { - state * const pState = state_freelist; - state_freelist = pState->prev; - memset( pState, 0, sizeof( state ) ); - return pState; - } - return (state *)BJAM_MALLOC( sizeof( state ) ); -} - - -static void free_state( state * const pState ) -{ - pState->prev = state_freelist; - state_freelist = pState; -} - - -static void clear_state_freelist() -{ - while ( state_freelist ) - { - state * const pState = state_freelist; - state_freelist = state_freelist->prev; - BJAM_FREE( pState ); - } -} - - -static state * current_state( stack * const pStack ) -{ - return pStack->stack; -} - - -static void pop_state( stack * const pStack ) -{ - if ( pStack->stack ) - { - state * const pState = pStack->stack->prev; - free_state( pStack->stack ); - pStack->stack = pState; - } -} - - -static state * push_state( stack * const pStack, TARGET * const t, - TARGET * const parent, int const curstate ) -{ - state * const pState = alloc_state(); - pState->t = t; - pState->parent = parent; - pState->prev = pStack->stack; - pState->curstate = curstate; - return pStack->stack = pState; -} - - -/* - * Pushes a stack onto another stack, effectively reversing the order. - */ - -static void push_stack_on_stack( stack * const pDest, stack * const pSrc ) -{ - while ( pSrc->stack ) - { - state * const pState = pSrc->stack; - pSrc->stack = pState->prev; - pState->prev = pDest->stack; - pDest->stack = pState; - } -} - - -/* - * make1() - execute commands to update a list of targets and all of their dependencies - */ - -static int intr = 0; -static int quit = 0; - -int make1( LIST * targets ) -{ - state * pState; - int status = 0; - - memset( (char *)counts, 0, sizeof( *counts ) ); - - { - LISTITER iter, end; - stack temp_stack = { NULL }; - for ( iter = list_begin( targets ), end = list_end( targets ); - iter != end; iter = list_next( iter ) ) - push_state( &temp_stack, bindtarget( list_item( iter ) ), NULL, T_STATE_MAKE1A ); - push_stack_on_stack( &state_stack, &temp_stack ); - } - - /* Clear any state left over from the past */ - quit = 0; - - /* Recursively make the target and its dependencies. */ - - while ( 1 ) - { - while ( ( pState = current_state( &state_stack ) ) ) - { - if ( quit ) - pop_state( &state_stack ); - - switch ( pState->curstate ) - { - case T_STATE_MAKE1A: make1a( pState ); break; - case T_STATE_MAKE1B: make1b( pState ); break; - case T_STATE_MAKE1C: make1c( pState ); break; - default: - assert( !"make1(): Invalid state detected." ); - } - } - if ( !cmdsrunning ) - break; - /* Wait for outstanding commands to finish running. */ - exec_wait(); - } - - clear_state_freelist(); - - /* Talk about it. */ - if ( counts->failed ) - printf( "...failed updating %d target%s...\n", counts->failed, - counts->failed > 1 ? "s" : "" ); - if ( DEBUG_MAKE && counts->skipped ) - printf( "...skipped %d target%s...\n", counts->skipped, - counts->skipped > 1 ? "s" : "" ); - if ( DEBUG_MAKE && counts->made ) - printf( "...updated %d target%s...\n", counts->made, - counts->made > 1 ? "s" : "" ); - - /* If we were interrupted, exit now that all child processes - have finished. */ - if ( intr ) - exit( 1 ); - - { - LISTITER iter, end; - for ( iter = list_begin( targets ), end = list_end( targets ); - iter != end; iter = list_next( iter ) ) - { - /* Check that the target was updated and that the - update succeeded. */ - TARGET * t = bindtarget( list_item( iter ) ); - if (t->progress == T_MAKE_DONE) - { - if (t->status != EXEC_CMD_OK) - status = 1; - } - else if ( ! ( t->progress == T_MAKE_NOEXEC_DONE && globs.noexec ) ) - { - status = 1; - } - } - } - return status; -} - - -/* - * make1a() - recursively schedules dependency builds and then goes to MAKE1B - * - * Called to start processing a specified target. Does nothing if the target is - * already being processed or otherwise starts processing all of its - * dependencies. - */ - -static void make1a( state * const pState ) -{ - TARGET * t = pState->t; - TARGET * const scc_root = target_scc( t ); - - if ( !pState->parent || target_scc( pState->parent ) != scc_root ) - pState->t = t = scc_root; - - /* If the parent is the first to try to build this target or this target is - * in the MAKE1C quagmire, arrange for the parent to be notified when this - * target has been built. - */ - if ( pState->parent && t->progress <= T_MAKE_RUNNING ) - { - TARGET * const parent_scc = target_scc( pState->parent ); - if ( t != parent_scc ) - { - t->parents = targetentry( t->parents, parent_scc ); - ++parent_scc->asynccnt; - } - } - - /* If the target has been previously updated with -n in effect, and we are - * now ignoring -n, update it for real. E.g. if the UPDATE_NOW rule was - * called for it twice - first with the -n option and then without. - */ - if ( !globs.noexec && t->progress == T_MAKE_NOEXEC_DONE ) - t->progress = T_MAKE_INIT; - - /* If this target is already being processed then do nothing. There is no - * need to start processing the same target all over again. - */ - if ( t->progress != T_MAKE_INIT ) - { - pop_state( &state_stack ); - return; - } - - /* Guard against circular dependencies. */ - t->progress = T_MAKE_ONSTACK; - - /* 'asynccnt' counts the dependencies preventing this target from proceeding - * to MAKE1C for actual building. We start off with a count of 1 to prevent - * anything from happening until we can notify all dependencies that they - * are needed. This 1 is then accounted for when we enter MAKE1B ourselves, - * below. Without this if a dependency gets built before we finish - * processing all of our other dependencies our build might be triggerred - * prematurely. - */ - t->asynccnt = 1; - - /* Push dependency build requests (to be executed in the natural order). */ - { - stack temp_stack = { NULL }; - TARGETS * c; - for ( c = t->depends; c && !quit; c = c->next ) - push_state( &temp_stack, c->target, t, T_STATE_MAKE1A ); - push_stack_on_stack( &state_stack, &temp_stack ); - } - - t->progress = T_MAKE_ACTIVE; - - /* Once all of our dependencies have started getting processed we can move - * onto MAKE1B. - */ - /* Implementation note: - * In theory this would be done by popping this state before pushing - * dependency target build requests but as a slight optimization we simply - * modify our current state and leave it on the stack instead. - */ - pState->curstate = T_STATE_MAKE1B; -} - - -/* - * make1b() - if nothing is blocking this target's build, proceed to MAKE1C - * - * Called after something stops blocking this target's build, e.g. that all of - * its dependencies have started being processed, one of its dependencies has - * been built or a semaphore this target has been waiting for is free again. - */ - -static void make1b( state * const pState ) -{ - TARGET * const t = pState->t; - TARGET * failed = 0; - char const * failed_name = "dependencies"; - - /* If any dependencies are still outstanding, wait until they signal their - * completion by pushing this same state for their parent targets. - */ - if ( --t->asynccnt ) - { - pop_state( &state_stack ); - return; - } - - /* Try to aquire a semaphore. If it is locked, wait until the target that - * locked it is built and signals completition. - */ -#ifdef OPT_SEMAPHORE - if ( t->semaphore && t->semaphore->asynccnt ) - { - /* Append 't' to the list of targets waiting on semaphore. */ - t->semaphore->parents = targetentry( t->semaphore->parents, t ); - t->asynccnt++; - - if ( DEBUG_EXECCMD ) - printf( "SEM: %s is busy, delaying launch of %s\n", - object_str( t->semaphore->name ), object_str( t->name ) ); - pop_state( &state_stack ); - return; - } -#endif - - /* Now ready to build target 't', if dependencies built OK. */ - - /* Collect status from dependencies. If -n was passed then act as though all - * dependencies built correctly (the only way they can fail is if UPDATE_NOW - * was called). If the dependencies can not be found or we got an interrupt, - * we can not get here. - */ - if ( !globs.noexec ) - { - TARGETS * c; - for ( c = t->depends; c; c = c->next ) - if ( c->target->status > t->status && !( c->target->flags & - T_FLAG_NOCARE ) ) - { - failed = c->target; - t->status = c->target->status; - } - } - - /* If an internal header node failed to build, we want to output the target - * that it failed on. - */ - if ( failed ) - failed_name = failed->flags & T_FLAG_INTERNAL - ? failed->failed - : object_str( failed->name ); - t->failed = failed_name; - - /* If actions for building any of the dependencies have failed, bail. - * Otherwise, execute all actions to make the current target. - */ - if ( ( t->status == EXEC_CMD_FAIL ) && t->actions ) - { - ++counts->skipped; - if ( ( t->flags & ( T_FLAG_RMOLD | T_FLAG_NOTFILE ) ) == T_FLAG_RMOLD ) - { - if ( !unlink( object_str( t->boundname ) ) ) - printf( "...removing outdated %s\n", object_str( t->boundname ) - ); - } - else - printf( "...skipped %s for lack of %s...\n", object_str( t->name ), - failed_name ); - } - - if ( t->status == EXEC_CMD_OK ) - switch ( t->fate ) - { - case T_FATE_STABLE: - case T_FATE_NEWER: - break; - - case T_FATE_CANTFIND: - case T_FATE_CANTMAKE: - t->status = EXEC_CMD_FAIL; - break; - - case T_FATE_ISTMP: - if ( DEBUG_MAKE ) - printf( "...using %s...\n", object_str( t->name ) ); - break; - - case T_FATE_TOUCHED: - case T_FATE_MISSING: - case T_FATE_NEEDTMP: - case T_FATE_OUTDATED: - case T_FATE_UPDATE: - case T_FATE_REBUILD: - /* Prepare commands for executing actions scheduled for this target. - * Commands have their embedded variables automatically expanded, - * including making use of any "on target" variables. - */ - if ( t->actions ) - { - ++counts->total; - if ( DEBUG_MAKE && !( counts->total % 100 ) ) - printf( "...on %dth target...\n", counts->total ); - - t->cmds = (char *)make1cmds( t ); - /* Update the target's "progress" so MAKE1C processing counts it - * among its successes/failures. - */ - t->progress = T_MAKE_RUNNING; - } - break; - - /* All valid fates should have been accounted for by now. */ - default: - printf( "ERROR: %s has bad fate %d", object_str( t->name ), - t->fate ); - abort(); - } - -#ifdef OPT_SEMAPHORE - /* If there is a semaphore, indicate that it is in use. */ - if ( t->semaphore ) - { - ++t->semaphore->asynccnt; - if ( DEBUG_EXECCMD ) - printf( "SEM: %s now used by %s\n", object_str( t->semaphore->name - ), object_str( t->name ) ); - } -#endif - - /* Proceed to MAKE1C to begin executing the chain of commands prepared for - * building the target. If we are not going to build the target (e.g. due to - * dependency failures or no commands needing to be run) the chain will be - * empty and MAKE1C processing will directly signal the target's completion. - */ - /* Implementation note: - * Morfing the current state on the stack instead of popping it and - * pushing a new one is a slight optimization with no side-effects since we - * pushed no other states while processing this one. - */ - pState->curstate = T_STATE_MAKE1C; -} - - -/* - * make1c() - launch target's next command, or go to parents' MAKE1B if none - * - * If there are (more) commands to run to build this target (and we have not hit - * an error running earlier comands) we launch the command using exec_cmd(). - * Command execution signals its completion in exec_wait() by calling our - * make1c_closure() callback. - * - * If there are no more commands to run, we collect the status from all the - * actions and report our completion to all the parents. - */ - -static void make1c( state const * const pState ) -{ - TARGET * const t = pState->t; - CMD * const cmd = (CMD *)t->cmds; - - if ( cmd && t->status == EXEC_CMD_OK ) - { - /* Pop state first in case something below (e.g. exec_cmd(), exec_wait() - * or make1c_closure()) pushes a new state. Note that we must not access - * the popped state data after this as the same stack node might have - * been reused internally for some newly pushed state. - */ - pop_state( &state_stack ); - - /* Increment the jobs running counter. */ - ++cmdsrunning; - - /* Execute the actual build command or fake it if no-op. */ - if ( globs.noexec || cmd->noop ) - { - timing_info time_info = { 0 }; - timestamp_current( &time_info.start ); - timestamp_copy( &time_info.end, &time_info.start ); - make1c_closure( t, EXEC_CMD_OK, &time_info, "", "", EXIT_OK ); - } - else - { - exec_cmd( cmd->buf, make1c_closure, t, cmd->shell ); - - /* Wait until under the concurrent command count limit. */ - /* FIXME: This wait could be skipped here and moved to just before - * trying to execute a command that would cross the command count - * limit. Note though that this might affect the order in which - * unrelated targets get built and would thus require that all - * affected Boost Build tests be updated. - */ - assert( 0 < globs.jobs ); - assert( globs.jobs <= MAXJOBS ); - while ( cmdsrunning >= globs.jobs ) - exec_wait(); - } - } - else - { - ACTIONS * actions; - - /* Collect status from actions, and distribute it as well. */ - for ( actions = t->actions; actions; actions = actions->next ) - if ( actions->action->status > t->status ) - t->status = actions->action->status; - for ( actions = t->actions; actions; actions = actions->next ) - if ( t->status > actions->action->status ) - actions->action->status = t->status; - - /* Tally success/failure for those we tried to update. */ - if ( t->progress == T_MAKE_RUNNING ) - switch ( t->status ) - { - case EXEC_CMD_OK: ++counts->made; break; - case EXEC_CMD_FAIL: ++counts->failed; break; - } - - /* Tell parents their dependency has been built. */ - { - TARGETS * c; - stack temp_stack = { NULL }; - TARGET * additional_includes = NULL; - - t->progress = globs.noexec ? T_MAKE_NOEXEC_DONE : T_MAKE_DONE; - - /* Target has been updated so rescan it for dependencies. */ - if ( t->fate >= T_FATE_MISSING && t->status == EXEC_CMD_OK && - !( t->flags & T_FLAG_INTERNAL ) ) - { - TARGET * saved_includes; - SETTINGS * s; - - t->rescanned = 1; - - /* Clean current includes. */ - saved_includes = t->includes; - t->includes = 0; - - s = copysettings( t->settings ); - pushsettings( root_module(), s ); - headers( t ); - popsettings( root_module(), s ); - freesettings( s ); - - if ( t->includes ) - { - /* Tricky. The parents have already been processed, but they - * have not seen the internal node, because it was just - * created. We need to: - * - push MAKE1A states that would have been pushed by the - * parents here - * - make sure all unprocessed parents will pick up the - * new includes - * - make sure processing the additional MAKE1A states is - * done before processing the MAKE1B state for our - * current target (which would mean this target has - * already been built), otherwise the parent would be - * considered built before the additional MAKE1A state - * processing even got a chance to start. - */ - make0( t->includes, t->parents->target, 0, 0, 0, t->includes - ); - /* Link the old includes on to make sure that it gets - * cleaned up correctly. - */ - t->includes->includes = saved_includes; - for ( c = t->dependants; c; c = c->next ) - c->target->depends = targetentry( c->target->depends, - t->includes ); - /* Will be processed below. */ - additional_includes = t->includes; - } - else - { - t->includes = saved_includes; - } - } - - if ( additional_includes ) - for ( c = t->parents; c; c = c->next ) - push_state( &temp_stack, additional_includes, c->target, - T_STATE_MAKE1A ); - - if ( t->scc_root ) - { - TARGET * const scc_root = target_scc( t ); - assert( scc_root->progress < T_MAKE_DONE ); - for ( c = t->parents; c; c = c->next ) - { - if ( target_scc( c->target ) == scc_root ) - push_state( &temp_stack, c->target, NULL, T_STATE_MAKE1B - ); - else - scc_root->parents = targetentry( scc_root->parents, - c->target ); - } - } - else - { - for ( c = t->parents; c; c = c->next ) - push_state( &temp_stack, c->target, NULL, T_STATE_MAKE1B ); - } - -#ifdef OPT_SEMAPHORE - /* If there is a semaphore, it is now free. */ - if ( t->semaphore ) - { - assert( t->semaphore->asynccnt == 1 ); - --t->semaphore->asynccnt; - - if ( DEBUG_EXECCMD ) - printf( "SEM: %s is now free\n", object_str( - t->semaphore->name ) ); - - /* If anything is waiting, notify the next target. There is no - * point in notifying all waiting targets, since they will be - * notified again. - */ - if ( t->semaphore->parents ) - { - TARGETS * first = t->semaphore->parents; - t->semaphore->parents = first->next; - if ( first->next ) - first->next->tail = first->tail; - - if ( DEBUG_EXECCMD ) - printf( "SEM: placing %s on stack\n", object_str( - first->target->name ) ); - push_state( &temp_stack, first->target, NULL, T_STATE_MAKE1B - ); - BJAM_FREE( first ); - } - } -#endif - - /* Must pop state before pushing any more. */ - pop_state( &state_stack ); - - /* Using stacks reverses the order of execution. Reverse it back. */ - push_stack_on_stack( &state_stack, &temp_stack ); - } - } -} - - -/* - * call_timing_rule() - Look up the __TIMING_RULE__ variable on the given - * target, and if non-empty, invoke the rule it names, passing the given - * timing_info. - */ - -static void call_timing_rule( TARGET * target, timing_info const * const time ) -{ - LIST * timing_rule; - - pushsettings( root_module(), target->settings ); - timing_rule = var_get( root_module(), constant_TIMING_RULE ); - popsettings( root_module(), target->settings ); - - if ( !list_empty( timing_rule ) ) - { - /* rule timing-rule ( args * : target : start end user system ) */ - - /* Prepare the argument list. */ - FRAME frame[ 1 ]; - OBJECT * rulename = list_front( timing_rule ); - frame_init( frame ); - - /* args * :: $(__TIMING_RULE__[2-]) */ - lol_add( frame->args, list_copy_range( timing_rule, list_next( - list_begin( timing_rule ) ), list_end( timing_rule ) ) ); - - /* target :: the name of the target */ - lol_add( frame->args, list_new( object_copy( target->name ) ) ); - - /* start end user system :: info about the action command */ - lol_add( frame->args, list_push_back( list_push_back( list_push_back( list_new( - outf_time( &time->start ) ), - outf_time( &time->end ) ), - outf_double( time->user ) ), - outf_double( time->system ) ) ); - - /* Call the rule. */ - evaluate_rule( bindrule( rulename , root_module() ), rulename, frame ); - - /* Clean up. */ - frame_free( frame ); - } -} - - -/* - * call_action_rule() - Look up the __ACTION_RULE__ variable on the given - * target, and if non-empty, invoke the rule it names, passing the given info, - * timing_info, executed command and command output. - */ - -static void call_action_rule -( - TARGET * target, - int status, - timing_info const * time, - char const * executed_command, - char const * command_output -) -{ - LIST * action_rule; - - pushsettings( root_module(), target->settings ); - action_rule = var_get( root_module(), constant_ACTION_RULE ); - popsettings( root_module(), target->settings ); - - if ( !list_empty( action_rule ) ) - { - /* rule action-rule ( - args * : - target : - command status start end user system : - output ? ) */ - - /* Prepare the argument list. */ - FRAME frame[ 1 ]; - OBJECT * rulename = list_front( action_rule ); - frame_init( frame ); - - /* args * :: $(__ACTION_RULE__[2-]) */ - lol_add( frame->args, list_copy_range( action_rule, list_next( - list_begin( action_rule ) ), list_end( action_rule ) ) ); - - /* target :: the name of the target */ - lol_add( frame->args, list_new( object_copy( target->name ) ) ); - - /* command status start end user system :: info about the action command - */ - lol_add( frame->args, - list_push_back( list_push_back( list_push_back( list_push_back( list_push_back( list_new( - object_new( executed_command ) ), - outf_int( status ) ), - outf_time( &time->start ) ), - outf_time( &time->end ) ), - outf_double( time->user ) ), - outf_double( time->system ) ) ); - - /* output ? :: the output of the action command */ - if ( command_output ) - lol_add( frame->args, list_new( object_new( command_output ) ) ); - else - lol_add( frame->args, L0 ); - - /* Call the rule. */ - evaluate_rule( bindrule( rulename, root_module() ), rulename, frame ); - - /* Clean up. */ - frame_free( frame ); - } -} - - -/* - * make1c_closure() - handle command execution completion and go to MAKE1C. - * - * Internal function passed as a notification callback for when a command - * finishes getting executed by the OS or called directly when faking that a - * command had been executed by the OS. - * - * Now all we need to do is fiddle with the command exit status and push a new - * MAKE1C state to execute the next command scheduled for building this target - * or close up the target's build process in case there are no more commands - * scheduled for it. On interrupts, we bail heavily. - */ - -static void make1c_closure -( - void * const closure, - int status_orig, - timing_info const * const time, - char const * const cmd_stdout, - char const * const cmd_stderr, - int const cmd_exit_reason -) -{ - TARGET * const t = (TARGET *)closure; - CMD * const cmd = (CMD *)t->cmds; - char const * rule_name = 0; - char const * target_name = 0; - - assert( cmd ); - - --cmdsrunning; - - /* Calculate the target's status from the cmd execution result. */ - { - /* Store the target's status. */ - t->status = status_orig; - - /* Invert OK/FAIL target status when FAIL_EXPECTED has been applied. */ - if ( t->flags & T_FLAG_FAIL_EXPECTED && !globs.noexec ) - { - switch ( t->status ) - { - case EXEC_CMD_FAIL: t->status = EXEC_CMD_OK; break; - case EXEC_CMD_OK: t->status = EXEC_CMD_FAIL; break; - } - } - - /* Ignore failures for actions marked as 'ignore'. */ - if ( t->status == EXEC_CMD_FAIL && cmd->rule->actions->flags & - RULE_IGNORE ) - t->status = EXEC_CMD_OK; - } - - if ( DEBUG_MAKEQ || - ( DEBUG_MAKE && !( cmd->rule->actions->flags & RULE_QUIETLY ) ) ) - { - rule_name = object_str( cmd->rule->name ); - target_name = object_str( list_front( lol_get( (LOL *)&cmd->args, 0 ) ) - ); - } - - out_action( rule_name, target_name, cmd->buf->value, cmd_stdout, cmd_stderr, - cmd_exit_reason ); - - if ( !globs.noexec ) - { - call_timing_rule( t, time ); - if ( DEBUG_EXECCMD ) - printf( "%f sec system; %f sec user\n", time->system, time->user ); - - /* Assume -p0 is in effect, i.e. cmd_stdout contains merged output. */ - call_action_rule( t, status_orig, time, cmd->buf->value, cmd_stdout ); - } - - /* Print command text on failure. */ - if ( t->status == EXEC_CMD_FAIL && DEBUG_MAKE ) - { - if ( !DEBUG_EXEC ) - printf( "%s\n", cmd->buf->value ); - - printf( "...failed %s ", object_str( cmd->rule->name ) ); - list_print( lol_get( (LOL *)&cmd->args, 0 ) ); - printf( "...\n" ); - } - - /* On interrupt, set quit so _everything_ fails. Do the same for failed - * commands if we were asked to stop the build in case of any errors. - */ - if ( t->status == EXEC_CMD_INTR ) - { - ++intr; - ++quit; - } - if ( t->status == EXEC_CMD_FAIL && globs.quitquick ) - ++quit; - - /* If the command was not successful remove all of its targets not marked as - * "precious". - */ - if ( t->status != EXEC_CMD_OK ) - { - LIST * const targets = lol_get( (LOL *)&cmd->args, 0 ); - LISTITER iter = list_begin( targets ); - LISTITER const end = list_end( targets ); - for ( ; iter != end; iter = list_next( iter ) ) - { - char const * const filename = object_str( list_item( iter ) ); - TARGET const * const t = bindtarget( list_item( iter ) ); - if ( !( t->flags & T_FLAG_PRECIOUS ) && !unlink( filename ) ) - printf( "...removing %s\n", filename ); - } - } - - /* Free this command and push the MAKE1C state to execute the next one - * scheduled for building this same target. - */ - t->cmds = (char *)cmd_next( cmd ); - cmd_free( cmd ); - push_state( &state_stack, t, NULL, T_STATE_MAKE1C ); -} - - -/* - * swap_settings() - replace the settings from the current module and target - * with those from the new module and target - */ - -static void swap_settings -( - module_t * * current_module, - TARGET * * current_target, - module_t * new_module, - TARGET * new_target -) -{ - if ( ( new_target == *current_target ) && - ( new_module == *current_module ) ) - return; - - if ( *current_target ) - popsettings( *current_module, (*current_target)->settings ); - - if ( new_target ) - pushsettings( new_module, new_target->settings ); - - *current_module = new_module; - *current_target = new_target; -} - - -/* - * make1cmds() - turn ACTIONS into CMDs, grouping, splitting, etc. - * - * Essentially copies a chain of ACTIONs to a chain of CMDs, grouping - * RULE_TOGETHER actions, splitting RULE_PIECEMEAL actions, and handling - * RULE_NEWSRCS actions. The result is a chain of CMDs which has already had all - * of its embedded variable references expanded and can now be executed using - * exec_cmd(). - */ - -static CMD * make1cmds( TARGET * t ) -{ - CMD * cmds = 0; - CMD * * cmds_next = &cmds; - LIST * shell = L0; - module_t * settings_module = 0; - TARGET * settings_target = 0; - ACTIONS * a0; - int const running_flag = globs.noexec ? A_RUNNING_NOEXEC : A_RUNNING; - - /* Step through actions. Actions may be shared with other targets or grouped - * using RULE_TOGETHER, so actions already seen are skipped. - */ - for ( a0 = t->actions; a0; a0 = a0->next ) - { - RULE * rule = a0->action->rule; - rule_actions * actions = rule->actions; - SETTINGS * boundvars; - LIST * nt; - LIST * ns; - ACTIONS * a1; - - /* Only do rules with commands to execute. If this action has already - * been executed, use saved status. - */ - if ( !actions || a0->action->running >= running_flag ) - continue; - - a0->action->running = running_flag; - - /* Make LISTS of targets and sources. If `execute together` has been - * specified for this rule, tack on sources from each instance of this - * rule for this target. - */ - nt = make1list( L0, a0->action->targets, 0 ); - ns = make1list( L0, a0->action->sources, actions->flags ); - if ( actions->flags & RULE_TOGETHER ) - for ( a1 = a0->next; a1; a1 = a1->next ) - if ( a1->action->rule == rule && - a1->action->running < running_flag ) - { - ns = make1list( ns, a1->action->sources, actions->flags ); - a1->action->running = running_flag; - } - - /* If doing only updated (or existing) sources, but none have been - * updated (or exist), skip this action. - */ - if ( list_empty( ns ) && - ( actions->flags & ( RULE_NEWSRCS | RULE_EXISTING ) ) ) - { - list_free( nt ); - continue; - } - - swap_settings( &settings_module, &settings_target, rule->module, t ); - if ( list_empty( shell ) ) - { - /* shell is per-target */ - shell = var_get( rule->module, constant_JAMSHELL ); - } - - /* If we had 'actions xxx bind vars' we bind the vars now. */ - boundvars = make1settings( rule->module, actions->bindlist ); - pushsettings( rule->module, boundvars ); - - /* - * Build command, starting with all source args. - * - * For actions that allow PIECEMEAL commands, if the constructed command - * string is too long, we retry constructing it with a reduced number of - * source arguments presented. - * - * While reducing slowly takes a bit of compute time to get things just - * right, it is worth it to get as close to maximum allowed command - * string length as possible, because launching the commands we are - * executing is likely to be much more compute intensive. - * - * Note that we loop through at least once, for sourceless actions. - */ - { - int const length = list_length( ns ); - int start = 0; - int chunk = length; - LIST * cmd_targets = L0; - LIST * cmd_shell = L0; - do - { - CMD * cmd; - int cmd_check_result; - int cmd_error_length; - int cmd_error_max_length; - int retry = 0; - int accept_command = 0; - - /* Build cmd: cmd_new() takes ownership of its lists. */ - if ( list_empty( cmd_targets ) ) cmd_targets = list_copy( nt ); - if ( list_empty( cmd_shell ) ) cmd_shell = list_copy( shell ); - cmd = cmd_new( rule, cmd_targets, list_sublist( ns, start, - chunk ), cmd_shell ); - - cmd_check_result = exec_check( cmd->buf, &cmd->shell, - &cmd_error_length, &cmd_error_max_length ); - - if ( cmd_check_result == EXEC_CHECK_OK ) - { - accept_command = 1; - } - else if ( cmd_check_result == EXEC_CHECK_NOOP ) - { - accept_command = 1; - cmd->noop = 1; - } - else if ( ( actions->flags & RULE_PIECEMEAL ) && ( chunk > 1 ) ) - { - /* Too long but splittable. Reduce chunk size slowly and - * retry. - */ - assert( cmd_check_result == EXEC_CHECK_TOO_LONG || - cmd_check_result == EXEC_CHECK_LINE_TOO_LONG ); - chunk = chunk * 9 / 10; - retry = 1; - } - else - { - /* Too long and not splittable. */ - char const * const error_message = cmd_check_result == - EXEC_CHECK_TOO_LONG - ? "is too long" - : "contains a line that is too long"; - assert( cmd_check_result == EXEC_CHECK_TOO_LONG || - cmd_check_result == EXEC_CHECK_LINE_TOO_LONG ); - printf( "%s action %s (%d, max %d):\n", object_str( - rule->name ), error_message, cmd_error_length, - cmd_error_max_length ); - - /* Tell the user what did not fit. */ - fputs( cmd->buf->value, stdout ); - exit( EXITBAD ); - } - - assert( !retry || !accept_command ); - - if ( accept_command ) - { - /* Chain it up. */ - *cmds_next = cmd; - cmds_next = &cmd->next; - - /* Mark lists we need recreated for the next command since - * they got consumed by the cmd object. - */ - cmd_targets = L0; - cmd_shell = L0; - } - else - { - /* We can reuse targets & shell lists for the next command - * if we do not let them die with this cmd object. - */ - cmd_release_targets_and_shell( cmd ); - cmd_free( cmd ); - } - - if ( !retry ) - start += chunk; - } - while ( start < length ); - } - - /* These were always copied when used. */ - list_free( nt ); - list_free( ns ); - - /* Free variables with values bound by 'actions xxx bind vars'. */ - popsettings( rule->module, boundvars ); - freesettings( boundvars ); - } - - swap_settings( &settings_module, &settings_target, 0, 0 ); - return cmds; -} - - -/* - * make1list() - turn a list of targets into a LIST, for $(<) and $(>) - */ - -static LIST * make1list( LIST * l, TARGETS * targets, int flags ) -{ - for ( ; targets; targets = targets->next ) - { - TARGET * t = targets->target; - - if ( t->binding == T_BIND_UNBOUND ) - make1bind( t ); - - if ( ( flags & RULE_EXISTING ) && ( flags & RULE_NEWSRCS ) ) - { - if ( ( t->binding != T_BIND_EXISTS ) && - ( t->fate <= T_FATE_STABLE ) ) - continue; - } - else if ( flags & RULE_EXISTING ) - { - if ( t->binding != T_BIND_EXISTS ) - continue; - } - else if ( flags & RULE_NEWSRCS ) - { - if ( t->fate <= T_FATE_STABLE ) - continue; - } - - /* Prohibit duplicates for RULE_TOGETHER. */ - if ( flags & RULE_TOGETHER ) - { - LISTITER iter = list_begin( l ); - LISTITER const end = list_end( l ); - for ( ; iter != end; iter = list_next( iter ) ) - if ( object_equal( list_item( iter ), t->boundname ) ) - break; - if ( iter != end ) - continue; - } - - /* Build new list. */ - l = list_push_back( l, object_copy( t->boundname ) ); - } - - return l; -} - - -/* - * make1settings() - for vars with bound values, build up replacement lists - */ - -static SETTINGS * make1settings( struct module_t * module, LIST * vars ) -{ - SETTINGS * settings = 0; - - LISTITER vars_iter = list_begin( vars ); - LISTITER const vars_end = list_end( vars ); - for ( ; vars_iter != vars_end; vars_iter = list_next( vars_iter ) ) - { - LIST * const l = var_get( module, list_item( vars_iter ) ); - LIST * nl = L0; - LISTITER iter = list_begin( l ); - LISTITER const end = list_end( l ); - - for ( ; iter != end; iter = list_next( iter ) ) - { - TARGET * const t = bindtarget( list_item( iter ) ); - - /* Make sure the target is bound. */ - if ( t->binding == T_BIND_UNBOUND ) - make1bind( t ); - - /* Build a new list. */ - nl = list_push_back( nl, object_copy( t->boundname ) ); - } - - /* Add to settings chain. */ - settings = addsettings( settings, VAR_SET, list_item( vars_iter ), nl ); - } - - return settings; -} - - -/* - * make1bind() - bind targets that were not bound during dependency analysis - * - * Spot the kludge! If a target is not in the dependency tree, it did not get - * bound by make0(), so we have to do it here. Ugly. - */ - -static void make1bind( TARGET * t ) -{ - if ( t->flags & T_FLAG_NOTFILE ) - return; - - pushsettings( root_module(), t->settings ); - object_free( t->boundname ); - t->boundname = search( t->name, &t->time, 0, t->flags & T_FLAG_ISFILE ); - t->binding = timestamp_empty( &t->time ) ? T_BIND_MISSING : T_BIND_EXISTS; - popsettings( root_module(), t->settings ); -}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/md5.c ---------------------------------------------------------------------- diff --git a/ext/kenlm b/ext/kenlm new file mode 160000 index 0000000..56fdb5c --- /dev/null +++ b/ext/kenlm @@ -0,0 +1 @@ +Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5 diff --git a/ext/kenlm/jam-files/engine/md5.c b/ext/kenlm/jam-files/engine/md5.c deleted file mode 100644 index c35d96c..0000000 --- a/ext/kenlm/jam-files/engine/md5.c +++ /dev/null @@ -1,381 +0,0 @@ -/* - Copyright (C) 1999, 2000, 2002 Aladdin Enterprises. All rights reserved. - - This software is provided 'as-is', without any express or implied - warranty. In no event will the authors be held liable for any damages - arising from the use of this software. - - Permission is granted to anyone to use this software for any purpose, - including commercial applications, and to alter it and redistribute it - freely, subject to the following restrictions: - - 1. The origin of this software must not be misrepresented; you must not - claim that you wrote the original software. If you use this software - in a product, an acknowledgment in the product documentation would be - appreciated but is not required. - 2. Altered source versions must be plainly marked as such, and must not be - misrepresented as being the original software. - 3. This notice may not be removed or altered from any source distribution. - - L. Peter Deutsch - [email protected] - - */ -/* $Id: md5.c,v 1.6 2002/04/13 19:20:28 lpd Exp $ */ -/* - Independent implementation of MD5 (RFC 1321). - - This code implements the MD5 Algorithm defined in RFC 1321, whose - text is available at - http://www.ietf.org/rfc/rfc1321.txt - The code is derived from the text of the RFC, including the test suite - (section A.5) but excluding the rest of Appendix A. It does not include - any code or documentation that is identified in the RFC as being - copyrighted. - - The original and principal author of md5.c is L. Peter Deutsch - <[email protected]>. Other authors are noted in the change history - that follows (in reverse chronological order): - - 2002-04-13 lpd Clarified derivation from RFC 1321; now handles byte order - either statically or dynamically; added missing #include <string.h> - in library. - 2002-03-11 lpd Corrected argument list for main(), and added int return - type, in test program and T value program. - 2002-02-21 lpd Added missing #include <stdio.h> in test program. - 2000-07-03 lpd Patched to eliminate warnings about "constant is - unsigned in ANSI C, signed in traditional"; made test program - self-checking. - 1999-11-04 lpd Edited comments slightly for automatic TOC extraction. - 1999-10-18 lpd Fixed typo in header comment (ansi2knr rather than md5). - 1999-05-03 lpd Original version. - */ - -#include "md5.h" -#include <string.h> - -#undef BYTE_ORDER /* 1 = big-endian, -1 = little-endian, 0 = unknown */ -#ifdef ARCH_IS_BIG_ENDIAN -# define BYTE_ORDER (ARCH_IS_BIG_ENDIAN ? 1 : -1) -#else -# define BYTE_ORDER 0 -#endif - -#define T_MASK ((md5_word_t)~0) -#define T1 /* 0xd76aa478 */ (T_MASK ^ 0x28955b87) -#define T2 /* 0xe8c7b756 */ (T_MASK ^ 0x173848a9) -#define T3 0x242070db -#define T4 /* 0xc1bdceee */ (T_MASK ^ 0x3e423111) -#define T5 /* 0xf57c0faf */ (T_MASK ^ 0x0a83f050) -#define T6 0x4787c62a -#define T7 /* 0xa8304613 */ (T_MASK ^ 0x57cfb9ec) -#define T8 /* 0xfd469501 */ (T_MASK ^ 0x02b96afe) -#define T9 0x698098d8 -#define T10 /* 0x8b44f7af */ (T_MASK ^ 0x74bb0850) -#define T11 /* 0xffff5bb1 */ (T_MASK ^ 0x0000a44e) -#define T12 /* 0x895cd7be */ (T_MASK ^ 0x76a32841) -#define T13 0x6b901122 -#define T14 /* 0xfd987193 */ (T_MASK ^ 0x02678e6c) -#define T15 /* 0xa679438e */ (T_MASK ^ 0x5986bc71) -#define T16 0x49b40821 -#define T17 /* 0xf61e2562 */ (T_MASK ^ 0x09e1da9d) -#define T18 /* 0xc040b340 */ (T_MASK ^ 0x3fbf4cbf) -#define T19 0x265e5a51 -#define T20 /* 0xe9b6c7aa */ (T_MASK ^ 0x16493855) -#define T21 /* 0xd62f105d */ (T_MASK ^ 0x29d0efa2) -#define T22 0x02441453 -#define T23 /* 0xd8a1e681 */ (T_MASK ^ 0x275e197e) -#define T24 /* 0xe7d3fbc8 */ (T_MASK ^ 0x182c0437) -#define T25 0x21e1cde6 -#define T26 /* 0xc33707d6 */ (T_MASK ^ 0x3cc8f829) -#define T27 /* 0xf4d50d87 */ (T_MASK ^ 0x0b2af278) -#define T28 0x455a14ed -#define T29 /* 0xa9e3e905 */ (T_MASK ^ 0x561c16fa) -#define T30 /* 0xfcefa3f8 */ (T_MASK ^ 0x03105c07) -#define T31 0x676f02d9 -#define T32 /* 0x8d2a4c8a */ (T_MASK ^ 0x72d5b375) -#define T33 /* 0xfffa3942 */ (T_MASK ^ 0x0005c6bd) -#define T34 /* 0x8771f681 */ (T_MASK ^ 0x788e097e) -#define T35 0x6d9d6122 -#define T36 /* 0xfde5380c */ (T_MASK ^ 0x021ac7f3) -#define T37 /* 0xa4beea44 */ (T_MASK ^ 0x5b4115bb) -#define T38 0x4bdecfa9 -#define T39 /* 0xf6bb4b60 */ (T_MASK ^ 0x0944b49f) -#define T40 /* 0xbebfbc70 */ (T_MASK ^ 0x4140438f) -#define T41 0x289b7ec6 -#define T42 /* 0xeaa127fa */ (T_MASK ^ 0x155ed805) -#define T43 /* 0xd4ef3085 */ (T_MASK ^ 0x2b10cf7a) -#define T44 0x04881d05 -#define T45 /* 0xd9d4d039 */ (T_MASK ^ 0x262b2fc6) -#define T46 /* 0xe6db99e5 */ (T_MASK ^ 0x1924661a) -#define T47 0x1fa27cf8 -#define T48 /* 0xc4ac5665 */ (T_MASK ^ 0x3b53a99a) -#define T49 /* 0xf4292244 */ (T_MASK ^ 0x0bd6ddbb) -#define T50 0x432aff97 -#define T51 /* 0xab9423a7 */ (T_MASK ^ 0x546bdc58) -#define T52 /* 0xfc93a039 */ (T_MASK ^ 0x036c5fc6) -#define T53 0x655b59c3 -#define T54 /* 0x8f0ccc92 */ (T_MASK ^ 0x70f3336d) -#define T55 /* 0xffeff47d */ (T_MASK ^ 0x00100b82) -#define T56 /* 0x85845dd1 */ (T_MASK ^ 0x7a7ba22e) -#define T57 0x6fa87e4f -#define T58 /* 0xfe2ce6e0 */ (T_MASK ^ 0x01d3191f) -#define T59 /* 0xa3014314 */ (T_MASK ^ 0x5cfebceb) -#define T60 0x4e0811a1 -#define T61 /* 0xf7537e82 */ (T_MASK ^ 0x08ac817d) -#define T62 /* 0xbd3af235 */ (T_MASK ^ 0x42c50dca) -#define T63 0x2ad7d2bb -#define T64 /* 0xeb86d391 */ (T_MASK ^ 0x14792c6e) - - -static void -md5_process(md5_state_t *pms, const md5_byte_t *data /*[64]*/) -{ - md5_word_t - a = pms->abcd[0], b = pms->abcd[1], - c = pms->abcd[2], d = pms->abcd[3]; - md5_word_t t; -#if BYTE_ORDER > 0 - /* Define storage only for big-endian CPUs. */ - md5_word_t X[16]; -#else - /* Define storage for little-endian or both types of CPUs. */ - md5_word_t xbuf[16]; - const md5_word_t *X; -#endif - - { -#if BYTE_ORDER == 0 - /* - * Determine dynamically whether this is a big-endian or - * little-endian machine, since we can use a more efficient - * algorithm on the latter. - */ - static const int w = 1; - - if (*((const md5_byte_t *)&w)) /* dynamic little-endian */ -#endif -#if BYTE_ORDER <= 0 /* little-endian */ - { - /* - * On little-endian machines, we can process properly aligned - * data without copying it. - */ - if (!((data - (const md5_byte_t *)0) & 3)) { - /* data are properly aligned */ - X = (const md5_word_t *)data; - } else { - /* not aligned */ - memcpy(xbuf, data, 64); - X = xbuf; - } - } -#endif -#if BYTE_ORDER == 0 - else /* dynamic big-endian */ -#endif -#if BYTE_ORDER >= 0 /* big-endian */ - { - /* - * On big-endian machines, we must arrange the bytes in the - * right order. - */ - const md5_byte_t *xp = data; - int i; - -# if BYTE_ORDER == 0 - X = xbuf; /* (dynamic only) */ -# else -# define xbuf X /* (static only) */ -# endif - for (i = 0; i < 16; ++i, xp += 4) - xbuf[i] = xp[0] + (xp[1] << 8) + (xp[2] << 16) + (xp[3] << 24); - } -#endif - } - -#define ROTATE_LEFT(x, n) (((x) << (n)) | ((x) >> (32 - (n)))) - - /* Round 1. */ - /* Let [abcd k s i] denote the operation - a = b + ((a + F(b,c,d) + X[k] + T[i]) <<< s). */ -#define F(x, y, z) (((x) & (y)) | (~(x) & (z))) -#define SET(a, b, c, d, k, s, Ti)\ - t = a + F(b,c,d) + X[k] + Ti;\ - a = ROTATE_LEFT(t, s) + b - /* Do the following 16 operations. */ - SET(a, b, c, d, 0, 7, T1); - SET(d, a, b, c, 1, 12, T2); - SET(c, d, a, b, 2, 17, T3); - SET(b, c, d, a, 3, 22, T4); - SET(a, b, c, d, 4, 7, T5); - SET(d, a, b, c, 5, 12, T6); - SET(c, d, a, b, 6, 17, T7); - SET(b, c, d, a, 7, 22, T8); - SET(a, b, c, d, 8, 7, T9); - SET(d, a, b, c, 9, 12, T10); - SET(c, d, a, b, 10, 17, T11); - SET(b, c, d, a, 11, 22, T12); - SET(a, b, c, d, 12, 7, T13); - SET(d, a, b, c, 13, 12, T14); - SET(c, d, a, b, 14, 17, T15); - SET(b, c, d, a, 15, 22, T16); -#undef SET - - /* Round 2. */ - /* Let [abcd k s i] denote the operation - a = b + ((a + G(b,c,d) + X[k] + T[i]) <<< s). */ -#define G(x, y, z) (((x) & (z)) | ((y) & ~(z))) -#define SET(a, b, c, d, k, s, Ti)\ - t = a + G(b,c,d) + X[k] + Ti;\ - a = ROTATE_LEFT(t, s) + b - /* Do the following 16 operations. */ - SET(a, b, c, d, 1, 5, T17); - SET(d, a, b, c, 6, 9, T18); - SET(c, d, a, b, 11, 14, T19); - SET(b, c, d, a, 0, 20, T20); - SET(a, b, c, d, 5, 5, T21); - SET(d, a, b, c, 10, 9, T22); - SET(c, d, a, b, 15, 14, T23); - SET(b, c, d, a, 4, 20, T24); - SET(a, b, c, d, 9, 5, T25); - SET(d, a, b, c, 14, 9, T26); - SET(c, d, a, b, 3, 14, T27); - SET(b, c, d, a, 8, 20, T28); - SET(a, b, c, d, 13, 5, T29); - SET(d, a, b, c, 2, 9, T30); - SET(c, d, a, b, 7, 14, T31); - SET(b, c, d, a, 12, 20, T32); -#undef SET - - /* Round 3. */ - /* Let [abcd k s t] denote the operation - a = b + ((a + H(b,c,d) + X[k] + T[i]) <<< s). */ -#define H(x, y, z) ((x) ^ (y) ^ (z)) -#define SET(a, b, c, d, k, s, Ti)\ - t = a + H(b,c,d) + X[k] + Ti;\ - a = ROTATE_LEFT(t, s) + b - /* Do the following 16 operations. */ - SET(a, b, c, d, 5, 4, T33); - SET(d, a, b, c, 8, 11, T34); - SET(c, d, a, b, 11, 16, T35); - SET(b, c, d, a, 14, 23, T36); - SET(a, b, c, d, 1, 4, T37); - SET(d, a, b, c, 4, 11, T38); - SET(c, d, a, b, 7, 16, T39); - SET(b, c, d, a, 10, 23, T40); - SET(a, b, c, d, 13, 4, T41); - SET(d, a, b, c, 0, 11, T42); - SET(c, d, a, b, 3, 16, T43); - SET(b, c, d, a, 6, 23, T44); - SET(a, b, c, d, 9, 4, T45); - SET(d, a, b, c, 12, 11, T46); - SET(c, d, a, b, 15, 16, T47); - SET(b, c, d, a, 2, 23, T48); -#undef SET - - /* Round 4. */ - /* Let [abcd k s t] denote the operation - a = b + ((a + I(b,c,d) + X[k] + T[i]) <<< s). */ -#define I(x, y, z) ((y) ^ ((x) | ~(z))) -#define SET(a, b, c, d, k, s, Ti)\ - t = a + I(b,c,d) + X[k] + Ti;\ - a = ROTATE_LEFT(t, s) + b - /* Do the following 16 operations. */ - SET(a, b, c, d, 0, 6, T49); - SET(d, a, b, c, 7, 10, T50); - SET(c, d, a, b, 14, 15, T51); - SET(b, c, d, a, 5, 21, T52); - SET(a, b, c, d, 12, 6, T53); - SET(d, a, b, c, 3, 10, T54); - SET(c, d, a, b, 10, 15, T55); - SET(b, c, d, a, 1, 21, T56); - SET(a, b, c, d, 8, 6, T57); - SET(d, a, b, c, 15, 10, T58); - SET(c, d, a, b, 6, 15, T59); - SET(b, c, d, a, 13, 21, T60); - SET(a, b, c, d, 4, 6, T61); - SET(d, a, b, c, 11, 10, T62); - SET(c, d, a, b, 2, 15, T63); - SET(b, c, d, a, 9, 21, T64); -#undef SET - - /* Then perform the following additions. (That is increment each - of the four registers by the value it had before this block - was started.) */ - pms->abcd[0] += a; - pms->abcd[1] += b; - pms->abcd[2] += c; - pms->abcd[3] += d; -} - -void -md5_init(md5_state_t *pms) -{ - pms->count[0] = pms->count[1] = 0; - pms->abcd[0] = 0x67452301; - pms->abcd[1] = /*0xefcdab89*/ T_MASK ^ 0x10325476; - pms->abcd[2] = /*0x98badcfe*/ T_MASK ^ 0x67452301; - pms->abcd[3] = 0x10325476; -} - -void -md5_append(md5_state_t *pms, const md5_byte_t *data, int nbytes) -{ - const md5_byte_t *p = data; - int left = nbytes; - int offset = (pms->count[0] >> 3) & 63; - md5_word_t nbits = (md5_word_t)(nbytes << 3); - - if (nbytes <= 0) - return; - - /* Update the message length. */ - pms->count[1] += nbytes >> 29; - pms->count[0] += nbits; - if (pms->count[0] < nbits) - pms->count[1]++; - - /* Process an initial partial block. */ - if (offset) { - int copy = (offset + nbytes > 64 ? 64 - offset : nbytes); - - memcpy(pms->buf + offset, p, copy); - if (offset + copy < 64) - return; - p += copy; - left -= copy; - md5_process(pms, pms->buf); - } - - /* Process full blocks. */ - for (; left >= 64; p += 64, left -= 64) - md5_process(pms, p); - - /* Process a final partial block. */ - if (left) - memcpy(pms->buf, p, left); -} - -void -md5_finish(md5_state_t *pms, md5_byte_t digest[16]) -{ - static const md5_byte_t pad[64] = { - 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 - }; - md5_byte_t data[8]; - int i; - - /* Save the length before padding. */ - for (i = 0; i < 8; ++i) - data[i] = (md5_byte_t)(pms->count[i >> 2] >> ((i & 3) << 3)); - /* Pad to 56 bytes mod 64. */ - md5_append(pms, pad, ((55 - (pms->count[0] >> 3)) & 63) + 1); - /* Append the length. */ - md5_append(pms, data, 8); - for (i = 0; i < 16; ++i) - digest[i] = (md5_byte_t)(pms->abcd[i >> 2] >> ((i & 3) << 3)); -} http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/md5.h ---------------------------------------------------------------------- diff --git a/ext/kenlm b/ext/kenlm new file mode 160000 index 0000000..56fdb5c --- /dev/null +++ b/ext/kenlm @@ -0,0 +1 @@ +Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5 diff --git a/ext/kenlm/jam-files/engine/md5.h b/ext/kenlm/jam-files/engine/md5.h deleted file mode 100644 index 698c995..0000000 --- a/ext/kenlm/jam-files/engine/md5.h +++ /dev/null @@ -1,91 +0,0 @@ -/* - Copyright (C) 1999, 2002 Aladdin Enterprises. All rights reserved. - - This software is provided 'as-is', without any express or implied - warranty. In no event will the authors be held liable for any damages - arising from the use of this software. - - Permission is granted to anyone to use this software for any purpose, - including commercial applications, and to alter it and redistribute it - freely, subject to the following restrictions: - - 1. The origin of this software must not be misrepresented; you must not - claim that you wrote the original software. If you use this software - in a product, an acknowledgment in the product documentation would be - appreciated but is not required. - 2. Altered source versions must be plainly marked as such, and must not be - misrepresented as being the original software. - 3. This notice may not be removed or altered from any source distribution. - - L. Peter Deutsch - [email protected] - - */ -/* $Id: md5.h,v 1.4 2002/04/13 19:20:28 lpd Exp $ */ -/* - Independent implementation of MD5 (RFC 1321). - - This code implements the MD5 Algorithm defined in RFC 1321, whose - text is available at - http://www.ietf.org/rfc/rfc1321.txt - The code is derived from the text of the RFC, including the test suite - (section A.5) but excluding the rest of Appendix A. It does not include - any code or documentation that is identified in the RFC as being - copyrighted. - - The original and principal author of md5.h is L. Peter Deutsch - <[email protected]>. Other authors are noted in the change history - that follows (in reverse chronological order): - - 2002-04-13 lpd Removed support for non-ANSI compilers; removed - references to Ghostscript; clarified derivation from RFC 1321; - now handles byte order either statically or dynamically. - 1999-11-04 lpd Edited comments slightly for automatic TOC extraction. - 1999-10-18 lpd Fixed typo in header comment (ansi2knr rather than md5); - added conditionalization for C++ compilation from Martin - Purschke <[email protected]>. - 1999-05-03 lpd Original version. - */ - -#ifndef md5_INCLUDED -# define md5_INCLUDED - -/* - * This package supports both compile-time and run-time determination of CPU - * byte order. If ARCH_IS_BIG_ENDIAN is defined as 0, the code will be - * compiled to run only on little-endian CPUs; if ARCH_IS_BIG_ENDIAN is - * defined as non-zero, the code will be compiled to run only on big-endian - * CPUs; if ARCH_IS_BIG_ENDIAN is not defined, the code will be compiled to - * run on either big- or little-endian CPUs, but will run slightly less - * efficiently on either one than if ARCH_IS_BIG_ENDIAN is defined. - */ - -typedef unsigned char md5_byte_t; /* 8-bit byte */ -typedef unsigned int md5_word_t; /* 32-bit word */ - -/* Define the state of the MD5 Algorithm. */ -typedef struct md5_state_s { - md5_word_t count[2]; /* message length in bits, lsw first */ - md5_word_t abcd[4]; /* digest buffer */ - md5_byte_t buf[64]; /* accumulate block */ -} md5_state_t; - -#ifdef __cplusplus -extern "C" -{ -#endif - -/* Initialize the algorithm. */ -void md5_init(md5_state_t *pms); - -/* Append a string to the message. */ -void md5_append(md5_state_t *pms, const md5_byte_t *data, int nbytes); - -/* Finish the message and return the digest. */ -void md5_finish(md5_state_t *pms, md5_byte_t digest[16]); - -#ifdef __cplusplus -} /* end extern "C" */ -#endif - -#endif /* md5_INCLUDED */ http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/mem.c ---------------------------------------------------------------------- diff --git a/ext/kenlm b/ext/kenlm new file mode 160000 index 0000000..56fdb5c --- /dev/null +++ b/ext/kenlm @@ -0,0 +1 @@ +Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5 diff --git a/ext/kenlm/jam-files/engine/mem.c b/ext/kenlm/jam-files/engine/mem.c deleted file mode 100644 index 6a11fb3..0000000 --- a/ext/kenlm/jam-files/engine/mem.c +++ /dev/null @@ -1,75 +0,0 @@ -/* -Copyright Rene Rivera 2006. -Distributed under the Boost Software License, Version 1.0. -(See accompanying file LICENSE_1_0.txt or copy at -http://www.boost.org/LICENSE_1_0.txt) -*/ - -#include "jam.h" - -#ifdef OPT_BOEHM_GC - - /* Compile the Boehm GC as one big chunk of code. It's much easier - this way, than trying to make radical changes to the bjam build - scripts. */ - - #define ATOMIC_UNCOLLECTABLE - #define NO_EXECUTE_PERMISSION - #define ALL_INTERIOR_POINTERS - - #define LARGE_CONFIG - /* - #define NO_SIGNALS - #define SILENT - */ - #ifndef GC_DEBUG - #define NO_DEBUGGING - #endif - - #ifdef __GLIBC__ - #define __USE_GNU - #endif - - #include "boehm_gc/reclaim.c" - #include "boehm_gc/allchblk.c" - #include "boehm_gc/misc.c" - #include "boehm_gc/alloc.c" - #include "boehm_gc/mach_dep.c" - #include "boehm_gc/os_dep.c" - #include "boehm_gc/mark_rts.c" - #include "boehm_gc/headers.c" - #include "boehm_gc/mark.c" - #include "boehm_gc/obj_map.c" - #include "boehm_gc/pcr_interface.c" - #include "boehm_gc/blacklst.c" - #include "boehm_gc/new_hblk.c" - #include "boehm_gc/real_malloc.c" - #include "boehm_gc/dyn_load.c" - #include "boehm_gc/dbg_mlc.c" - #include "boehm_gc/malloc.c" - #include "boehm_gc/stubborn.c" - #include "boehm_gc/checksums.c" - #include "boehm_gc/pthread_support.c" - #include "boehm_gc/pthread_stop_world.c" - #include "boehm_gc/darwin_stop_world.c" - #include "boehm_gc/typd_mlc.c" - #include "boehm_gc/ptr_chck.c" - #include "boehm_gc/mallocx.c" - #include "boehm_gc/gcj_mlc.c" - #include "boehm_gc/specific.c" - #include "boehm_gc/gc_dlopen.c" - #include "boehm_gc/backgraph.c" - #include "boehm_gc/win32_threads.c" - - /* Needs to be last. */ - #include "boehm_gc/finalize.c" - -#elif defined(OPT_DUMA) - - #ifdef OS_NT - #define WIN32 - #endif - #include "duma/duma.c" - #include "duma/print.c" - -#endif http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/mem.h ---------------------------------------------------------------------- diff --git a/ext/kenlm b/ext/kenlm new file mode 160000 index 0000000..56fdb5c --- /dev/null +++ b/ext/kenlm @@ -0,0 +1 @@ +Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5 diff --git a/ext/kenlm/jam-files/engine/mem.h b/ext/kenlm/jam-files/engine/mem.h deleted file mode 100644 index 8718b07..0000000 --- a/ext/kenlm/jam-files/engine/mem.h +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Copyright 2006. Rene Rivera - * Distributed under the Boost Software License, Version 1.0. - * (See accompanying file LICENSE_1_0.txt or copy at - * http://www.boost.org/LICENSE_1_0.txt) - */ - -#ifndef BJAM_MEM_H -#define BJAM_MEM_H - -#ifdef OPT_BOEHM_GC - - /* Use Boehm GC memory allocator. */ - #include <gc.h> - - #define bjam_malloc_x(s) memset(GC_malloc(s),0,s) - #define bjam_malloc_atomic_x(s) memset(GC_malloc_atomic(s),0,s) - #define bjam_calloc_x(n,s) memset(GC_malloc((n)*(s)),0,(n)*(s)) - #define bjam_calloc_atomic_x(n,s) memset(GC_malloc_atomic((n)*(s)),0,(n)*(s)) - #define bjam_realloc_x(p,s) GC_realloc(p,s) - #define bjam_free_x(p) GC_free(p) - #define bjam_mem_init_x() GC_init(); GC_enable_incremental() - - #define bjam_malloc_raw_x(s) malloc(s) - #define bjam_calloc_raw_x(n,s) calloc(n,s) - #define bjam_realloc_raw_x(p,s) realloc(p,s) - #define bjam_free_raw_x(p) free(p) - - #ifndef BJAM_NEWSTR_NO_ALLOCATE - # define BJAM_NEWSTR_NO_ALLOCATE - #endif - -#elif defined( OPT_DUMA ) - - /* Use Duma memory debugging library. */ - #include <stdlib.h> - - #define _DUMA_CONFIG_H_ - #define DUMA_NO_GLOBAL_MALLOC_FREE - #define DUMA_EXPLICIT_INIT - #define DUMA_NO_THREAD_SAFETY - #define DUMA_NO_CPP_SUPPORT - /* #define DUMA_NO_LEAKDETECTION */ - /* #define DUMA_USE_FRAMENO */ - /* #define DUMA_PREFER_ATEXIT */ - /* #define DUMA_OLD_DEL_MACRO */ - /* #define DUMA_NO_HANG_MSG */ - #define DUMA_PAGE_SIZE 4096 - #define DUMA_MIN_ALIGNMENT 1 - /* #define DUMA_GNU_INIT_ATTR 0 */ - typedef unsigned int DUMA_ADDR; - typedef unsigned int DUMA_SIZE; - #include <duma.h> - - #define bjam_malloc_x(s) malloc(s) - #define bjam_calloc_x(n,s) calloc(n,s) - #define bjam_realloc_x(p,s) realloc(p,s) - #define bjam_free_x(p) free(p) - - #ifndef BJAM_NEWSTR_NO_ALLOCATE - # define BJAM_NEWSTR_NO_ALLOCATE - #endif - -#else - - /* Standard C memory allocation. */ - #include <stdlib.h> - - #define bjam_malloc_x(s) malloc(s) - #define bjam_calloc_x(n,s) calloc(n,s) - #define bjam_realloc_x(p,s) realloc(p,s) - #define bjam_free_x(p) free(p) - -#endif - -#ifndef bjam_malloc_atomic_x - #define bjam_malloc_atomic_x(s) bjam_malloc_x(s) -#endif -#ifndef bjam_calloc_atomic_x - #define bjam_calloc_atomic_x(n,s) bjam_calloc_x(n,s) -#endif -#ifndef bjam_mem_init_x - #define bjam_mem_init_x() -#endif -#ifndef bjam_mem_close_x - #define bjam_mem_close_x() -#endif -#ifndef bjam_malloc_raw_x - #define bjam_malloc_raw_x(s) bjam_malloc_x(s) -#endif -#ifndef bjam_calloc_raw_x - #define bjam_calloc_raw_x(n,s) bjam_calloc_x(n,s) -#endif -#ifndef bjam_realloc_raw_x - #define bjam_realloc_raw_x(p,s) bjam_realloc_x(p,s) -#endif -#ifndef bjam_free_raw_x - #define bjam_free_raw_x(p) bjam_free_x(p) -#endif - -#ifdef OPT_DEBUG_PROFILE - /* Profile tracing of memory allocations. */ - #include "debug.h" - - #define BJAM_MALLOC(s) (profile_memory(s), bjam_malloc_x(s)) - #define BJAM_MALLOC_ATOMIC(s) (profile_memory(s), bjam_malloc_atomic_x(s)) - #define BJAM_CALLOC(n,s) (profile_memory(n*s), bjam_calloc_x(n,s)) - #define BJAM_CALLOC_ATOMIC(n,s) (profile_memory(n*s), bjam_calloc_atomic_x(n,s)) - #define BJAM_REALLOC(p,s) (profile_memory(s), bjam_realloc_x(p,s)) - - #define BJAM_MALLOC_RAW(s) (profile_memory(s), bjam_malloc_raw_x(s)) - #define BJAM_CALLOC_RAW(n,s) (profile_memory(n*s), bjam_calloc_raw_x(n,s)) - #define BJAM_REALLOC_RAW(p,s) (profile_memory(s), bjam_realloc_raw_x(p,s)) -#else - /* No mem tracing. */ - #define BJAM_MALLOC(s) bjam_malloc_x(s) - #define BJAM_MALLOC_ATOMIC(s) bjam_malloc_atomic_x(s) - #define BJAM_CALLOC(n,s) bjam_calloc_x(n,s) - #define BJAM_CALLOC_ATOMIC(n,s) bjam_calloc_atomic_x(n,s) - #define BJAM_REALLOC(p,s) bjam_realloc_x(p,s) - - #define BJAM_MALLOC_RAW(s) bjam_malloc_raw_x(s) - #define BJAM_CALLOC_RAW(n,s) bjam_calloc_raw_x(n,s) - #define BJAM_REALLOC_RAW(p,s) bjam_realloc_raw_x(p,s) -#endif - -#define BJAM_MEM_INIT() bjam_mem_init_x() -#define BJAM_MEM_CLOSE() bjam_mem_close_x() - -#define BJAM_FREE(p) bjam_free_x(p) -#define BJAM_FREE_RAW(p) bjam_free_raw_x(p) - -#endif http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/mkjambase.c ---------------------------------------------------------------------- diff --git a/ext/kenlm b/ext/kenlm new file mode 160000 index 0000000..56fdb5c --- /dev/null +++ b/ext/kenlm @@ -0,0 +1 @@ +Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5 diff --git a/ext/kenlm/jam-files/engine/mkjambase.c b/ext/kenlm/jam-files/engine/mkjambase.c deleted file mode 100644 index cdf5998..0000000 --- a/ext/kenlm/jam-files/engine/mkjambase.c +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Copyright 1993, 1995 Christopher Seiwald. - * - * This file is part of Jam - see jam.c for Copyright information. - */ - -/* This file is ALSO: - * Copyright 2001-2004 David Abrahams. - * Distributed under the Boost Software License, Version 1.0. - * (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt) - */ - -/* - * mkjambase.c - turn Jambase into a big C structure - * - * Usage: mkjambase jambase.c Jambase ... - * - * Results look like this: - * - * char *jambase[] = { - * "...\n", - * ... - * 0 }; - * - * Handles \'s and "'s specially; knows to delete blank and comment lines. - * - */ - -#include <stdio.h> -#include <string.h> - - -int main( int argc, char * * argv, char * * envp ) -{ - char buf[ 1024 ]; - FILE * fin; - FILE * fout; - char * p; - int doDotC = 0; - - if ( argc < 3 ) - { - fprintf( stderr, "usage: %s jambase.c Jambase ...\n", argv[ 0 ] ); - return -1; - } - - if ( !( fout = fopen( argv[1], "w" ) ) ) - { - perror( argv[ 1 ] ); - return -1; - } - - /* If the file ends in .c generate a C source file. */ - if ( ( p = strrchr( argv[1], '.' ) ) && !strcmp( p, ".c" ) ) - doDotC++; - - /* Now process the files. */ - - argc -= 2; - argv += 2; - - if ( doDotC ) - { - fprintf( fout, "/* Generated by mkjambase from Jambase */\n" ); - fprintf( fout, "char *jambase[] = {\n" ); - } - - for ( ; argc--; ++argv ) - { - if ( !( fin = fopen( *argv, "r" ) ) ) - { - perror( *argv ); - return -1; - } - - if ( doDotC ) - fprintf( fout, "/* %s */\n", *argv ); - else - fprintf( fout, "### %s ###\n", *argv ); - - while ( fgets( buf, sizeof( buf ), fin ) ) - { - if ( doDotC ) - { - char * p = buf; - - /* Strip leading whitespace. */ - while ( ( *p == ' ' ) || ( *p == '\t' ) || ( *p == '\n' ) ) - ++p; - - /* Drop comments and empty lines. */ - if ( ( *p == '#' ) || !*p ) - continue; - - /* Copy. */ - putc( '"', fout ); - for ( ; *p && ( *p != '\n' ); ++p ) - switch ( *p ) - { - case '\\': putc( '\\', fout ); putc( '\\', fout ); break; - case '"' : putc( '\\', fout ); putc( '"' , fout ); break; - case '\r': break; - default: putc( *p, fout ); break; - } - - fprintf( fout, "\\n\",\n" ); - } - else - { - fprintf( fout, "%s", buf ); - } - } - - fclose( fin ); - } - - if ( doDotC ) - fprintf( fout, "0 };\n" ); - - fclose( fout ); - - return 0; -} http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/modules.c ---------------------------------------------------------------------- diff --git a/ext/kenlm b/ext/kenlm new file mode 160000 index 0000000..56fdb5c --- /dev/null +++ b/ext/kenlm @@ -0,0 +1 @@ +Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5 diff --git a/ext/kenlm/jam-files/engine/modules.c b/ext/kenlm/jam-files/engine/modules.c deleted file mode 100644 index 6be82fe..0000000 --- a/ext/kenlm/jam-files/engine/modules.c +++ /dev/null @@ -1,431 +0,0 @@ -/* - * Copyright 2001-2004 David Abrahams. - * Distributed under the Boost Software License, Version 1.0. - * (See accompanying file LICENSE_1_0.txt or copy at - * http://www.boost.org/LICENSE_1_0.txt) - */ - -#include "jam.h" -#include "modules.h" - -#include "hash.h" -#include "lists.h" -#include "native.h" -#include "object.h" -#include "parse.h" -#include "rules.h" -#include "strings.h" -#include "variable.h" - -#include <assert.h> -#include <string.h> - -static struct hash * module_hash = 0; -static module_t root; - - -module_t * bindmodule( OBJECT * name ) -{ - if ( !name ) - return &root; - - { - PROFILE_ENTER( BINDMODULE ); - - module_t * m; - int found; - - if ( !module_hash ) - module_hash = hashinit( sizeof( module_t ), "modules" ); - - m = (module_t *)hash_insert( module_hash, name, &found ); - if ( !found ) - { - m->name = object_copy( name ); - m->variables = 0; - m->variable_indices = 0; - m->num_fixed_variables = 0; - m->fixed_variables = 0; - m->rules = 0; - m->imported_modules = 0; - m->class_module = 0; - m->native_rules = 0; - m->user_module = 0; - } - - PROFILE_EXIT( BINDMODULE ); - - return m; - } -} - - -/* - * demand_rules() - Get the module's "rules" hash on demand. - */ -struct hash * demand_rules( module_t * m ) -{ - if ( !m->rules ) - m->rules = hashinit( sizeof( RULE ), "rules" ); - return m->rules; -} - - -/* - * delete_module() - wipe out the module's rules and variables. - */ - -static void delete_rule_( void * xrule, void * data ) -{ - rule_free( (RULE *)xrule ); -} - - -static void delete_native_rule( void * xrule, void * data ) -{ - native_rule_t * rule = (native_rule_t *)xrule; - object_free( rule->name ); - if ( rule->procedure ) - function_free( rule->procedure ); -} - - -static void delete_imported_modules( void * xmodule_name, void * data ) -{ - object_free( *(OBJECT * *)xmodule_name ); -} - - -static void free_fixed_variable( void * xvar, void * data ); - -void delete_module( module_t * m ) -{ - /* Clear out all the rules. */ - if ( m->rules ) - { - hashenumerate( m->rules, delete_rule_, (void *)0 ); - hash_free( m->rules ); - m->rules = 0; - } - - if ( m->native_rules ) - { - hashenumerate( m->native_rules, delete_native_rule, (void *)0 ); - hash_free( m->native_rules ); - m->native_rules = 0; - } - - if ( m->variables ) - { - var_done( m ); - m->variables = 0; - } - - if ( m->fixed_variables ) - { - int i; - for ( i = 0; i < m->num_fixed_variables; ++i ) - { - list_free( m->fixed_variables[ i ] ); - } - BJAM_FREE( m->fixed_variables ); - m->fixed_variables = 0; - } - - if ( m->variable_indices ) - { - hashenumerate( m->variable_indices, &free_fixed_variable, (void *)0 ); - hash_free( m->variable_indices ); - m->variable_indices = 0; - } - - if ( m->imported_modules ) - { - hashenumerate( m->imported_modules, delete_imported_modules, (void *)0 ); - hash_free( m->imported_modules ); - m->imported_modules = 0; - } -} - - -struct module_stats -{ - OBJECT * module_name; - struct hashstats rules_stats[ 1 ]; - struct hashstats variables_stats[ 1 ]; - struct hashstats variable_indices_stats[ 1 ]; - struct hashstats imported_modules_stats[ 1 ]; -}; - - -static void module_stat( struct hash * hp, OBJECT * module, const char * name ) -{ - if ( hp ) - { - struct hashstats stats[ 1 ]; - string id[ 1 ]; - hashstats_init( stats ); - string_new( id ); - string_append( id, object_str( module ) ); - string_push_back( id, ' ' ); - string_append( id, name ); - - hashstats_add( stats, hp ); - hashstats_print( stats, id->value ); - - string_free( id ); - } -} - - -static void class_module_stat( struct hashstats * stats, OBJECT * module, const char * name ) -{ - if ( stats->item_size ) - { - string id[ 1 ]; - string_new( id ); - string_append( id, object_str( module ) ); - string_append( id, " object " ); - string_append( id, name ); - - hashstats_print( stats, id->value ); - - string_free( id ); - } -} - - -static void stat_module( void * xmodule, void * data ) -{ - module_t *m = (module_t *)xmodule; - - if ( DEBUG_MEM || DEBUG_PROFILE ) - { - struct hash * class_info = (struct hash *)data; - if ( m->class_module ) - { - int found; - struct module_stats * ms = (struct module_stats *)hash_insert( class_info, m->class_module->name, &found ); - if ( !found ) - { - ms->module_name = m->class_module->name; - hashstats_init( ms->rules_stats ); - hashstats_init( ms->variables_stats ); - hashstats_init( ms->variable_indices_stats ); - hashstats_init( ms->imported_modules_stats ); - } - - hashstats_add( ms->rules_stats, m->rules ); - hashstats_add( ms->variables_stats, m->variables ); - hashstats_add( ms->variable_indices_stats, m->variable_indices ); - hashstats_add( ms->imported_modules_stats, m->imported_modules ); - } - else - { - module_stat( m->rules, m->name, "rules" ); - module_stat( m->variables, m->name, "variables" ); - module_stat( m->variable_indices, m->name, "fixed variables" ); - module_stat( m->imported_modules, m->name, "imported modules" ); - } - } - - delete_module( m ); - object_free( m->name ); -} - -static void print_class_stats( void * xstats, void * data ) -{ - struct module_stats * stats = (struct module_stats *)xstats; - class_module_stat( stats->rules_stats, stats->module_name, "rules" ); - class_module_stat( stats->variables_stats, stats->module_name, "variables" ); - class_module_stat( stats->variable_indices_stats, stats->module_name, "fixed variables" ); - class_module_stat( stats->imported_modules_stats, stats->module_name, "imported modules" ); -} - - -static void delete_module_( void * xmodule, void * data ) -{ - module_t *m = (module_t *)xmodule; - - delete_module( m ); - object_free( m->name ); -} - - -void modules_done() -{ - if ( DEBUG_MEM || DEBUG_PROFILE ) - { - struct hash * class_hash = hashinit( sizeof( struct module_stats ), "object info" ); - hashenumerate( module_hash, stat_module, (void *)class_hash ); - hashenumerate( class_hash, print_class_stats, (void *)0 ); - hash_free( class_hash ); - } - hashenumerate( module_hash, delete_module_, (void *)0 ); - hashdone( module_hash ); - module_hash = 0; - delete_module( &root ); -} - -module_t * root_module() -{ - return &root; -} - - -void import_module( LIST * module_names, module_t * target_module ) -{ - PROFILE_ENTER( IMPORT_MODULE ); - - struct hash * h; - LISTITER iter; - LISTITER end; - - if ( !target_module->imported_modules ) - target_module->imported_modules = hashinit( sizeof( char * ), "imported" - ); - h = target_module->imported_modules; - - iter = list_begin( module_names ); - end = list_end( module_names ); - for ( ; iter != end; iter = list_next( iter ) ) - { - int found; - OBJECT * const s = list_item( iter ); - OBJECT * * const ss = (OBJECT * *)hash_insert( h, s, &found ); - if ( !found ) - *ss = object_copy( s ); - } - - PROFILE_EXIT( IMPORT_MODULE ); -} - - -static void add_module_name( void * r_, void * result_ ) -{ - OBJECT * * const r = (OBJECT * *)r_; - LIST * * const result = (LIST * *)result_; - *result = list_push_back( *result, object_copy( *r ) ); -} - - -LIST * imported_modules( module_t * module ) -{ - LIST * result = L0; - if ( module->imported_modules ) - hashenumerate( module->imported_modules, add_module_name, &result ); - return result; -} - - -FUNCTION * function_bind_variables( FUNCTION *, module_t *, int * counter ); -FUNCTION * function_unbind_variables( FUNCTION * ); - -struct fixed_variable -{ - OBJECT * key; - int n; -}; - -struct bind_vars_t -{ - module_t * module; - int counter; -}; - - -static void free_fixed_variable( void * xvar, void * data ) -{ - object_free( ( (struct fixed_variable *)xvar )->key ); -} - - -static void bind_variables_for_rule( void * xrule, void * xdata ) -{ - RULE * rule = (RULE *)xrule; - struct bind_vars_t * data = (struct bind_vars_t *)xdata; - if ( rule->procedure && rule->module == data->module ) - rule->procedure = function_bind_variables( rule->procedure, - data->module, &data->counter ); -} - - -void module_bind_variables( struct module_t * m ) -{ - if ( m != root_module() && m->rules ) - { - struct bind_vars_t data; - data.module = m; - data.counter = m->num_fixed_variables; - hashenumerate( m->rules, &bind_variables_for_rule, &data ); - module_set_fixed_variables( m, data.counter ); - } -} - - -int module_add_fixed_var( struct module_t * m, OBJECT * name, int * counter ) -{ - struct fixed_variable * v; - int found; - - assert( !m->class_module ); - - if ( !m->variable_indices ) - m->variable_indices = hashinit( sizeof( struct fixed_variable ), "variable index table" ); - - v = (struct fixed_variable *)hash_insert( m->variable_indices, name, &found ); - if ( !found ) - { - v->key = object_copy( name ); - v->n = (*counter)++; - } - - return v->n; -} - - -LIST * var_get_and_clear_raw( module_t * m, OBJECT * name ); - -static void load_fixed_variable( void * xvar, void * data ) -{ - struct fixed_variable * var = (struct fixed_variable *)xvar; - struct module_t * m = (struct module_t *)data; - if ( var->n >= m->num_fixed_variables ) - m->fixed_variables[ var->n ] = var_get_and_clear_raw( m, var->key ); -} - - -void module_set_fixed_variables( struct module_t * m, int n_variables ) -{ - /* Reallocate */ - struct hash * variable_indices; - LIST * * fixed_variables = BJAM_MALLOC( n_variables * sizeof( LIST * ) ); - if ( m->fixed_variables ) - { - memcpy( fixed_variables, m->fixed_variables, m->num_fixed_variables * sizeof( LIST * ) ); - BJAM_FREE( m->fixed_variables ); - } - m->fixed_variables = fixed_variables; - variable_indices = m->class_module - ? m->class_module->variable_indices - : m->variable_indices; - if ( variable_indices ) - hashenumerate( variable_indices, &load_fixed_variable, m ); - m->num_fixed_variables = n_variables; -} - - -int module_get_fixed_var( struct module_t * m_, OBJECT * name ) -{ - struct fixed_variable * v; - struct module_t * m = m_; - - if ( m->class_module ) - m = m->class_module; - - if ( !m->variable_indices ) - return -1; - - v = (struct fixed_variable *)hash_find( m->variable_indices, name ); - return v && v->n < m_->num_fixed_variables ? v->n : -1; -} http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/modules.h ---------------------------------------------------------------------- diff --git a/ext/kenlm b/ext/kenlm new file mode 160000 index 0000000..56fdb5c --- /dev/null +++ b/ext/kenlm @@ -0,0 +1 @@ +Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5 diff --git a/ext/kenlm/jam-files/engine/modules.h b/ext/kenlm/jam-files/engine/modules.h deleted file mode 100644 index 1b161c6..0000000 --- a/ext/kenlm/jam-files/engine/modules.h +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright 2001-2004 David Abrahams. - * Distributed under the Boost Software License, Version 1.0. - * (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt) - */ -#ifndef MODULES_DWA10182001_H -#define MODULES_DWA10182001_H - -#include "lists.h" -#include "object.h" - -typedef struct module_t module_t ; -struct module_t -{ - OBJECT * name; - struct hash * rules; - struct hash * variables; - struct hash * variable_indices; - int num_fixed_variables; - LIST * * fixed_variables; - struct hash * imported_modules; - module_t * class_module; - struct hash * native_rules; - int user_module; -}; - -module_t * bindmodule( OBJECT * name ); -module_t * root_module(); -void delete_module( module_t * ); - -void import_module( LIST * module_names, module_t * target_module ); -LIST * imported_modules( module_t * ); - -struct hash * demand_rules( module_t * ); - -void module_bind_variables( module_t * ); - -/* - * After calling module_add_fixed_var, module_set_fixed_variables must be called - * before accessing any variables in the module. - */ -int module_add_fixed_var( module_t *, OBJECT * name, int * n ); -void module_set_fixed_variables( module_t *, int n ); - -/* - * Returns the index of the variable or -1 if none exists. - */ -int module_get_fixed_var( module_t *, OBJECT * name ); - -void modules_done(); - -#endif
