http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/execnt.c ---------------------------------------------------------------------- diff --git a/ext/kenlm b/ext/kenlm new file mode 160000 index 0000000..56fdb5c --- /dev/null +++ b/ext/kenlm @@ -0,0 +1 @@ +Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5 diff --git a/ext/kenlm/jam-files/engine/execnt.c b/ext/kenlm/jam-files/engine/execnt.c deleted file mode 100644 index ca89bd0..0000000 --- a/ext/kenlm/jam-files/engine/execnt.c +++ /dev/null @@ -1,1232 +0,0 @@ -/* - * Copyright 1993, 1995 Christopher Seiwald. - * - * This file is part of Jam - see jam.c for Copyright information. - */ - -/* This file is ALSO: - * Copyright 2001-2004 David Abrahams. - * Copyright 2007 Rene Rivera. - * Distributed under the Boost Software License, Version 1.0. - * (See accompanying file LICENSE_1_0.txt or copy at - * http://www.boost.org/LICENSE_1_0.txt) - */ - -/* - * execnt.c - execute a shell command on Windows NT - * - * If $(JAMSHELL) is defined, uses that to formulate the actual command. The - * default is: cmd.exe /Q/C - * - * In $(JAMSHELL), % expands to the command string and ! expands to the slot - * number (starting at 1) for multiprocess (-j) invocations. If $(JAMSHELL) does - * not include a %, it is tacked on as the last argument. - * - * Each $(JAMSHELL) placeholder must be specified as a separate individual - * element in a jam variable value. - * - * Do not just set JAMSHELL to cmd.exe - it will not work! - * - * External routines: - * exec_check() - preprocess and validate the command - * exec_cmd() - launch an async command execution - * exec_wait() - wait for any of the async command processes to terminate - * - * Internal routines: - * filetime_to_seconds() - Windows FILETIME --> number of seconds conversion - */ - -#include "jam.h" -#ifdef USE_EXECNT -#include "execcmd.h" - -#include "lists.h" -#include "output.h" -#include "pathsys.h" -#include "string.h" - -#include <assert.h> -#include <ctype.h> -#include <errno.h> -#include <time.h> - -#define WIN32_LEAN_AND_MEAN -#include <windows.h> -#include <process.h> -#include <tlhelp32.h> - - -/* get the maximum shell command line length according to the OS */ -static int maxline(); -/* valid raw command string length */ -static long raw_command_length( char const * command ); -/* add two 64-bit unsigned numbers, h1l1 and h2l2 */ -static FILETIME add_64( - unsigned long h1, unsigned long l1, - unsigned long h2, unsigned long l2 ); -/* */ -static FILETIME add_FILETIME( FILETIME t1, FILETIME t2 ); -/* */ -static FILETIME negate_FILETIME( FILETIME t ); -/* record the timing info for the process */ -static void record_times( HANDLE const, timing_info * const ); -/* calc the current running time of an *active* process */ -static double running_time( HANDLE const ); -/* terminate the given process, after terminating all its children first */ -static void kill_process_tree( DWORD const procesdId, HANDLE const ); -/* waits for a command to complete or time out */ -static int try_wait( int const timeoutMillis ); -/* reads any pending output for running commands */ -static void read_output(); -/* checks if a command ran out of time, and kills it */ -static int try_kill_one(); -/* is the first process a parent (direct or indirect) to the second one */ -static int is_parent_child( DWORD const parent, DWORD const child ); -/* */ -static void close_alert( PROCESS_INFORMATION const * const ); -/* close any alerts hanging around */ -static void close_alerts(); -/* prepare a command file to be executed using an external shell */ -static char const * prepare_command_file( string const * command, int slot ); -/* invoke the actual external process using the given command line */ -static void invoke_cmd( char const * const command, int const slot ); -/* find a free slot in the running commands table */ -static int get_free_cmdtab_slot(); -/* put together the final command string we are to run */ -static void string_new_from_argv( string * result, char const * const * argv ); -/* frees and renews the given string */ -static void string_renew( string * const ); -/* reports the last failed Windows API related error message */ -static void reportWindowsError( char const * const apiName ); -/* closes a Windows HANDLE and resets its variable to 0. */ -static void closeWinHandle( HANDLE * const handle ); - -/* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ - -/* CreateProcessA() Windows API places a limit of 32768 characters (bytes) on - * the allowed command-line length, including a trailing Unicode (2-byte) - * nul-terminator character. - */ -#define MAX_RAW_COMMAND_LENGTH 32766 - -/* We hold handles for pipes used to communicate with child processes in two - * element arrays indexed as follows. - */ -#define EXECCMD_PIPE_READ 0 -#define EXECCMD_PIPE_WRITE 1 - -static int intr_installed; - - -/* The list of commands we run. */ -static struct -{ - /* Temporary command file used to execute the action when needed. */ - string command_file[ 1 ]; - - /* Pipes for communicating with the child process. Parent reads from (0), - * child writes to (1). - */ - HANDLE pipe_out[ 2 ]; - HANDLE pipe_err[ 2 ]; - - string buffer_out[ 1 ]; /* buffer to hold stdout, if any */ - string buffer_err[ 1 ]; /* buffer to hold stderr, if any */ - - PROCESS_INFORMATION pi; /* running process information */ - - /* Function called when the command completes. */ - ExecCmdCallback func; - - /* Opaque data passed back to the 'func' callback. */ - void * closure; -} -cmdtab[ MAXJOBS ] = { { 0 } }; - - -/* - * Execution unit tests. - */ - -void execnt_unit_test() -{ -#if !defined( NDEBUG ) - /* vc6 preprocessor is broken, so assert with these strings gets confused. - * Use a table instead. - */ - { - typedef struct test { char * command; int result; } test; - test tests[] = { - { "", 0 }, - { " ", 0 }, - { "x", 1 }, - { "\nx", 1 }, - { "x\n", 1 }, - { "\nx\n", 1 }, - { "\nx \n", 2 }, - { "\nx \n ", 2 }, - { " \n\t\t\v\r\r\n \t x \v \t\t\r\n\n\n \n\n\v\t", 8 }, - { "x\ny", -1 }, - { "x\n\n y", -1 }, - { "echo x > foo.bar", -1 }, - { "echo x < foo.bar", -1 }, - { "echo x | foo.bar", -1 }, - { "echo x \">\" foo.bar", 18 }, - { "echo x '<' foo.bar", 18 }, - { "echo x \"|\" foo.bar", 18 }, - { "echo x \\\">\\\" foo.bar", -1 }, - { "echo x \\\"<\\\" foo.bar", -1 }, - { "echo x \\\"|\\\" foo.bar", -1 }, - { "\"echo x > foo.bar\"", 18 }, - { "echo x \"'\"<' foo.bar", -1 }, - { "echo x \\\\\"<\\\\\" foo.bar", 22 }, - { "echo x \\x\\\"<\\\\\" foo.bar", -1 }, - { 0 } }; - test const * t; - for ( t = tests; t->command; ++t ) - assert( raw_command_length( t->command ) == t->result ); - } - - { - int const length = maxline() + 9; - char * const cmd = (char *)BJAM_MALLOC_ATOMIC( length + 1 ); - memset( cmd, 'x', length ); - cmd[ length ] = 0; - assert( raw_command_length( cmd ) == length ); - BJAM_FREE( cmd ); - } -#endif -} - - -/* - * exec_check() - preprocess and validate the command - */ - -int exec_check -( - string const * command, - LIST * * pShell, - int * error_length, - int * error_max_length -) -{ - /* Default shell does nothing when triggered with an empty or a - * whitespace-only command so we simply skip running it in that case. We - * still pass them on to non-default shells as we do not really know what - * they are going to do with such commands. - */ - if ( list_empty( *pShell ) ) - { - char const * s = command->value; - while ( isspace( *s ) ) ++s; - if ( !*s ) - return EXEC_CHECK_NOOP; - } - - /* Check prerequisites for executing raw commands. */ - if ( is_raw_command_request( *pShell ) ) - { - int const raw_cmd_length = raw_command_length( command->value ); - if ( raw_cmd_length < 0 ) - { - /* Invalid characters detected - fallback to default shell. */ - list_free( *pShell ); - *pShell = L0; - } - else if ( raw_cmd_length > MAX_RAW_COMMAND_LENGTH ) - { - *error_length = raw_cmd_length; - *error_max_length = MAX_RAW_COMMAND_LENGTH; - return EXEC_CHECK_TOO_LONG; - } - else - return raw_cmd_length ? EXEC_CHECK_OK : EXEC_CHECK_NOOP; - } - - /* Now we know we are using an external shell. Note that there is no need to - * check for too long command strings when using an external shell since we - * use a command file and assume no one is going to set up a JAMSHELL format - * string longer than a few hundred bytes at most which should be well under - * the total command string limit. Should someone actually construct such a - * JAMSHELL value it will get reported as an 'invalid parameter' - * CreateProcessA() Windows API failure which seems like a good enough - * result for such intentional mischief. - */ - - /* Check for too long command lines. */ - return check_cmd_for_too_long_lines( command->value, maxline(), - error_length, error_max_length ); -} - - -/* - * exec_cmd() - launch an async command execution - * - * We assume exec_check() already verified that the given command can have its - * command string constructed as requested. - */ - -void exec_cmd -( - string const * cmd_orig, - ExecCmdCallback func, - void * closure, - LIST * shell -) -{ - int const slot = get_free_cmdtab_slot(); - int const is_raw_cmd = is_raw_command_request( shell ); - string cmd_local[ 1 ]; - - /* Initialize default shell - anything more than /Q/C is non-portable. */ - static LIST * default_shell; - if ( !default_shell ) - default_shell = list_new( object_new( "cmd.exe /Q/C" ) ); - - /* Specifying no shell means requesting the default shell. */ - if ( list_empty( shell ) ) - shell = default_shell; - - if ( DEBUG_EXECCMD ) - if ( is_raw_cmd ) - printf( "Executing raw command directly\n" ); - else - { - printf( "Executing using a command file and the shell: " ); - list_print( shell ); - printf( "\n" ); - } - - /* If we are running a raw command directly - trim its leading whitespaces - * as well as any trailing all-whitespace lines but keep any trailing - * whitespace in the final/only line containing something other than - * whitespace). - */ - if ( is_raw_cmd ) - { - char const * start = cmd_orig->value; - char const * p = cmd_orig->value + cmd_orig->size; - char const * end = p; - while ( isspace( *start ) ) ++start; - while ( p > start && isspace( p[ -1 ] ) ) - if ( *--p == '\n' ) - end = p; - string_new( cmd_local ); - string_append_range( cmd_local, start, end ); - assert( cmd_local->size == raw_command_length( cmd_orig->value ) ); - } - /* If we are not running a raw command directly, prepare a command file to - * be executed using an external shell and the actual command string using - * that command file. - */ - else - { - char const * const cmd_file = prepare_command_file( cmd_orig, slot ); - char const * argv[ MAXARGC + 1 ]; /* +1 for NULL */ - argv_from_shell( argv, shell, cmd_file, slot ); - string_new_from_argv( cmd_local, argv ); - } - - /* Catch interrupts whenever commands are running. */ - if ( !intr_installed ) - { - intr_installed = 1; - signal( SIGINT, onintr ); - } - - /* Save input data into the selected running commands table slot. */ - cmdtab[ slot ].func = func; - cmdtab[ slot ].closure = closure; - - /* Invoke the actual external process using the constructed command line. */ - invoke_cmd( cmd_local->value, slot ); - - /* Free our local command string copy. */ - string_free( cmd_local ); -} - - -/* - * exec_wait() - wait for any of the async command processes to terminate - * - * Wait and drive at most one execution completion, while processing the I/O for - * all ongoing commands. - */ - -void exec_wait() -{ - int i = -1; - int exit_reason; /* reason why a command completed */ - - /* Wait for a command to complete, while snarfing up any output. */ - while ( 1 ) - { - /* Check for a complete command, briefly. */ - i = try_wait( 500 ); - /* Read in the output of all running commands. */ - read_output(); - /* Close out pending debug style dialogs. */ - close_alerts(); - /* Process the completed command we found. */ - if ( i >= 0 ) { exit_reason = EXIT_OK; break; } - /* Check if a command ran out of time. */ - i = try_kill_one(); - if ( i >= 0 ) { exit_reason = EXIT_TIMEOUT; break; } - } - - /* We have a command... process it. */ - { - DWORD exit_code; - timing_info time; - int rstat; - - /* The time data for the command. */ - record_times( cmdtab[ i ].pi.hProcess, &time ); - - /* Removed the used temporary command file. */ - if ( cmdtab[ i ].command_file->size ) - unlink( cmdtab[ i ].command_file->value ); - - /* Find out the process exit code. */ - GetExitCodeProcess( cmdtab[ i ].pi.hProcess, &exit_code ); - - /* The dispossition of the command. */ - if ( interrupted() ) - rstat = EXEC_CMD_INTR; - else if ( exit_code ) - rstat = EXEC_CMD_FAIL; - else - rstat = EXEC_CMD_OK; - - /* Call the callback, may call back to jam rule land. */ - (*cmdtab[ i ].func)( cmdtab[ i ].closure, rstat, &time, - cmdtab[ i ].buffer_out->value, cmdtab[ i ].buffer_err->value, - exit_reason ); - - /* Clean up our child process tracking data. No need to clear the - * temporary command file name as it gets reused. - */ - closeWinHandle( &cmdtab[ i ].pi.hProcess ); - closeWinHandle( &cmdtab[ i ].pi.hThread ); - closeWinHandle( &cmdtab[ i ].pipe_out[ EXECCMD_PIPE_READ ] ); - closeWinHandle( &cmdtab[ i ].pipe_out[ EXECCMD_PIPE_WRITE ] ); - closeWinHandle( &cmdtab[ i ].pipe_err[ EXECCMD_PIPE_READ ] ); - closeWinHandle( &cmdtab[ i ].pipe_err[ EXECCMD_PIPE_WRITE ] ); - string_renew( cmdtab[ i ].buffer_out ); - string_renew( cmdtab[ i ].buffer_err ); - } -} - - -/* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ - -/* - * Invoke the actual external process using the given command line. Track the - * process in our running commands table. - */ - -static void invoke_cmd( char const * const command, int const slot ) -{ - SECURITY_ATTRIBUTES sa = { sizeof( SECURITY_ATTRIBUTES ), 0, 0 }; - SECURITY_DESCRIPTOR sd; - STARTUPINFO si = { sizeof( STARTUPINFO ), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0 }; - - /* Init the security data. */ - InitializeSecurityDescriptor( &sd, SECURITY_DESCRIPTOR_REVISION ); - SetSecurityDescriptorDacl( &sd, TRUE, NULL, FALSE ); - sa.lpSecurityDescriptor = &sd; - sa.bInheritHandle = TRUE; - - /* Create pipes for communicating with the child process. */ - if ( !CreatePipe( &cmdtab[ slot ].pipe_out[ EXECCMD_PIPE_READ ], - &cmdtab[ slot ].pipe_out[ EXECCMD_PIPE_WRITE ], &sa, 0 ) ) - { - reportWindowsError( "CreatePipe" ); - exit( EXITBAD ); - } - if ( globs.pipe_action && !CreatePipe( &cmdtab[ slot ].pipe_err[ - EXECCMD_PIPE_READ ], &cmdtab[ slot ].pipe_err[ EXECCMD_PIPE_WRITE ], - &sa, 0 ) ) - { - reportWindowsError( "CreatePipe" ); - exit( EXITBAD ); - } - - /* Set handle inheritance off for the pipe ends the parent reads from. */ - SetHandleInformation( cmdtab[ slot ].pipe_out[ EXECCMD_PIPE_READ ], - HANDLE_FLAG_INHERIT, 0 ); - if ( globs.pipe_action ) - SetHandleInformation( cmdtab[ slot ].pipe_err[ EXECCMD_PIPE_READ ], - HANDLE_FLAG_INHERIT, 0 ); - - /* Hide the child window, if any. */ - si.dwFlags |= STARTF_USESHOWWINDOW; - si.wShowWindow = SW_HIDE; - - /* Redirect the child's output streams to our pipes. */ - si.dwFlags |= STARTF_USESTDHANDLES; - si.hStdOutput = cmdtab[ slot ].pipe_out[ EXECCMD_PIPE_WRITE ]; - si.hStdError = globs.pipe_action - ? cmdtab[ slot ].pipe_err[ EXECCMD_PIPE_WRITE ] - : cmdtab[ slot ].pipe_out[ EXECCMD_PIPE_WRITE ]; - - /* Let the child inherit stdin, as some commands assume it is available. */ - si.hStdInput = GetStdHandle( STD_INPUT_HANDLE ); - - /* Create output buffers. */ - string_new( cmdtab[ slot ].buffer_out ); - string_new( cmdtab[ slot ].buffer_err ); - - if ( DEBUG_EXECCMD ) - printf( "Command string for CreateProcessA(): '%s'\n", command ); - - /* Run the command by creating a sub-process for it. */ - if ( !CreateProcessA( - NULL , /* application name */ - (char *)command , /* command line */ - NULL , /* process attributes */ - NULL , /* thread attributes */ - TRUE , /* inherit handles */ - CREATE_NEW_PROCESS_GROUP, /* create flags */ - NULL , /* env vars, null inherits env */ - NULL , /* current dir, null is our current dir */ - &si , /* startup info */ - &cmdtab[ slot ].pi ) ) /* child process info, if created */ - { - reportWindowsError( "CreateProcessA" ); - exit( EXITBAD ); - } -} - - -/* - * For more details on Windows cmd.exe shell command-line length limitations see - * the following MSDN article: - * http://support.microsoft.com/default.aspx?scid=kb;en-us;830473 - */ - -static int raw_maxline() -{ - OSVERSIONINFO os_info; - os_info.dwOSVersionInfoSize = sizeof( os_info ); - GetVersionEx( &os_info ); - - if ( os_info.dwMajorVersion >= 5 ) return 8191; /* XP */ - if ( os_info.dwMajorVersion == 4 ) return 2047; /* NT 4.x */ - return 996; /* NT 3.5.1 */ -} - -static int maxline() -{ - static result; - if ( !result ) result = raw_maxline(); - return result; -} - - -/* - * Closes a Windows HANDLE and resets its variable to 0. - */ - -static void closeWinHandle( HANDLE * const handle ) -{ - if ( *handle ) - { - CloseHandle( *handle ); - *handle = 0; - } -} - - -/* - * Frees and renews the given string. - */ - -static void string_renew( string * const s ) -{ - string_free( s ); - string_new( s ); -} - - -/* - * raw_command_length() - valid raw command string length - * - * Checks whether the given command may be executed as a raw command. If yes, - * returns the corresponding command string length. If not, returns -1. - * - * Rules for constructing raw command strings: - * - Command may not contain unquoted shell I/O redirection characters. - * - May have at most one command line with non-whitespace content. - * - Leading whitespace trimmed. - * - Trailing all-whitespace lines trimmed. - * - Trailing whitespace on the sole command line kept (may theoretically - * affect the executed command). - */ - -static long raw_command_length( char const * command ) -{ - char const * p; - char const * escape = 0; - char inquote = 0; - char const * newline = 0; - - /* Skip leading whitespace. */ - while ( isspace( *command ) ) - ++command; - - p = command; - - /* Look for newlines and unquoted I/O redirection. */ - do - { - p += strcspn( p, "\n\"'<>|\\" ); - switch ( *p ) - { - case '\n': - /* If our command contains non-whitespace content split over - * multiple lines we can not execute it directly. - */ - newline = p; - while ( isspace( *++p ) ); - if ( *p ) return -1; - break; - - case '\\': - escape = escape && escape == p - 1 ? 0 : p; - ++p; - break; - - case '"': - case '\'': - if ( escape && escape == p - 1 ) - escape = 0; - else if ( inquote == *p ) - inquote = 0; - else if ( !inquote ) - inquote = *p; - ++p; - break; - - case '<': - case '>': - case '|': - if ( !inquote ) - return -1; - ++p; - break; - } - } - while ( *p ); - - /* Return the number of characters the command will occupy. */ - return ( newline ? newline : p ) - command; -} - - -/* 64-bit arithmetic helpers. */ - -/* Compute the carry bit from the addition of two 32-bit unsigned numbers. */ -#define add_carry_bit( a, b ) ((((a) | (b)) >> 31) & (~((a) + (b)) >> 31) & 0x1) - -/* Compute the high 32 bits of the addition of two 64-bit unsigned numbers, h1l1 - * and h2l2. - */ -#define add_64_hi( h1, l1, h2, l2 ) ((h1) + (h2) + add_carry_bit(l1, l2)) - - -/* - * Add two 64-bit unsigned numbers, h1l1 and h2l2. - */ - -static FILETIME add_64 -( - unsigned long h1, unsigned long l1, - unsigned long h2, unsigned long l2 -) -{ - FILETIME result; - result.dwLowDateTime = l1 + l2; - result.dwHighDateTime = add_64_hi( h1, l1, h2, l2 ); - return result; -} - - -static FILETIME add_FILETIME( FILETIME t1, FILETIME t2 ) -{ - return add_64( t1.dwHighDateTime, t1.dwLowDateTime, t2.dwHighDateTime, - t2.dwLowDateTime ); -} - - -static FILETIME negate_FILETIME( FILETIME t ) -{ - /* 2s complement negation */ - return add_64( ~t.dwHighDateTime, ~t.dwLowDateTime, 0, 1 ); -} - - -/* - * filetime_to_seconds() - Windows FILETIME --> number of seconds conversion - */ - -static double filetime_to_seconds( FILETIME const ft ) -{ - return ft.dwHighDateTime * ( (double)( 1UL << 31 ) * 2.0 * 1.0e-7 ) + - ft.dwLowDateTime * 1.0e-7; -} - - -static void record_times( HANDLE const process, timing_info * const time ) -{ - FILETIME creation; - FILETIME exit; - FILETIME kernel; - FILETIME user; - if ( GetProcessTimes( process, &creation, &exit, &kernel, &user ) ) - { - time->system = filetime_to_seconds( kernel ); - time->user = filetime_to_seconds( user ); - timestamp_from_filetime( &time->start, &creation ); - timestamp_from_filetime( &time->end, &exit ); - } -} - - -#define IO_BUFFER_SIZE ( 16 * 1024 ) - -static char ioBuffer[ IO_BUFFER_SIZE + 1 ]; - - -static void read_pipe -( - HANDLE in, /* the pipe to read from */ - string * out -) -{ - DWORD bytesInBuffer = 0; - DWORD bytesAvailable = 0; - - do - { - /* check if we have any data to read */ - if ( !PeekNamedPipe( in, ioBuffer, IO_BUFFER_SIZE, &bytesInBuffer, - &bytesAvailable, NULL ) ) - bytesAvailable = 0; - - /* read in the available data */ - if ( bytesAvailable > 0 ) - { - /* we only read in the available bytes, to avoid blocking */ - if ( ReadFile( in, ioBuffer, bytesAvailable <= IO_BUFFER_SIZE ? - bytesAvailable : IO_BUFFER_SIZE, &bytesInBuffer, NULL ) ) - { - if ( bytesInBuffer > 0 ) - { - /* Clean up some illegal chars. */ - int i; - for ( i = 0; i < bytesInBuffer; ++i ) - { - if ( ( (unsigned char)ioBuffer[ i ] < 1 ) ) - ioBuffer[ i ] = '?'; - } - /* Null, terminate. */ - ioBuffer[ bytesInBuffer ] = '\0'; - /* Append to the output. */ - string_append( out, ioBuffer ); - /* Subtract what we read in. */ - bytesAvailable -= bytesInBuffer; - } - else - { - /* Likely read a error, bail out. */ - bytesAvailable = 0; - } - } - else - { - /* Definitely read a error, bail out. */ - bytesAvailable = 0; - } - } - } - while ( bytesAvailable > 0 ); -} - - -static void read_output() -{ - int i; - for ( i = 0; i < globs.jobs; ++i ) - if ( cmdtab[ i ].pi.hProcess ) - { - /* Read stdout data. */ - if ( cmdtab[ i ].pipe_out[ EXECCMD_PIPE_READ ] ) - read_pipe( cmdtab[ i ].pipe_out[ EXECCMD_PIPE_READ ], - cmdtab[ i ].buffer_out ); - /* Read stderr data. */ - if ( cmdtab[ i ].pipe_err[ EXECCMD_PIPE_READ ] ) - read_pipe( cmdtab[ i ].pipe_err[ EXECCMD_PIPE_READ ], - cmdtab[ i ].buffer_err ); - } -} - - -/* - * Waits for a single child process command to complete, or the timeout, - * whichever comes first. Returns the index of the completed command in the - * cmdtab array, or -1. - */ - -static int try_wait( int const timeoutMillis ) -{ - int i; - int num_active; - int wait_api_result; - HANDLE active_handles[ MAXJOBS ]; - int active_procs[ MAXJOBS ]; - - /* Prepare a list of all active processes to wait for. */ - for ( num_active = 0, i = 0; i < globs.jobs; ++i ) - if ( cmdtab[ i ].pi.hProcess ) - { - active_handles[ num_active ] = cmdtab[ i ].pi.hProcess; - active_procs[ num_active ] = i; - ++num_active; - } - - /* Wait for a child to complete, or for our timeout window to expire. */ - wait_api_result = WaitForMultipleObjects( num_active, active_handles, - FALSE, timeoutMillis ); - if ( ( WAIT_OBJECT_0 <= wait_api_result ) && - ( wait_api_result < WAIT_OBJECT_0 + num_active ) ) - { - /* Terminated process detected - return its index. */ - return active_procs[ wait_api_result - WAIT_OBJECT_0 ]; - } - - /* Timeout. */ - return -1; -} - - -static int try_kill_one() -{ - /* Only need to check if a timeout was specified with the -l option. */ - if ( globs.timeout > 0 ) - { - int i; - for ( i = 0; i < globs.jobs; ++i ) - if ( cmdtab[ i ].pi.hProcess ) - { - double const t = running_time( cmdtab[ i ].pi.hProcess ); - if ( t > (double)globs.timeout ) - { - /* The job may have left an alert dialog around, try and get - * rid of it before killing the job itself. - */ - close_alert( &cmdtab[ i ].pi ); - /* We have a "runaway" job, kill it. */ - kill_process_tree( cmdtab[ i ].pi.dwProcessId, - cmdtab[ i ].pi.hProcess ); - /* And return its running commands table slot. */ - return i; - } - } - } - return -1; -} - - -static void close_alerts() -{ - /* We only attempt this every 5 seconds or so, because it is not a cheap - * operation, and we will catch the alerts eventually. This check uses - * floats as some compilers define CLOCKS_PER_SEC as a float or double. - */ - if ( ( (float)clock() / (float)( CLOCKS_PER_SEC * 5 ) ) < ( 1.0 / 5.0 ) ) - { - int i; - for ( i = 0; i < globs.jobs; ++i ) - if ( cmdtab[ i ].pi.hProcess ) - close_alert( &cmdtab[ i ].pi ); - } -} - - -/* - * Calc the current running time of an *active* process. - */ - -static double running_time( HANDLE const process ) -{ - FILETIME creation; - FILETIME exit; - FILETIME kernel; - FILETIME user; - if ( GetProcessTimes( process, &creation, &exit, &kernel, &user ) ) - { - /* Compute the elapsed time. */ - FILETIME current; - GetSystemTimeAsFileTime( ¤t ); - return filetime_to_seconds( add_FILETIME( current, - negate_FILETIME( creation ) ) ); - } - return 0.0; -} - - -/* - * Not really optimal, or efficient, but it is easier this way, and it is not - * like we are going to be killing thousands, or even tens of processes. - */ - -static void kill_process_tree( DWORD const pid, HANDLE const process ) -{ - HANDLE const process_snapshot_h = CreateToolhelp32Snapshot( - TH32CS_SNAPPROCESS, 0 ); - if ( INVALID_HANDLE_VALUE != process_snapshot_h ) - { - BOOL ok = TRUE; - PROCESSENTRY32 pinfo; - pinfo.dwSize = sizeof( PROCESSENTRY32 ); - for ( - ok = Process32First( process_snapshot_h, &pinfo ); - ok == TRUE; - ok = Process32Next( process_snapshot_h, &pinfo ) ) - { - if ( pinfo.th32ParentProcessID == pid ) - { - /* Found a child, recurse to kill it and anything else below it. - */ - HANDLE const ph = OpenProcess( PROCESS_ALL_ACCESS, FALSE, - pinfo.th32ProcessID ); - if ( ph ) - { - kill_process_tree( pinfo.th32ProcessID, ph ); - CloseHandle( ph ); - } - } - } - CloseHandle( process_snapshot_h ); - } - /* Now that the children are all dead, kill the root. */ - TerminateProcess( process, -2 ); -} - - -static double creation_time( HANDLE const process ) -{ - FILETIME creation; - FILETIME exit; - FILETIME kernel; - FILETIME user; - return GetProcessTimes( process, &creation, &exit, &kernel, &user ) - ? filetime_to_seconds( creation ) - : 0.0; -} - - -/* - * Recursive check if first process is parent (directly or indirectly) of the - * second one. Both processes are passed as process ids, not handles. Special - * return value 2 means that the second process is smss.exe and its parent - * process is System (first argument is ignored). - */ - -static int is_parent_child( DWORD const parent, DWORD const child ) -{ - HANDLE process_snapshot_h = INVALID_HANDLE_VALUE; - - if ( !child ) - return 0; - if ( parent == child ) - return 1; - - process_snapshot_h = CreateToolhelp32Snapshot( TH32CS_SNAPPROCESS, 0 ); - if ( INVALID_HANDLE_VALUE != process_snapshot_h ) - { - BOOL ok = TRUE; - PROCESSENTRY32 pinfo; - pinfo.dwSize = sizeof( PROCESSENTRY32 ); - for ( - ok = Process32First( process_snapshot_h, &pinfo ); - ok == TRUE; - ok = Process32Next( process_snapshot_h, &pinfo ) ) - { - if ( pinfo.th32ProcessID == child ) - { - /* Unfortunately, process ids are not really unique. There might - * be spurious "parent and child" relationship match between two - * non-related processes if real parent process of a given - * process has exited (while child process kept running as an - * "orphan") and the process id of such parent process has been - * reused by internals of the operating system when creating - * another process. - * - * Thus an additional check is needed - process creation time. - * This check may fail (i.e. return 0) for system processes due - * to insufficient privileges, and that is OK. - */ - double tchild = 0.0; - double tparent = 0.0; - HANDLE const hchild = OpenProcess( PROCESS_QUERY_INFORMATION, - FALSE, pinfo.th32ProcessID ); - CloseHandle( process_snapshot_h ); - - /* csrss.exe may display message box like following: - * xyz.exe - Unable To Locate Component - * This application has failed to start because - * boost_foo-bar.dll was not found. Re-installing the - * application may fix the problem - * This actually happens when starting a test process that - * depends on a dynamic library which failed to build. We want - * to automatically close these message boxes even though - * csrss.exe is not our child process. We may depend on the fact - * that (in all current versions of Windows) csrss.exe is a - * direct child of the smss.exe process, which in turn is a - * direct child of the System process, which always has process - * id == 4. This check must be performed before comparing - * process creation times. - */ - if ( !stricmp( pinfo.szExeFile, "csrss.exe" ) && - is_parent_child( parent, pinfo.th32ParentProcessID ) == 2 ) - return 1; - if ( !stricmp( pinfo.szExeFile, "smss.exe" ) && - ( pinfo.th32ParentProcessID == 4 ) ) - return 2; - - if ( hchild ) - { - HANDLE hparent = OpenProcess( PROCESS_QUERY_INFORMATION, - FALSE, pinfo.th32ParentProcessID ); - if ( hparent ) - { - tchild = creation_time( hchild ); - tparent = creation_time( hparent ); - CloseHandle( hparent ); - } - CloseHandle( hchild ); - } - - /* Return 0 if one of the following is true: - * 1. we failed to read process creation time - * 2. child was created before alleged parent - */ - if ( ( tchild == 0.0 ) || ( tparent == 0.0 ) || - ( tchild < tparent ) ) - return 0; - - return is_parent_child( parent, pinfo.th32ParentProcessID ) & 1; - } - } - - CloseHandle( process_snapshot_h ); - } - - return 0; -} - - -/* - * Called by the OS for each topmost window. - */ - -BOOL CALLBACK close_alert_window_enum( HWND hwnd, LPARAM lParam ) -{ - char buf[ 7 ] = { 0 }; - PROCESS_INFORMATION const * const pi = (PROCESS_INFORMATION *)lParam; - DWORD pid; - DWORD tid; - - /* We want to find and close any window that: - * 1. is visible and - * 2. is a dialog and - * 3. is displayed by any of our child processes - */ - if ( - /* We assume hidden windows do not require user interaction. */ - !IsWindowVisible( hwnd ) - /* Failed to read class name; presume it is not a dialog. */ - || !GetClassNameA( hwnd, buf, sizeof( buf ) ) - /* All Windows system dialogs use the same Window class name. */ - || strcmp( buf, "#32770" ) ) - return TRUE; - - /* GetWindowThreadProcessId() returns 0 on error, otherwise thread id of - * the window's message pump thread. - */ - tid = GetWindowThreadProcessId( hwnd, &pid ); - if ( !tid || !is_parent_child( pi->dwProcessId, pid ) ) - return TRUE; - - /* Ask real nice. */ - PostMessageA( hwnd, WM_CLOSE, 0, 0 ); - - /* Wait and see if it worked. If not, insist. */ - if ( WaitForSingleObject( pi->hProcess, 200 ) == WAIT_TIMEOUT ) - { - PostThreadMessageA( tid, WM_QUIT, 0, 0 ); - WaitForSingleObject( pi->hProcess, 300 ); - } - - /* Done, we do not want to check any other windows now. */ - return FALSE; -} - - -static void close_alert( PROCESS_INFORMATION const * const pi ) -{ - EnumWindows( &close_alert_window_enum, (LPARAM)pi ); -} - - -/* - * Open a command file to store the command into for executing using an external - * shell. Returns a pointer to a FILE open for writing or 0 in case such a file - * could not be opened. The file name used is stored back in the corresponding - * running commands table slot. - * - * Expects the running commands table slot's command_file attribute to contain - * either a zeroed out string object or one prepared previously by this same - * function. - */ - -static FILE * open_command_file( int const slot ) -{ - string * const command_file = cmdtab[ slot ].command_file; - - /* If the temporary command file name has not already been prepared for this - * slot number, prepare a new one containing a '##' place holder that will - * be changed later and needs to be located at a fixed distance from the - * end. - */ - if ( !command_file->value ) - { - DWORD const procID = GetCurrentProcessId(); - string const * const tmpdir = path_tmpdir(); - string_new( command_file ); - string_reserve( command_file, tmpdir->size + 64 ); - command_file->size = sprintf( command_file->value, - "%s\\jam%d-%02d-##.bat", tmpdir->value, procID, slot ); - } - - /* For some reason opening a command file can fail intermittently. But doing - * some retries works. Most likely this is due to a previously existing file - * of the same name that happens to still be opened by an active virus - * scanner. Originally pointed out and fixed by Bronek Kozicki. - * - * We first try to open several differently named files to avoid having to - * wait idly if not absolutely necessary. Our temporary command file names - * contain a fixed position place holder we use for generating different - * file names. - */ - { - char * const index1 = command_file->value + command_file->size - 6; - char * const index2 = index1 + 1; - int waits_remaining; - assert( command_file->value < index1 ); - assert( index2 + 1 < command_file->value + command_file->size ); - assert( index2[ 1 ] == '.' ); - for ( waits_remaining = 3; ; --waits_remaining ) - { - int index; - for ( index = 0; index != 20; ++index ) - { - FILE * f; - *index1 = '0' + index / 10; - *index2 = '0' + index % 10; - f = fopen( command_file->value, "w" ); - if ( f ) return f; - } - if ( !waits_remaining ) break; - Sleep( 250 ); - } - } - - return 0; -} - - -/* - * Prepare a command file to be executed using an external shell. - */ - -static char const * prepare_command_file( string const * command, int slot ) -{ - FILE * const f = open_command_file( slot ); - if ( !f ) - { - printf( "failed to write command file!\n" ); - exit( EXITBAD ); - } - fputs( command->value, f ); - fclose( f ); - return cmdtab[ slot ].command_file->value; -} - - -/* - * Find a free slot in the running commands table. - */ - -static int get_free_cmdtab_slot() -{ - int slot; - for ( slot = 0; slot < MAXJOBS; ++slot ) - if ( !cmdtab[ slot ].pi.hProcess ) - return slot; - printf( "no slots for child!\n" ); - exit( EXITBAD ); -} - - -/* - * Put together the final command string we are to run. - */ - -static void string_new_from_argv( string * result, char const * const * argv ) -{ - assert( argv ); - assert( argv[ 0 ] ); - string_copy( result, *(argv++) ); - while ( *argv ) - { - string_push_back( result, ' ' ); - string_append( result, *(argv++) ); - } -} - - -/* - * Reports the last failed Windows API related error message. - */ - -static void reportWindowsError( char const * const apiName ) -{ - char * errorMessage; - DWORD const errorCode = GetLastError(); - DWORD apiResult = FormatMessageA( - FORMAT_MESSAGE_ALLOCATE_BUFFER | /* __in DWORD dwFlags */ - FORMAT_MESSAGE_FROM_SYSTEM | - FORMAT_MESSAGE_IGNORE_INSERTS, - NULL, /* __in_opt LPCVOID lpSource */ - errorCode, /* __in DWORD dwMessageId */ - 0, /* __in DWORD dwLanguageId */ - (LPSTR)&errorMessage, /* __out LPTSTR lpBuffer */ - 0, /* __in DWORD nSize */ - 0 ); /* __in_opt va_list * Arguments */ - if ( !apiResult ) - printf( "%s() Windows API failed: %d.\n", apiName, errorCode ); - else - { - printf( "%s() Windows API failed: %d - %s\n", apiName, errorCode, - errorMessage ); - LocalFree( errorMessage ); - } -} - - -#endif /* USE_EXECNT */
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/execunix.c ---------------------------------------------------------------------- diff --git a/ext/kenlm b/ext/kenlm new file mode 160000 index 0000000..56fdb5c --- /dev/null +++ b/ext/kenlm @@ -0,0 +1 @@ +Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5 diff --git a/ext/kenlm/jam-files/engine/execunix.c b/ext/kenlm/jam-files/engine/execunix.c deleted file mode 100644 index 965e580..0000000 --- a/ext/kenlm/jam-files/engine/execunix.c +++ /dev/null @@ -1,559 +0,0 @@ -/* - * Copyright 1993, 1995 Christopher Seiwald. - * Copyright 2007 Noel Belcourt. - * - * This file is part of Jam - see jam.c for Copyright information. - */ - -#include "jam.h" -#include "execcmd.h" - -#include "lists.h" -#include "output.h" -#include "strings.h" - -#include <errno.h> -#include <signal.h> -#include <stdio.h> -#include <time.h> -#include <unistd.h> /* vfork(), _exit(), STDOUT_FILENO and such */ -#include <sys/resource.h> -#include <sys/times.h> -#include <sys/wait.h> - -#if defined(sun) || defined(__sun) - #include <wait.h> -#endif - -#ifdef USE_EXECUNIX - -#include <sys/times.h> - -#if defined(__APPLE__) - #define NO_VFORK -#endif - -#ifdef NO_VFORK - #define vfork() fork() -#endif - - -/* - * execunix.c - execute a shell script on UNIX/OS2/AmigaOS - * - * If $(JAMSHELL) is defined, uses that to formulate execvp()/spawnvp(). The - * default is: /bin/sh -c - * - * In $(JAMSHELL), % expands to the command string and ! expands to the slot - * number (starting at 1) for multiprocess (-j) invocations. If $(JAMSHELL) does - * not include a %, it is tacked on as the last argument. - * - * Each word must be an individual element in a jam variable value. - * - * Do not just set JAMSHELL to /bin/sh - it will not work! - * - * External routines: - * exec_check() - preprocess and validate the command. - * exec_cmd() - launch an async command execution. - * exec_wait() - wait for any of the async command processes to terminate. - */ - -/* find a free slot in the running commands table */ -static int get_free_cmdtab_slot(); - -/* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ - -static clock_t tps; -static int old_time_initialized; -static struct tms old_time; - -/* We hold stdout & stderr child process information in two element arrays - * indexed as follows. - */ -#define OUT 0 -#define ERR 1 - -static struct -{ - int pid; /* on win32, a real process handle */ - int fd[ 2 ]; /* file descriptors for stdout and stderr */ - FILE * stream[ 2 ]; /* child's stdout and stderr file streams */ - clock_t start_time; /* start time of child process */ - int exit_reason; /* termination status */ - char * buffer[ 2 ]; /* buffers to hold stdout and stderr, if any */ - int buf_size[ 2 ]; /* buffer sizes in bytes */ - timestamp start_dt; /* start of command timestamp */ - - /* Function called when the command completes. */ - ExecCmdCallback func; - - /* Opaque data passed back to the 'func' callback. */ - void * closure; -} cmdtab[ MAXJOBS ] = { { 0 } }; - - -/* - * exec_check() - preprocess and validate the command. - */ - -int exec_check -( - string const * command, - LIST * * pShell, - int * error_length, - int * error_max_length -) -{ - int const is_raw_cmd = is_raw_command_request( *pShell ); - - /* We allow empty commands for non-default shells since we do not really - * know what they are going to do with such commands. - */ - if ( !command->size && ( is_raw_cmd || list_empty( *pShell ) ) ) - return EXEC_CHECK_NOOP; - - return is_raw_cmd - ? EXEC_CHECK_OK - : check_cmd_for_too_long_lines( command->value, MAXLINE, error_length, - error_max_length ); -} - - -/* - * exec_cmd() - launch an async command execution. - */ - -/* We hold file descriptors for pipes used to communicate with child processes - * in two element arrays indexed as follows. - */ -#define EXECCMD_PIPE_READ 0 -#define EXECCMD_PIPE_WRITE 1 - -void exec_cmd -( - string const * command, - ExecCmdCallback func, - void * closure, - LIST * shell -) -{ - int const slot = get_free_cmdtab_slot(); - int out[ 2 ]; - int err[ 2 ]; - int len; - char const * argv[ MAXARGC + 1 ]; /* +1 for NULL */ - - /* Initialize default shell. */ - static LIST * default_shell; - if ( !default_shell ) - default_shell = list_push_back( list_new( - object_new( "/bin/sh" ) ), - object_new( "-c" ) ); - - if ( list_empty( shell ) ) - shell = default_shell; - - /* Forumulate argv. If shell was defined, be prepared for % and ! subs. - * Otherwise, use stock /bin/sh. - */ - argv_from_shell( argv, shell, command->value, slot ); - - if ( DEBUG_EXECCMD ) - { - int i; - printf( "Using shell: " ); - list_print( shell ); - printf( "\n" ); - for ( i = 0; argv[ i ]; ++i ) - printf( " argv[%d] = '%s'\n", i, argv[ i ] ); - } - - /* Create pipes for collecting child output. */ - if ( pipe( out ) < 0 || ( globs.pipe_action && pipe( err ) < 0 ) ) - { - perror( "pipe" ); - exit( EXITBAD ); - } - - /* Initialize old_time only once. */ - if ( !old_time_initialized ) - { - times( &old_time ); - old_time_initialized = 1; - } - - /* Start the command */ - - timestamp_current( &cmdtab[ slot ].start_dt ); - - if ( 0 < globs.timeout ) - { - /* Handle hung processes by manually tracking elapsed time and signal - * process when time limit expires. - */ - struct tms buf; - cmdtab[ slot ].start_time = times( &buf ); - - /* Make a global, only do this once. */ - if ( !tps ) tps = sysconf( _SC_CLK_TCK ); - } - - /* Child does not need the read pipe ends used by the parent. */ - fcntl( out[ EXECCMD_PIPE_READ ], F_SETFD, FD_CLOEXEC ); - if ( globs.pipe_action ) - fcntl( err[ EXECCMD_PIPE_READ ], F_SETFD, FD_CLOEXEC ); - - if ( ( cmdtab[ slot ].pid = vfork() ) == -1 ) - { - perror( "vfork" ); - exit( EXITBAD ); - } - - if ( cmdtab[ slot ].pid == 0 ) - { - /*****************/ - /* Child process */ - /*****************/ - int const pid = getpid(); - - /* Redirect stdout and stderr to pipes inherited from the parent. */ - dup2( out[ EXECCMD_PIPE_WRITE ], STDOUT_FILENO ); - dup2( globs.pipe_action ? err[ EXECCMD_PIPE_WRITE ] : - out[ EXECCMD_PIPE_WRITE ], STDERR_FILENO ); - close( out[ EXECCMD_PIPE_WRITE ] ); - if ( globs.pipe_action ) - close( err[ EXECCMD_PIPE_WRITE ] ); - - /* Make this process a process group leader so that when we kill it, all - * child processes of this process are terminated as well. We use - * killpg( pid, SIGKILL ) to kill the process group leader and all its - * children. - */ - if ( 0 < globs.timeout ) - { - struct rlimit r_limit; - r_limit.rlim_cur = globs.timeout; - r_limit.rlim_max = globs.timeout; - setrlimit( RLIMIT_CPU, &r_limit ); - } - setpgid( pid, pid ); - execvp( argv[ 0 ], (char * *)argv ); - perror( "execvp" ); - _exit( 127 ); - } - - /******************/ - /* Parent process */ - /******************/ - setpgid( cmdtab[ slot ].pid, cmdtab[ slot ].pid ); - - /* Parent not need the write pipe ends used by the child. */ - close( out[ EXECCMD_PIPE_WRITE ] ); - if ( globs.pipe_action ) - close( err[ EXECCMD_PIPE_WRITE ] ); - - /* Set both pipe read file descriptors to non-blocking. */ - fcntl( out[ EXECCMD_PIPE_READ ], F_SETFL, O_NONBLOCK ); - if ( globs.pipe_action ) - fcntl( err[ EXECCMD_PIPE_READ ], F_SETFL, O_NONBLOCK ); - - /* Parent reads from out[ EXECCMD_PIPE_READ ]. */ - cmdtab[ slot ].fd[ OUT ] = out[ EXECCMD_PIPE_READ ]; - cmdtab[ slot ].stream[ OUT ] = fdopen( cmdtab[ slot ].fd[ OUT ], "rb" ); - if ( !cmdtab[ slot ].stream[ OUT ] ) - { - perror( "fdopen" ); - exit( EXITBAD ); - } - - /* Parent reads from err[ EXECCMD_PIPE_READ ]. */ - if ( globs.pipe_action ) - { - cmdtab[ slot ].fd[ ERR ] = err[ EXECCMD_PIPE_READ ]; - cmdtab[ slot ].stream[ ERR ] = fdopen( cmdtab[ slot ].fd[ ERR ], "rb" ); - if ( !cmdtab[ slot ].stream[ ERR ] ) - { - perror( "fdopen" ); - exit( EXITBAD ); - } - } - - /* Save input data into the selected running commands table slot. */ - cmdtab[ slot ].func = func; - cmdtab[ slot ].closure = closure; -} - -#undef EXECCMD_PIPE_READ -#undef EXECCMD_PIPE_WRITE - - -/* Returns 1 if file descriptor is closed, or 0 if it is still alive. - * - * i is index into cmdtab - * - * s (stream) indexes: - * - cmdtab[ i ].stream[ s ] - * - cmdtab[ i ].buffer[ s ] - * - cmdtab[ i ].fd [ s ] - */ - -static int read_descriptor( int i, int s ) -{ - int ret; - char buffer[ BUFSIZ ]; - - while ( 0 < ( ret = fread( buffer, sizeof( char ), BUFSIZ - 1, - cmdtab[ i ].stream[ s ] ) ) ) - { - buffer[ ret ] = 0; - if ( !cmdtab[ i ].buffer[ s ] ) - { - /* Never been allocated. */ - if ( globs.max_buf && ret > globs.max_buf ) - { - ret = globs.max_buf; - buffer[ ret ] = 0; - } - cmdtab[ i ].buf_size[ s ] = ret + 1; - cmdtab[ i ].buffer[ s ] = (char*)BJAM_MALLOC_ATOMIC( ret + 1 ); - memcpy( cmdtab[ i ].buffer[ s ], buffer, ret + 1 ); - } - else - { - /* Previously allocated. */ - if ( cmdtab[ i ].buf_size[ s ] < globs.max_buf || !globs.max_buf ) - { - char * tmp = cmdtab[ i ].buffer[ s ]; - int const old_len = cmdtab[ i ].buf_size[ s ] - 1; - int const new_len = old_len + ret + 1; - cmdtab[ i ].buf_size[ s ] = new_len; - cmdtab[ i ].buffer[ s ] = (char*)BJAM_MALLOC_ATOMIC( new_len ); - memcpy( cmdtab[ i ].buffer[ s ], tmp, old_len ); - memcpy( cmdtab[ i ].buffer[ s ] + old_len, buffer, ret + 1 ); - BJAM_FREE( tmp ); - } - } - } - - /* If buffer full, ensure last buffer char is newline so that jam log - * contains the command status at beginning of it own line instead of - * appended to end of the previous output. - */ - if ( globs.max_buf && globs.max_buf <= cmdtab[ i ].buf_size[ s ] ) - cmdtab[ i ].buffer[ s ][ cmdtab[ i ].buf_size[ s ] - 2 ] = '\n'; - - return feof( cmdtab[ i ].stream[ s ] ); -} - - -/* - * close_streams() - Close the stream and pipe descriptor. - */ - -static void close_streams( int const i, int const s ) -{ - fclose( cmdtab[ i ].stream[ s ] ); - cmdtab[ i ].stream[ s ] = 0; - - close( cmdtab[ i ].fd[ s ] ); - cmdtab[ i ].fd[ s ] = 0; -} - - -/* - * Populate the file descriptors collection for use in select() and return the - * maximal included file descriptor value. - */ - -static int populate_file_descriptors( fd_set * const fds ) -{ - int i; - int fd_max = 0; - - FD_ZERO( fds ); - for ( i = 0; i < globs.jobs; ++i ) - { - int fd; - if ( ( fd = cmdtab[ i ].fd[ OUT ] ) > 0 ) - { - if ( fd > fd_max ) fd_max = fd; - FD_SET( fd, fds ); - } - if ( globs.pipe_action ) - { - if ( ( fd = cmdtab[ i ].fd[ ERR ] ) > 0 ) - { - if ( fd > fd_max ) fd_max = fd; - FD_SET( fd, fds ); - } - } - } - return fd_max; -} - - -/* - * exec_wait() - wait for any of the async command processes to terminate. - * - * May register more than one terminated child process but will exit as soon as - * at least one has been registered. - */ - -void exec_wait() -{ - int finished = 0; - - /* Process children that signaled. */ - while ( !finished ) - { - int i; - struct timeval tv; - struct timeval * ptv = NULL; - int select_timeout = globs.timeout; - - /* Prepare file descriptor information for use in select(). */ - fd_set fds; - int const fd_max = populate_file_descriptors( &fds ); - - /* Check for timeouts: - * - kill children that already timed out - * - decide how long until the next one times out - */ - if ( globs.timeout > 0 ) - { - struct tms buf; - clock_t const current = times( &buf ); - for ( i = 0; i < globs.jobs; ++i ) - if ( cmdtab[ i ].pid ) - { - clock_t const consumed = - ( current - cmdtab[ i ].start_time ) / tps; - if ( consumed >= globs.timeout ) - { - killpg( cmdtab[ i ].pid, SIGKILL ); - cmdtab[ i ].exit_reason = EXIT_TIMEOUT; - } - else if ( globs.timeout - consumed < select_timeout ) - select_timeout = globs.timeout - consumed; - } - - /* If nothing else causes our select() call to exit, force it after - * however long it takes for the next one of our child processes to - * crossed its alloted processing time so we can terminate it. - */ - tv.tv_sec = select_timeout; - tv.tv_usec = 0; - ptv = &tv; - } - - /* select() will wait for I/O on a descriptor, a signal, or timeout. */ - { - int ret; - while ( ( ret = select( fd_max + 1, &fds, 0, 0, ptv ) ) == -1 ) - if ( errno != EINTR ) - break; - if ( ret <= 0 ) - continue; - } - - for ( i = 0; i < globs.jobs; ++i ) - { - int out_done = 0; - int err_done = 0; - if ( FD_ISSET( cmdtab[ i ].fd[ OUT ], &fds ) ) - out_done = read_descriptor( i, OUT ); - - if ( globs.pipe_action && FD_ISSET( cmdtab[ i ].fd[ ERR ], &fds ) ) - err_done = read_descriptor( i, ERR ); - - /* If feof on either descriptor, we are done. */ - if ( out_done || err_done ) - { - int pid; - int status; - int rstat; - timing_info time_info; - - /* We found a terminated child process - our search is done. */ - finished = 1; - - /* Close the stream and pipe descriptors. */ - close_streams( i, OUT ); - if ( globs.pipe_action ) - close_streams( i, ERR ); - - /* Reap the child and release resources. */ - while ( ( pid = waitpid( cmdtab[ i ].pid, &status, 0 ) ) == -1 ) - if ( errno != EINTR ) - break; - if ( pid != cmdtab[ i ].pid ) - { - printf( "unknown pid %d with errno = %d\n", pid, errno ); - exit( EXITBAD ); - } - - /* Set reason for exit if not timed out. */ - if ( WIFEXITED( status ) ) - cmdtab[ i ].exit_reason = WEXITSTATUS( status ) - ? EXIT_FAIL - : EXIT_OK; - - { - struct tms new_time; - times( &new_time ); - time_info.system = (double)( new_time.tms_cstime - - old_time.tms_cstime ) / CLOCKS_PER_SEC; - time_info.user = (double)( new_time.tms_cutime - - old_time.tms_cutime ) / CLOCKS_PER_SEC; - timestamp_copy( &time_info.start, &cmdtab[ i ].start_dt ); - timestamp_current( &time_info.end ); - old_time = new_time; - } - - /* Drive the completion. */ - if ( interrupted() ) - rstat = EXEC_CMD_INTR; - else if ( status ) - rstat = EXEC_CMD_FAIL; - else - rstat = EXEC_CMD_OK; - - /* Call the callback, may call back to jam rule land. */ - (*cmdtab[ i ].func)( cmdtab[ i ].closure, rstat, &time_info, - cmdtab[ i ].buffer[ OUT ], cmdtab[ i ].buffer[ ERR ], - cmdtab[ i ].exit_reason ); - - /* Clean up the command's running commands table slot. */ - BJAM_FREE( cmdtab[ i ].buffer[ OUT ] ); - cmdtab[ i ].buffer[ OUT ] = 0; - cmdtab[ i ].buf_size[ OUT ] = 0; - - BJAM_FREE( cmdtab[ i ].buffer[ ERR ] ); - cmdtab[ i ].buffer[ ERR ] = 0; - cmdtab[ i ].buf_size[ ERR ] = 0; - - cmdtab[ i ].pid = 0; - cmdtab[ i ].func = 0; - cmdtab[ i ].closure = 0; - cmdtab[ i ].start_time = 0; - } - } - } -} - - -/* - * Find a free slot in the running commands table. - */ - -static int get_free_cmdtab_slot() -{ - int slot; - for ( slot = 0; slot < MAXJOBS; ++slot ) - if ( !cmdtab[ slot ].pid ) - return slot; - printf( "no slots for child!\n" ); - exit( EXITBAD ); -} - -# endif /* USE_EXECUNIX */ http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/filent.c ---------------------------------------------------------------------- diff --git a/ext/kenlm b/ext/kenlm new file mode 160000 index 0000000..56fdb5c --- /dev/null +++ b/ext/kenlm @@ -0,0 +1 @@ +Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5 diff --git a/ext/kenlm/jam-files/engine/filent.c b/ext/kenlm/jam-files/engine/filent.c deleted file mode 100644 index e4ac331..0000000 --- a/ext/kenlm/jam-files/engine/filent.c +++ /dev/null @@ -1,435 +0,0 @@ -/* - * Copyright 1993, 1995 Christopher Seiwald. - * - * This file is part of Jam - see jam.c for Copyright information. - */ - -/* This file is ALSO: - * Copyright 2001-2004 David Abrahams. - * Copyright 2005 Rene Rivera. - * Distributed under the Boost Software License, Version 1.0. - * (See accompanying file LICENSE_1_0.txt or copy at - * http://www.boost.org/LICENSE_1_0.txt) - */ - -/* - * filent.c - scan directories and archives on NT - * - * External routines: - * file_archscan() - scan an archive for files - * file_mkdir() - create a directory - * file_supported_fmt_resolution() - file modification timestamp resolution - * - * External routines called only via routines in filesys.c: - * file_collect_dir_content_() - collects directory content information - * file_dirscan_() - OS specific file_dirscan() implementation - * file_query_() - query information about a path from the OS - */ - -#include "jam.h" -#ifdef OS_NT -#include "filesys.h" - -#include "object.h" -#include "pathsys.h" -#include "strings.h" - -#ifdef __BORLANDC__ -# undef FILENAME /* cpp namespace collision */ -#endif - -#define WIN32_LEAN_AND_MEAN -#include <windows.h> - -#include <assert.h> -#include <ctype.h> -#include <direct.h> -#include <io.h> - - -/* - * file_collect_dir_content_() - collects directory content information - */ - -int file_collect_dir_content_( file_info_t * const d ) -{ - PATHNAME f; - string pathspec[ 1 ]; - string pathname[ 1 ]; - LIST * files = L0; - int d_length; - - assert( d ); - assert( d->is_dir ); - assert( list_empty( d->files ) ); - - d_length = strlen( object_str( d->name ) ); - - memset( (char *)&f, '\0', sizeof( f ) ); - f.f_dir.ptr = object_str( d->name ); - f.f_dir.len = d_length; - - /* Prepare file search specification for the FindXXX() Windows API. */ - if ( !d_length ) - string_copy( pathspec, ".\\*" ); - else - { - /* We can not simply assume the given folder name will never include its - * trailing path separator or otherwise we would not support the Windows - * root folder specified without its drive letter, i.e. '\'. - */ - char const trailingChar = object_str( d->name )[ d_length - 1 ] ; - string_copy( pathspec, object_str( d->name ) ); - if ( ( trailingChar != '\\' ) && ( trailingChar != '/' ) ) - string_append( pathspec, "\\" ); - string_append( pathspec, "*" ); - } - - /* The following code for collecting information about all files in a folder - * needs to be kept synchronized with how the file_query() operation is - * implemented (collects information about a single file). - */ - { - /* FIXME: Avoid duplicate FindXXX Windows API calls here and in the code - * determining a normalized path. - */ - WIN32_FIND_DATA finfo; - HANDLE const findHandle = FindFirstFileA( pathspec->value, &finfo ); - if ( findHandle == INVALID_HANDLE_VALUE ) - { - string_free( pathspec ); - return -1; - } - - string_new( pathname ); - do - { - OBJECT * pathname_obj; - - f.f_base.ptr = finfo.cFileName; - f.f_base.len = strlen( finfo.cFileName ); - - string_truncate( pathname, 0 ); - path_build( &f, pathname ); - - pathname_obj = object_new( pathname->value ); - path_register_key( pathname_obj ); - files = list_push_back( files, pathname_obj ); - { - int found; - file_info_t * const ff = file_info( pathname_obj, &found ); - ff->is_dir = finfo.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY; - ff->is_file = !ff->is_dir; - ff->exists = 1; - timestamp_from_filetime( &ff->time, &finfo.ftLastWriteTime ); - } - } - while ( FindNextFile( findHandle, &finfo ) ); - - FindClose( findHandle ); - } - - string_free( pathname ); - string_free( pathspec ); - - d->files = files; - return 0; -} - - -/* - * file_dirscan_() - OS specific file_dirscan() implementation - */ - -void file_dirscan_( file_info_t * const d, scanback func, void * closure ) -{ - assert( d ); - assert( d->is_dir ); - - /* Special case \ or d:\ : enter it */ - { - char const * const name = object_str( d->name ); - if ( name[ 0 ] == '\\' && !name[ 1 ] ) - { - (*func)( closure, d->name, 1 /* stat()'ed */, &d->time ); - } - else if ( name[ 0 ] && name[ 1 ] == ':' && name[ 2 ] && !name[ 3 ] ) - { - /* We have just entered a 3-letter drive name spelling (with a - * trailing slash), into the hash table. Now enter its two-letter - * variant, without the trailing slash, so that if we try to check - * whether "c:" exists, we hit it. - * - * Jam core has workarounds for that. Given: - * x = c:\whatever\foo ; - * p = $(x:D) ; - * p2 = $(p:D) ; - * There will be no trailing slash in $(p), but there will be one in - * $(p2). But, that seems rather fragile. - */ - OBJECT * const dir_no_slash = object_new_range( name, 2 ); - (*func)( closure, d->name, 1 /* stat()'ed */, &d->time ); - (*func)( closure, dir_no_slash, 1 /* stat()'ed */, &d->time ); - object_free( dir_no_slash ); - } - } -} - - -/* - * file_mkdir() - create a directory - */ - -int file_mkdir( char const * const path ) -{ - return _mkdir( path ); -} - - -/* - * file_query_() - query information about a path from the OS - * - * The following code for collecting information about a single file needs to be - * kept synchronized with how the file_collect_dir_content_() operation is - * implemented (collects information about all files in a folder). - */ - -int try_file_query_root( file_info_t * const info ) -{ - WIN32_FILE_ATTRIBUTE_DATA fileData; - char buf[ 4 ]; - char const * const pathstr = object_str( info->name ); - if ( !pathstr[ 0 ] ) - { - buf[ 0 ] = '.'; - buf[ 1 ] = 0; - } - else if ( pathstr[ 0 ] == '\\' && ! pathstr[ 1 ] ) - { - buf[ 0 ] = '\\'; - buf[ 1 ] = '\0'; - } - else if ( pathstr[ 1 ] == ':' ) - { - if ( !pathstr[ 2 ] ) - { - } - else if ( !pathstr[ 2 ] || ( pathstr[ 2 ] == '\\' && !pathstr[ 3 ] ) ) - { - buf[ 0 ] = pathstr[ 0 ]; - buf[ 1 ] = ':'; - buf[ 2 ] = '\\'; - buf[ 3 ] = '\0'; - } - else - { - return 0; - } - } - else - { - return 0; - } - - /* We have a root path */ - if ( !GetFileAttributesExA( buf, GetFileExInfoStandard, &fileData ) ) - { - info->is_dir = 0; - info->is_file = 0; - info->exists = 0; - timestamp_clear( &info->time ); - } - else - { - info->is_dir = fileData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY; - info->is_file = !info->is_dir; - info->exists = 1; - timestamp_from_filetime( &info->time, &fileData.ftLastWriteTime ); - } - return 1; -} - -void file_query_( file_info_t * const info ) -{ - char const * const pathstr = object_str( info->name ); - const char * dir; - OBJECT * parent; - file_info_t * parent_info; - - if ( try_file_query_root( info ) ) - return; - - if ( ( dir = strrchr( pathstr, '\\' ) ) ) - { - parent = object_new_range( pathstr, dir - pathstr ); - } - else - { - parent = object_copy( constant_empty ); - } - parent_info = file_query( parent ); - object_free( parent ); - if ( !parent_info || !parent_info->is_dir ) - { - info->is_dir = 0; - info->is_file = 0; - info->exists = 0; - timestamp_clear( &info->time ); - } - else - { - info->is_dir = 0; - info->is_file = 0; - info->exists = 0; - timestamp_clear( &info->time ); - if ( list_empty( parent_info->files ) ) - file_collect_dir_content_( parent_info ); - } -} - - -/* - * file_supported_fmt_resolution() - file modification timestamp resolution - * - * Returns the minimum file modification timestamp resolution supported by this - * Boost Jam implementation. File modification timestamp changes of less than - * the returned value might not be recognized. - * - * Does not take into consideration any OS or file system related restrictions. - * - * Return value 0 indicates that any value supported by the OS is also supported - * here. - */ - -void file_supported_fmt_resolution( timestamp * const t ) -{ - /* On Windows we support nano-second file modification timestamp resolution, - * just the same as the Windows OS itself. - */ - timestamp_init( t, 0, 0 ); -} - - -/* - * file_archscan() - scan an archive for files - */ - -/* Straight from SunOS */ - -#define ARMAG "!<arch>\n" -#define SARMAG 8 - -#define ARFMAG "`\n" - -struct ar_hdr -{ - char ar_name[ 16 ]; - char ar_date[ 12 ]; - char ar_uid[ 6 ]; - char ar_gid[ 6 ]; - char ar_mode[ 8 ]; - char ar_size[ 10 ]; - char ar_fmag[ 2 ]; -}; - -#define SARFMAG 2 -#define SARHDR sizeof( struct ar_hdr ) - -void file_archscan( char const * archive, scanback func, void * closure ) -{ - struct ar_hdr ar_hdr; - char * string_table = 0; - char buf[ MAXJPATH ]; - long offset; - int const fd = open( archive, O_RDONLY | O_BINARY, 0 ); - - if ( fd < 0 ) - return; - - if ( read( fd, buf, SARMAG ) != SARMAG || strncmp( ARMAG, buf, SARMAG ) ) - { - close( fd ); - return; - } - - offset = SARMAG; - - if ( DEBUG_BINDSCAN ) - printf( "scan archive %s\n", archive ); - - while ( ( read( fd, &ar_hdr, SARHDR ) == SARHDR ) && - !memcmp( ar_hdr.ar_fmag, ARFMAG, SARFMAG ) ) - { - long lar_date; - long lar_size; - char * name = 0; - char * endname; - - sscanf( ar_hdr.ar_date, "%ld", &lar_date ); - sscanf( ar_hdr.ar_size, "%ld", &lar_size ); - - lar_size = ( lar_size + 1 ) & ~1; - - if ( ar_hdr.ar_name[ 0 ] == '/' && ar_hdr.ar_name[ 1 ] == '/' ) - { - /* This is the "string table" entry of the symbol table, holding - * filename strings longer than 15 characters, i.e. those that do - * not fit into ar_name. - */ - string_table = BJAM_MALLOC_ATOMIC( lar_size + 1 ); - if ( read( fd, string_table, lar_size ) != lar_size ) - printf( "error reading string table\n" ); - string_table[ lar_size ] = '\0'; - offset += SARHDR + lar_size; - continue; - } - else if ( ar_hdr.ar_name[ 0 ] == '/' && ar_hdr.ar_name[ 1 ] != ' ' ) - { - /* Long filenames are recognized by "/nnnn" where nnnn is the - * string's offset in the string table represented in ASCII - * decimals. - */ - name = string_table + atoi( ar_hdr.ar_name + 1 ); - for ( endname = name; *endname && *endname != '\n'; ++endname ); - } - else - { - /* normal name */ - name = ar_hdr.ar_name; - endname = name + sizeof( ar_hdr.ar_name ); - } - - /* strip trailing white-space, slashes, and backslashes */ - - while ( endname-- > name ) - if ( !isspace( *endname ) && ( *endname != '\\' ) && ( *endname != - '/' ) ) - break; - *++endname = 0; - - /* strip leading directory names, an NT specialty */ - { - char * c; - if ( c = strrchr( name, '/' ) ) - name = c + 1; - if ( c = strrchr( name, '\\' ) ) - name = c + 1; - } - - sprintf( buf, "%s(%.*s)", archive, endname - name, name ); - { - OBJECT * const member = object_new( buf ); - timestamp time; - timestamp_init( &time, (time_t)lar_date, 0 ); - (*func)( closure, member, 1 /* time valid */, &time ); - object_free( member ); - } - - offset += SARHDR + lar_size; - lseek( fd, offset, 0 ); - } - - close( fd ); -} - -#endif /* OS_NT */ http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/filesys.c ---------------------------------------------------------------------- diff --git a/ext/kenlm b/ext/kenlm new file mode 160000 index 0000000..56fdb5c --- /dev/null +++ b/ext/kenlm @@ -0,0 +1 @@ +Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5 diff --git a/ext/kenlm/jam-files/engine/filesys.c b/ext/kenlm/jam-files/engine/filesys.c deleted file mode 100644 index dadaef8..0000000 --- a/ext/kenlm/jam-files/engine/filesys.c +++ /dev/null @@ -1,326 +0,0 @@ -/* - * Copyright 2001-2004 David Abrahams. - * Copyright 2005 Rene Rivera. - * Distributed under the Boost Software License, Version 1.0. - * (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt) - */ - -/* - * filesys.c - OS independant file system manipulation support - * - * External routines: - * file_build1() - construct a path string based on PATHNAME information - * file_dirscan() - scan a directory for files - * file_done() - module cleanup called on shutdown - * file_info() - return cached information about a path - * file_is_file() - return whether a path identifies an existing file - * file_query() - get cached information about a path, query the OS if - * needed - * file_remove_atexit() - schedule a path to be removed on program exit - * file_time() - get a file timestamp - * - * External routines - utilites for OS specific module implementations: - * file_query_posix_() - query information about a path using POSIX stat() - * - * Internal routines: - * file_dirscan_impl() - no-profiling worker for file_dirscan() - */ - - -#include "jam.h" -#include "filesys.h" - -#include "lists.h" -#include "object.h" -#include "pathsys.h" -#include "strings.h" - -#include <assert.h> -#include <sys/stat.h> - - -/* Internal OS specific implementation details - have names ending with an - * underscore and are expected to be implemented in an OS specific fileXXX.c - * module. - */ -void file_dirscan_( file_info_t * const dir, scanback func, void * closure ); -int file_collect_dir_content_( file_info_t * const dir ); -void file_query_( file_info_t * const ); - -static void file_dirscan_impl( OBJECT * dir, scanback func, void * closure ); -static void free_file_info( void * xfile, void * data ); -static void remove_files_atexit( void ); - - -static struct hash * filecache_hash; - - -/* - * file_build1() - construct a path string based on PATHNAME information - */ - -void file_build1( PATHNAME * const f, string * file ) -{ - if ( DEBUG_SEARCH ) - { - printf( "build file: " ); - if ( f->f_root.len ) - printf( "root = '%.*s' ", f->f_root.len, f->f_root.ptr ); - if ( f->f_dir.len ) - printf( "dir = '%.*s' ", f->f_dir.len, f->f_dir.ptr ); - if ( f->f_base.len ) - printf( "base = '%.*s' ", f->f_base.len, f->f_base.ptr ); - printf( "\n" ); - } - - /* Start with the grist. If the current grist is not surrounded by <>'s, add - * them. - */ - if ( f->f_grist.len ) - { - if ( f->f_grist.ptr[ 0 ] != '<' ) - string_push_back( file, '<' ); - string_append_range( - file, f->f_grist.ptr, f->f_grist.ptr + f->f_grist.len ); - if ( file->value[ file->size - 1 ] != '>' ) - string_push_back( file, '>' ); - } -} - - -/* - * file_dirscan() - scan a directory for files - */ - -void file_dirscan( OBJECT * dir, scanback func, void * closure ) -{ - PROFILE_ENTER( FILE_DIRSCAN ); - file_dirscan_impl( dir, func, closure ); - PROFILE_EXIT( FILE_DIRSCAN ); -} - - -/* - * file_done() - module cleanup called on shutdown - */ - -void file_done() -{ - remove_files_atexit(); - if ( filecache_hash ) - { - hashenumerate( filecache_hash, free_file_info, (void *)0 ); - hashdone( filecache_hash ); - } -} - - -/* - * file_info() - return cached information about a path - * - * Returns a default initialized structure containing only the path's normalized - * name in case this is the first time this file system entity has been - * referenced. - */ - -file_info_t * file_info( OBJECT * const path, int * found ) -{ - OBJECT * const path_key = path_as_key( path ); - file_info_t * finfo; - - if ( !filecache_hash ) - filecache_hash = hashinit( sizeof( file_info_t ), "file_info" ); - - finfo = (file_info_t *)hash_insert( filecache_hash, path_key, found ); - if ( !*found ) - { - finfo->name = path_key; - finfo->files = L0; - } - else - object_free( path_key ); - - return finfo; -} - - -/* - * file_is_file() - return whether a path identifies an existing file - */ - -int file_is_file( OBJECT * const path ) -{ - file_info_t const * const ff = file_query( path ); - return ff ? ff->is_file : -1; -} - - -/* - * file_time() - get a file timestamp - */ - -int file_time( OBJECT * const path, timestamp * const time ) -{ - file_info_t const * const ff = file_query( path ); - if ( !ff ) return -1; - timestamp_copy( time, &ff->time ); - return 0; -} - - -/* - * file_query() - get cached information about a path, query the OS if needed - * - * Returns 0 in case querying the OS about the given path fails, e.g. because - * the path does not reference an existing file system object. - */ - -file_info_t * file_query( OBJECT * const path ) -{ - /* FIXME: Add tracking for disappearing files (i.e. those that can not be - * detected by stat() even though they had been detected successfully - * before) and see how they should be handled in the rest of Boost Jam code. - * Possibly allow Jamfiles to specify some files as 'volatile' which would - * make Boost Jam avoid caching information about those files and instead - * ask the OS about them every time. - */ - int found; - file_info_t * const ff = file_info( path, &found ); - if ( !found ) - { - file_query_( ff ); - if ( ff->exists ) - { - /* Set the path's timestamp to 1 in case it is 0 or undetected to avoid - * confusion with non-existing paths. - */ - if ( timestamp_empty( &ff->time ) ) - timestamp_init( &ff->time, 1, 0 ); - } - } - if ( !ff->exists ) - { - return 0; - } - return ff; -} - - -/* - * file_query_posix_() - query information about a path using POSIX stat() - * - * Fallback file_query_() implementation for OS specific modules. - * - * Note that the Windows POSIX stat() function implementation suffers from - * several issues: - * * Does not support file timestamps with resolution finer than 1 second, - * meaning it can not be used to detect file timestamp changes of less than - * 1 second. One possible consequence is that some fast-paced touch commands - * (such as those done by Boost Build's internal testing system if it does - * not do some extra waiting) will not be detected correctly by the build - * system. - * * Returns file modification times automatically adjusted for daylight - * savings time even though daylight savings time should have nothing to do - * with internal time representation. - */ - -void file_query_posix_( file_info_t * const info ) -{ - struct stat statbuf; - char const * const pathstr = object_str( info->name ); - char const * const pathspec = *pathstr ? pathstr : "."; - - if ( stat( pathspec, &statbuf ) < 0 ) - { - info->is_file = 0; - info->is_dir = 0; - info->exists = 0; - timestamp_clear( &info->time ); - } - else - { - info->is_file = statbuf.st_mode & S_IFREG ? 1 : 0; - info->is_dir = statbuf.st_mode & S_IFDIR ? 1 : 0; - info->exists = 1; - timestamp_init( &info->time, statbuf.st_mtime, 0 ); - } -} - - -/* - * file_remove_atexit() - schedule a path to be removed on program exit - */ - -static LIST * files_to_remove = L0; - -void file_remove_atexit( OBJECT * const path ) -{ - files_to_remove = list_push_back( files_to_remove, object_copy( path ) ); -} - - -/* - * file_dirscan_impl() - no-profiling worker for file_dirscan() - */ - -static void file_dirscan_impl( OBJECT * dir, scanback func, void * closure ) -{ - file_info_t * const d = file_query( dir ); - if ( !d || !d->is_dir ) - return; - - /* Lazy collect the directory content information. */ - if ( list_empty( d->files ) ) - { - if ( DEBUG_BINDSCAN ) - printf( "scan directory %s\n", object_str( d->name ) ); - if ( file_collect_dir_content_( d ) < 0 ) - return; - } - - /* OS specific part of the file_dirscan operation. */ - file_dirscan_( d, func, closure ); - - /* Report the collected directory content. */ - { - LISTITER iter = list_begin( d->files ); - LISTITER const end = list_end( d->files ); - for ( ; iter != end; iter = list_next( iter ) ) - { - OBJECT * const path = list_item( iter ); - file_info_t const * const ffq = file_query( path ); - /* Using a file name read from a file_info_t structure allows OS - * specific implementations to store some kind of a normalized file - * name there. Using such a normalized file name then allows us to - * correctly recognize different file paths actually identifying the - * same file. For instance, an implementation may: - * - convert all file names internally to lower case on a case - * insensitive file system - * - convert the NTFS paths to their long path variants as that - * file system each file system entity may have a long and a - * short path variant thus allowing for many different path - * strings identifying the same file. - */ - (*func)( closure, ffq->name, 1 /* stat()'ed */, &ffq->time ); - } - } -} - - -static void free_file_info( void * xfile, void * data ) -{ - file_info_t * const file = (file_info_t *)xfile; - object_free( file->name ); - list_free( file->files ); -} - - -static void remove_files_atexit( void ) -{ - LISTITER iter = list_begin( files_to_remove ); - LISTITER const end = list_end( files_to_remove ); - for ( ; iter != end; iter = list_next( iter ) ) - remove( object_str( list_item( iter ) ) ); - list_free( files_to_remove ); - files_to_remove = L0; -} http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/filesys.h ---------------------------------------------------------------------- diff --git a/ext/kenlm b/ext/kenlm new file mode 160000 index 0000000..56fdb5c --- /dev/null +++ b/ext/kenlm @@ -0,0 +1 @@ +Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5 diff --git a/ext/kenlm/jam-files/engine/filesys.h b/ext/kenlm/jam-files/engine/filesys.h deleted file mode 100644 index 74fa395..0000000 --- a/ext/kenlm/jam-files/engine/filesys.h +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright 1993-2002 Christopher Seiwald and Perforce Software, Inc. - * - * This file is part of Jam - see jam.c for Copyright information. - */ - -/* This file is ALSO: - * Copyright 2001-2004 David Abrahams. - * Distributed under the Boost Software License, Version 1.0. - * (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt) - */ - -/* - * filesys.h - OS specific file routines - */ - -#ifndef FILESYS_DWA20011025_H -#define FILESYS_DWA20011025_H - -#include "hash.h" -#include "lists.h" -#include "object.h" -#include "pathsys.h" -#include "timestamp.h" - - -typedef struct file_info_t -{ - OBJECT * name; - char is_file; - char is_dir; - char exists; - timestamp time; - LIST * files; -} file_info_t; - -typedef void (*scanback)( void * closure, OBJECT * path, int found, - timestamp const * const ); - - -void file_archscan( char const * arch, scanback func, void * closure ); -void file_build1( PATHNAME * const f, string * file ) ; -void file_dirscan( OBJECT * dir, scanback func, void * closure ); -file_info_t * file_info( OBJECT * const path, int * found ); -int file_is_file( OBJECT * const path ); -int file_mkdir( char const * const path ); -file_info_t * file_query( OBJECT * const path ); -void file_remove_atexit( OBJECT * const path ); -void file_supported_fmt_resolution( timestamp * const ); -int file_time( OBJECT * const path, timestamp * const ); - -/* Internal utility worker functions. */ -void file_query_posix_( file_info_t * const ); - -void file_done(); - -#endif
