Revision: 5593
http://jnode.svn.sourceforge.net/jnode/?rev=5593&view=rev
Author: crawley
Date: 2009-07-06 14:33:04 +0000 (Mon, 06 Jul 2009)
Log Message:
-----------
Change the shell / interpreter interactions to handle multi-line input
more cleanly. This will allow some bjorne features to be implemented.
Modified Paths:
--------------
trunk/shell/src/shell/org/jnode/shell/CommandInterpreter.java
trunk/shell/src/shell/org/jnode/shell/CommandShell.java
trunk/shell/src/shell/org/jnode/shell/DefaultInterpreter.java
trunk/shell/src/shell/org/jnode/shell/RedirectingInterpreter.java
trunk/shell/src/shell/org/jnode/shell/bjorne/BjorneContext.java
trunk/shell/src/shell/org/jnode/shell/bjorne/BjorneInterpreter.java
trunk/shell/src/shell/org/jnode/shell/bjorne/BjorneParser.java
trunk/shell/src/shell/org/jnode/shell/bjorne/BjorneTokenizer.java
trunk/shell/src/shell/org/jnode/shell/bjorne/SetBuiltin.java
trunk/shell/src/shell/org/jnode/shell/bjorne/SourceBuiltin.java
trunk/shell/src/test/org/jnode/test/shell/bjorne/BjorneParserTest.java
trunk/shell/src/test/org/jnode/test/shell/bjorne/BjorneTokenizerTest.java
Modified: trunk/shell/src/shell/org/jnode/shell/CommandInterpreter.java
===================================================================
--- trunk/shell/src/shell/org/jnode/shell/CommandInterpreter.java
2009-07-05 06:40:00 UTC (rev 5592)
+++ trunk/shell/src/shell/org/jnode/shell/CommandInterpreter.java
2009-07-06 14:33:04 UTC (rev 5593)
@@ -41,24 +41,14 @@
}
/**
- * Parse and execute a command line, and return the resulting return code.
+ * Parse and execute commands read from a reader, returning the resulting
return code.
*
* @param shell the CommandShell that provides low-level command
invocation,
* command history and so on.
- * @param line the line of input to be interpreted.
- * @return the return code.
- * @throws ShellException
- */
- int interpret(CommandShell shell, String line) throws ShellException;
-
- /**
- * Parse and execute a command file, returning the resulting return code.
- *
- * @param shell the CommandShell that provides low-level command
invocation,
- * command history and so on.
* @param reader the reader to be interpreted. <b>The implementation must
close it.</b>
- * @param alias this will supply the script's notional command name to the
interpreter.
- * @param args command line arguments to be passed to the script. If this
parameter
+ * @param alias this will supply a script's notional command name to the
interpreter. If
+ * this parameter is {...@code null}, no command name passed.
+ * @param args optional command line arguments to be passed to the script.
If this parameter
* is {...@code null}, no arguments are passed.
* @return the return code.
* @throws ShellException
Modified: trunk/shell/src/shell/org/jnode/shell/CommandShell.java
===================================================================
--- trunk/shell/src/shell/org/jnode/shell/CommandShell.java 2009-07-05
06:40:00 UTC (rev 5592)
+++ trunk/shell/src/shell/org/jnode/shell/CommandShell.java 2009-07-06
14:33:04 UTC (rev 5593)
@@ -379,44 +379,10 @@
readingCommand = true;
input = readInputLine();
if (input.length() > 0) {
- // This hairy bit of code deals with shell commands that
span multiple
- // input lines. If an interpreter encounters the end of
the line that
- // we have given it and it requires more input to get a
complete command,
- // it may throw IncompleteCommandException. The shell
responds by
- // outputting a different prompt (supplied in the
exception), and then
- // attempting to get the next line. If that succeeds, the
line is
- // appended to the input we gave the interpreter last
time, and the
- // interpreter is called again. This continues until
either the
- // interpreter manages to run the command, or we get some
other
- // shell syntax exception.
- boolean done = false;
- do {
- try {
- runCommand(input, true, this.interpreter);
- done = true;
- } catch (IncompleteCommandException ex) {
- String continuation = null;
- // (Tell completer to use command history not app.
history)
- readingCommand = true;
- if (this.interpreter.supportsMultilineCommands()) {
- String prompt = ex.getPrompt();
- if (prompt != null) {
- outPW.print(prompt);
- }
- continuation = readInputLine();
- }
- if (continuation == null) {
- diagnose(ex, null);
- break;
- } else {
- input = input + "\n" + continuation;
- }
- } catch (ShellException ex) {
- diagnose(ex, null);
- done = true;
- }
- } while (!done);
+ runCommand(input, true, this.interpreter);
}
+ } catch (ShellException ex) {
+ diagnose(ex, null);
} catch (VmExit ex) {
// This should only happen if the interpreter wants the shell
to
// exit. The interpreter will typically intercept any VmExits
@@ -427,6 +393,7 @@
+ ex.getMessage());
stackTrace(ex);
} finally {
+ // FIXME ...
if (input != null && input.trim().length() > 0) {
String lines[] = input.split("\\n");
for (String line : lines) {
@@ -564,8 +531,8 @@
}
}
- private int runCommand(String cmdLineStr, boolean interactive,
- CommandInterpreter interpreter) throws ShellException {
+ private int runCommand(String command, boolean interactive,
CommandInterpreter interpreter)
+ throws ShellException {
try {
if (interactive) {
clearEof();
@@ -574,7 +541,7 @@
// for input completion
applicationHistory.set(new InputHistory());
}
- return interpreter.interpret(this, cmdLineStr);
+ return interpreter.interpret(this, new StringReader(command),
null, null);
} finally {
if (interactive) {
applicationHistory.set(null);
Modified: trunk/shell/src/shell/org/jnode/shell/DefaultInterpreter.java
===================================================================
--- trunk/shell/src/shell/org/jnode/shell/DefaultInterpreter.java
2009-07-05 06:40:00 UTC (rev 5592)
+++ trunk/shell/src/shell/org/jnode/shell/DefaultInterpreter.java
2009-07-06 14:33:04 UTC (rev 5593)
@@ -89,20 +89,11 @@
public String getName() {
return "default";
}
-
- @Override
- public int interpret(CommandShell shell, String line) throws
ShellException {
- CommandLine cmd = doParseCommandLine(line);
- if (cmd == null) {
- return 0;
- }
- return shell.invoke(cmd, null, null);
- }
-
+
/**
* {...@inheritdoc}
*
- * The default interpreter and its subtypes treat a command script as a
sequence of commands.
+ * The default interpreter treats a command script as a sequence of
commands.
* Commands are expected to consist of exactly one line. Any line whose
first non-whitespace
* character is '#' will be ignored. Command line arguments from the
script are not supported,
* and will result in a {...@link ShellException} being thrown.
@@ -167,6 +158,15 @@
return false;
}
+ protected int interpret(CommandShell shell, String line)
+ throws ShellException {
+ CommandLine cmd = doParseCommandLine(line);
+ if (cmd == null) {
+ return 0;
+ }
+ return shell.invoke(cmd, null, null);
+ }
+
private CommandLine doParseCommandLine(String line) throws ShellException {
Tokenizer tokenizer = new Tokenizer(line);
if (!tokenizer.hasNext()) {
Modified: trunk/shell/src/shell/org/jnode/shell/RedirectingInterpreter.java
===================================================================
--- trunk/shell/src/shell/org/jnode/shell/RedirectingInterpreter.java
2009-07-05 06:40:00 UTC (rev 5592)
+++ trunk/shell/src/shell/org/jnode/shell/RedirectingInterpreter.java
2009-07-06 14:33:04 UTC (rev 5593)
@@ -68,7 +68,7 @@
}
@Override
- public int interpret(CommandShell shell, String line) throws
ShellException {
+ protected int interpret(CommandShell shell, String line) throws
ShellException {
Tokenizer tokenizer = new Tokenizer(line, REDIRECTS_FLAG);
List<CommandDescriptor> commands = new LinkedList<CommandDescriptor>();
parse(tokenizer, commands, false);
Modified: trunk/shell/src/shell/org/jnode/shell/bjorne/BjorneContext.java
===================================================================
--- trunk/shell/src/shell/org/jnode/shell/bjorne/BjorneContext.java
2009-07-05 06:40:00 UTC (rev 5592)
+++ trunk/shell/src/shell/org/jnode/shell/bjorne/BjorneContext.java
2009-07-06 14:33:04 UTC (rev 5593)
@@ -53,7 +53,6 @@
import org.jnode.shell.Command;
import org.jnode.shell.CommandLine;
import org.jnode.shell.CommandShell;
-import org.jnode.shell.IncompleteCommandException;
import org.jnode.shell.PathnamePattern;
import org.jnode.shell.ShellException;
import org.jnode.shell.ShellFailureException;
@@ -595,7 +594,7 @@
private StringBuffer runBacktickCommand(String commandLine) throws
ShellException {
StringWriter capture = new StringWriter();
- interpreter.interpret(interpreter.getShell(), commandLine, capture,
false);
+ interpreter.interpret(interpreter.getShell(), new
StringReader(commandLine), capture, false);
StringBuffer output = capture.getBuffer();
while (output.length() > 0 && output.charAt(output.length() - 1) ==
'\n') {
output.setLength(output.length() - 1);
@@ -1300,8 +1299,7 @@
return interpreter.getUniqueName();
}
- public BjorneToken[] substituteAliases(BjorneToken[] words)
- throws IncompleteCommandException {
+ public BjorneToken[] substituteAliases(BjorneToken[] words) throws
ShellSyntaxException {
String alias = aliases.get(words[0].getText());
if (alias == null) {
return words;
@@ -1311,8 +1309,7 @@
return list.toArray(new BjorneToken[list.size()]);
}
- private void substituteAliases(List<BjorneToken> list, int pos, int depth)
- throws IncompleteCommandException {
+ private void substituteAliases(List<BjorneToken> list, int pos, int depth)
throws ShellSyntaxException {
if (depth > 10) {
throw new ShellFailureException("probable cycle detected in alias
expansion");
}
Modified: trunk/shell/src/shell/org/jnode/shell/bjorne/BjorneInterpreter.java
===================================================================
--- trunk/shell/src/shell/org/jnode/shell/bjorne/BjorneInterpreter.java
2009-07-05 06:40:00 UTC (rev 5592)
+++ trunk/shell/src/shell/org/jnode/shell/bjorne/BjorneInterpreter.java
2009-07-06 14:33:04 UTC (rev 5593)
@@ -30,7 +30,6 @@
import static org.jnode.shell.bjorne.BjorneToken.TOK_LESSAND;
import static org.jnode.shell.bjorne.BjorneToken.TOK_LESSGREAT;
-import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.PrintStream;
@@ -45,10 +44,8 @@
import org.jnode.shell.CommandLine;
import org.jnode.shell.CommandShell;
import org.jnode.shell.Completable;
-import org.jnode.shell.IncompleteCommandException;
import org.jnode.shell.ShellException;
import org.jnode.shell.ShellFailureException;
-import org.jnode.shell.ShellInvocationException;
import org.jnode.shell.ShellSyntaxException;
import org.jnode.shell.io.CommandIO;
import org.jnode.shell.io.CommandOutput;
@@ -164,10 +161,10 @@
return "bjorne";
}
- @Override
- public int interpret(CommandShell shell, String command) throws
ShellException {
+ public int interpret(CommandShell shell, Reader reader)
+ throws ShellException {
try {
- return interpret(shell, command, null, false);
+ return interpret(shell, reader, null, false);
} catch (BjorneControlException ex) {
switch (ex.getControl()) {
case BjorneInterpreter.BRANCH_EXIT:
@@ -194,7 +191,7 @@
BjorneTokenizer tokens = new BjorneTokenizer(partial);
BjorneCompleter completions = new BjorneCompleter(context);
try {
- new BjorneParser(tokens, "> ").parse(completions);
+ new BjorneParser(tokens).parse(completions);
} catch (ShellSyntaxException ex) {
// squelch both syntax and incomplete command exceptions.
}
@@ -217,7 +214,7 @@
}
}
- int interpret(CommandShell shell, String command, StringWriter capture,
boolean source)
+ int interpret(CommandShell shell, Reader reader, StringWriter capture,
boolean source)
throws ShellException {
BjorneContext myContext;
// FIXME ... I think there is something wrong / incomplete with the
way I'm handling
@@ -229,8 +226,8 @@
myContext = new BjorneContext(this);
myContext.setIO(1, new CommandOutput(capture), true);
}
- BjorneTokenizer tokens = new BjorneTokenizer(command);
- CommandNode tree = new BjorneParser(tokens, "> ").parse();
+ BjorneTokenizer tokens = new BjorneTokenizer(reader);
+ CommandNode tree = new BjorneParser(tokens).parse();
if (tree == null) {
// An empty command line
return myContext.getLastReturnCode();
@@ -252,44 +249,27 @@
context.setCommand(alias == null ? "" : alias);
context.setArgs(args == null ? new String[0] : args);
try {
- BufferedReader br = new BufferedReader(reader);
- String line;
- int rc = 0;
- while ((line = br.readLine()) != null) {
- boolean done = false;
- do {
- try {
- rc = interpret(shell, line, null, false);
- done = true;
- } catch (BjorneControlException ex) {
- switch (ex.getControl()) {
- case BjorneInterpreter.BRANCH_EXIT:
- // The script will exit immediately
- return ex.getCount();
- case BjorneInterpreter.BRANCH_BREAK:
- throw new ShellSyntaxException(
- "'break' has been executed in an
inappropriate context");
- case BjorneInterpreter.BRANCH_CONTINUE:
- throw new ShellSyntaxException(
- "'continue' has been executed in an
inappropriate context");
- case BjorneInterpreter.BRANCH_RETURN:
- throw new ShellSyntaxException(
- "'return' has been executed in an
inappropriate context");
- }
- } catch (IncompleteCommandException ex) {
- String continuation = br.readLine();
- if (continuation == null) {
- throw ex;
- }
- line = line + "\n" + continuation;
- } catch (VmExit ex) {
- return ex.getStatus();
- }
- } while (!done);
+ return interpret(shell, reader, null, false);
+ } catch (BjorneControlException ex) {
+ switch (ex.getControl()) {
+ case BjorneInterpreter.BRANCH_EXIT:
+ // The script will exit immediately
+ return ex.getCount();
+ case BjorneInterpreter.BRANCH_BREAK:
+ throw new ShellSyntaxException(
+ "'break' has been executed in an inappropriate
context");
+ case BjorneInterpreter.BRANCH_CONTINUE:
+ throw new ShellSyntaxException(
+ "'continue' has been executed in an inappropriate
context");
+ case BjorneInterpreter.BRANCH_RETURN:
+ throw new ShellSyntaxException(
+ "'return' has been executed in an inappropriate
context");
+ default:
+ throw new ShellFailureException(
+ "unknown 'control' in BjorneControlException");
}
- return rc;
- } catch (IOException ex) {
- throw new ShellInvocationException("Problem reading command file:
" + ex.getMessage(), ex);
+ } catch (VmExit ex) {
+ return ex.getStatus();
} finally {
if (reader != null) {
try {
@@ -318,7 +298,8 @@
Properties sysProps, Map<String, String> env, boolean isBuiltin)
throws ShellException {
if (isBuiltin) {
- BjorneBuiltinCommandInfo builtin =
BUILTINS.get(cmdLine.getCommandName()).buildCommandInfo(context);
+ BjorneBuiltinCommandInfo builtin =
+
BUILTINS.get(cmdLine.getCommandName()).buildCommandInfo(context);
cmdLine.setCommandInfo(builtin);
}
cmdLine.setStreams(streams);
Modified: trunk/shell/src/shell/org/jnode/shell/bjorne/BjorneParser.java
===================================================================
--- trunk/shell/src/shell/org/jnode/shell/bjorne/BjorneParser.java
2009-07-05 06:40:00 UTC (rev 5592)
+++ trunk/shell/src/shell/org/jnode/shell/bjorne/BjorneParser.java
2009-07-06 14:33:04 UTC (rev 5593)
@@ -107,7 +107,6 @@
import java.util.LinkedList;
import java.util.List;
-import org.jnode.shell.IncompleteCommandException;
import org.jnode.shell.ShellSyntaxException;
/**
@@ -122,14 +121,11 @@
private final BjorneTokenizer tokens;
private BjorneCompleter completer;
- private final String continuationPrompt;
-
private final List<RedirectionNode> hereRedirections = new
ArrayList<RedirectionNode>();
private boolean allowLineBreaks;
- public BjorneParser(BjorneTokenizer tokens, String continuationPrompt) {
+ public BjorneParser(BjorneTokenizer tokens) {
this.tokens = tokens;
- this.continuationPrompt = continuationPrompt;
}
/**
@@ -143,6 +139,9 @@
public CommandNode parse() throws ShellSyntaxException {
hereRedirections.clear();
List<CommandNode> commands = new LinkedList<CommandNode>();
+ // (The POSIX syntax doesn't seem to allow line breaks at the start,
but I
+ // don't think that can be right ...)
+ skipLineBreaks();
while (peek().getTokenType() != TOK_END_OF_STREAM) {
CommandNode command = parseList();
commands.add(command);
@@ -344,12 +343,6 @@
// FIXME ... built-in commands should use the Syntax
mechanisms so
// that completion, help, etc will work as expected.
}
- } catch (IncompleteCommandException ex) {
- if (completer != null) {
- completer.setCommand(new SimpleCommandNode(CMD_COMMAND,
- words.toArray(new BjorneToken[words.size()]),
builtin));
- }
- throw ex;
} catch (ShellSyntaxException ex) {
if (completer != null) {
completer.setCommand(words.size() == 0 ? null :
@@ -693,21 +686,21 @@
return token;
}
- private BjorneToken next() throws IncompleteCommandException {
+ private BjorneToken next() throws ShellSyntaxException {
if (allowLineBreaks) {
doLineBreaks(0L, false);
}
return tokens.next();
}
- private BjorneToken peek() throws IncompleteCommandException {
+ private BjorneToken peek() throws ShellSyntaxException {
if (allowLineBreaks) {
doLineBreaks(0L, false);
}
return tokens.peek();
}
- private BjorneToken peekEager() throws IncompleteCommandException {
+ private BjorneToken peekEager() throws ShellSyntaxException {
if (allowLineBreaks) {
doLineBreaks(0L, true);
}
@@ -721,9 +714,8 @@
if (((1L << tt) & expectedSet) == 0L) {
if (mandatory) {
if (tt == TOK_END_OF_STREAM) {
- throw new IncompleteCommandException(
- "EOF reached while looking for " +
BjorneToken.formatExpectedSet(expectedSet),
- continuationPrompt);
+ throw new ShellSyntaxException(
+ "EOF reached while looking for " +
BjorneToken.formatExpectedSet(expectedSet));
} else {
throw new ShellSyntaxException(
"expected " +
BjorneToken.formatExpectedSet(expectedSet) + " but got " + token);
@@ -765,16 +757,16 @@
}
}
- private void skipLineBreaks() throws IncompleteCommandException {
+ private void skipLineBreaks() throws ShellSyntaxException {
this.allowLineBreaks = true;
doLineBreaks(0L, false);
}
- private void allowLineBreaks() throws IncompleteCommandException {
+ private void allowLineBreaks() throws ShellSyntaxException {
this.allowLineBreaks = true;
}
- private void doLineBreaks(long expectedSet, boolean needMore) throws
IncompleteCommandException {
+ private void doLineBreaks(long expectedSet, boolean needMore) throws
ShellSyntaxException {
// NB: use tokens.peek() / next() rather than the wrappers here!!
this.allowLineBreaks = false;
BjorneToken token = tokens.peek();
@@ -782,9 +774,8 @@
if (tt == TOK_END_OF_STREAM) {
captureCompletions(token, expectedSet);
if (needMore) {
- throw new IncompleteCommandException(
- "EOF reached while looking for optional linebreak(s)",
- continuationPrompt);
+ throw new ShellSyntaxException(
+ "EOF reached while looking for optional linebreak(s)");
}
} else if (tt == TOK_END_OF_LINE) {
tokens.next();
@@ -797,9 +788,8 @@
} else if (tt == TOK_END_OF_STREAM) {
captureCompletions(token, expectedSet);
if (needMore) {
- throw new IncompleteCommandException(
- "EOF reached while dealing with optional
linebreak(s)",
- continuationPrompt);
+ throw new ShellSyntaxException(
+ "EOF reached while dealing with optional
linebreak(s)");
} else {
break;
}
@@ -810,7 +800,7 @@
}
}
- private void captureHereDocuments() throws IncompleteCommandException {
+ private void captureHereDocuments() throws ShellSyntaxException {
for (RedirectionNode redirection : hereRedirections) {
StringBuilder sb = new StringBuilder();
String marker = redirection.getArg().getText();
@@ -818,8 +808,8 @@
while (true) {
String line = tokens.readHereLine(trimTabs);
if (line == null) {
- throw new IncompleteCommandException("EOF reached while
looking for '" +
- marker + "' to end a HERE document",
continuationPrompt);
+ throw new ShellSyntaxException("EOF reached while looking
for '" +
+ marker + "' to end a HERE document");
}
if (line.equals(marker)) {
break;
Modified: trunk/shell/src/shell/org/jnode/shell/bjorne/BjorneTokenizer.java
===================================================================
--- trunk/shell/src/shell/org/jnode/shell/bjorne/BjorneTokenizer.java
2009-07-05 06:40:00 UTC (rev 5592)
+++ trunk/shell/src/shell/org/jnode/shell/bjorne/BjorneTokenizer.java
2009-07-06 14:33:04 UTC (rev 5593)
@@ -66,92 +66,58 @@
import static org.jnode.shell.bjorne.BjorneToken.TOK_WHILE;
import static org.jnode.shell.bjorne.BjorneToken.TOK_WORD;
-import org.jnode.shell.IncompleteCommandException;
+import java.io.IOException;
+import java.io.Reader;
+import java.io.StringReader;
+
import org.jnode.shell.ShellFailureException;
+import org.jnode.shell.ShellSyntaxException;
public class BjorneTokenizer {
- private final char[] chars;
+ private final Reader reader;
- private final int len;
-
- private int pos;
-
private BjorneToken prev, current, next;
-
private static final int EOS = -1;
+ private static final int INVALID = -2;
+
+ private int pos = 0;
+ private int lastCh = INVALID;
+ private int nextCh = INVALID;
private final boolean debug;
/**
* Create a tokenizer for the supplied shell input text.
* @param text the text to be tokenized
- * @throws IncompleteCommandException if the text ends with a line
continuation
+ * @throws ShellSyntaxException
*/
- public BjorneTokenizer(String text) throws IncompleteCommandException {
- this(text, false);
+ public BjorneTokenizer(String text)
+ throws ShellSyntaxException {
+ this(new StringReader(text), false);
}
/**
- * Create a tokenizer for the supplied shell input text.
- * @param text the text to be tokenized
- * @param debug if {...@code true}, produce debug output
- * @throws IncompleteCommandException if the text ends with a line
continuation
+ * Create a tokenizer for the supplied shell input reader.
+ * @param reader the reader to be tokenized.
+ * @throws ShellSyntaxException
*/
- public BjorneTokenizer(String text, boolean debug) throws
IncompleteCommandException {
- chars = foldContinuations(text);
- len = chars.length;
- this.debug = debug;
+ public BjorneTokenizer(Reader reader)
+ throws ShellSyntaxException {
+ this(reader, false);
}
/**
- * Rewrite the supplied text to fold any line continuations.
- *
- * @param text the text to be processed
- * @return the characters of text with any line continuations removed.
- * @throws IncompleteCommandException
+ * Create a tokenizer for the supplied shell input text.
+ * @param reader the reader to be tokenized.
+ * @param debug if {...@code true}, produce debug output
+ * @throws ShellSyntaxException
*/
- private char[] foldContinuations(String text) throws
IncompleteCommandException {
- // FIXME this is wrong ... if we are going to imitate the documented
behaviour
- // of bash. (In bash, if the the MARKER is quoted, '\<newline>' is
apparently
- // not interpreted as a continuation.)
- if (text.indexOf('\\') == -1) {
- return text.toCharArray();
- }
- int len = text.length();
- StringBuilder sb = new StringBuilder(len);
- boolean escape = false;
- for (int i = 0; i < len; i++) {
- char ch = text.charAt(i);
- switch (ch) {
- case '\\':
- if (escape) {
- sb.append('\\');
- } else if (i == len - 1) {
- // If we get a continuation sequence at the end of the
- // text, the simplest thing is to ask for more input.
- throw new IncompleteCommandException(
- "More input required after '\\<newline>'", " >
");
- }
- escape = !escape;
- break;
- case '\n':
- if (!escape) {
- sb.append('\n');
- } else {
- escape = false;
- }
- break;
- default:
- if (escape) {
- sb.append('\\');
- escape = false;
- }
- sb.append(ch);
- }
- }
- return sb.toString().toCharArray();
+ public BjorneTokenizer(Reader reader, boolean debug)
+ throws ShellSyntaxException {
+ this.reader = reader;
+ this.debug = debug;
}
/**
@@ -180,7 +146,7 @@
public BjorneToken peek(int context) {
BjorneToken res = reinterpret(peek(), context);
if (debug) {
- System.err.println("--> " + res);
+ System.err.println("peek(" + context + ") --> " + res);
}
return res;
}
@@ -240,7 +206,7 @@
public BjorneToken next(int context) {
BjorneToken res = reinterpret(next(), context);
if (debug) {
- System.err.println("--> " + res);
+ System.err.println("next(" + context + ") --> " + res);
}
return res;
}
@@ -290,15 +256,17 @@
System.err.print("advance() ... {" + prev + "," + current + ","
+ next + "} ...");
}
- int ch = nextCh();
+ int ch = peekCh();
while (ch == '\t' || ch == ' ') {
- ch = nextCh();
+ nextCh();
+ ch = peekCh();
}
- int start = pos - 1;
+ int start = getPos() - 1;
switch (ch) {
case EOS:
- return makeToken(TOK_END_OF_STREAM, len);
+ return makeToken(TOK_END_OF_STREAM, getPos());
case '\n':
+ nextCh();
return makeToken(TOK_END_OF_LINE, start);
case '#':
while ((ch = nextCh()) != EOS) {
@@ -308,8 +276,10 @@
}
return makeToken(TOK_END_OF_STREAM, start);
case '(':
+ nextCh();
return makeToken(TOK_LPAREN, start);
case ')':
+ nextCh();
return makeToken(TOK_RPAREN, start);
case '<':
case '>':
@@ -323,18 +293,18 @@
}
private BjorneToken makeToken(int tokenType, int start) {
- return new BjorneToken(tokenType, "", start, pos);
+ return new BjorneToken(tokenType, "", start, getPos());
}
private BjorneToken makeToken(int tokenType, String value, int start) {
- return new BjorneToken(tokenType, value, start, pos);
+ return new BjorneToken(tokenType, value, start, getPos());
}
private BjorneToken parseWord() {
int quoteChar = 0;
StringBuffer sb = new StringBuffer();
- int ch = prevCh();
- int start = pos - 1;
+ int ch = peekCh();
+ int start = getPos() - 1;
LOOP:
while (true) {
switch (ch) {
@@ -364,31 +334,36 @@
}
break;
case '\\':
- ch = nextCh();
+ nextCh();
+ ch = peekCh();
if (ch == '\n') {
- ch = nextCh();
+ // A '\\' followed by a newline is a line continuation:
+ // the two characters are skipped.
+ nextCh();
+ ch = peekCh();
continue;
+ } else if (ch == EOS) {
+ // Silently eat a '\\' at the end of stream position.
+ nextCh();
+ break LOOP;
} else {
+ // The '\\' is included in the (raw) word.
sb.append('\\');
- if (ch == EOS) {
- break LOOP;
- }
}
break;
default:
- /* empty */
+ // include anything else in the word.
break;
}
sb.append((char) ch);
- ch = nextCh();
+ nextCh();
+ ch = peekCh();
}
- if (ch != EOS) {
- backupCh();
- }
if (ch == '<' || ch == '>') {
boolean allDigits = true;
for (int i = 0; i < sb.length(); i++) {
ch = sb.charAt(i);
+ // FIXME ... I should deal with "\\\n" here I think.
if (ch < '0' || ch > '9') {
allDigits = false;
break;
@@ -402,8 +377,8 @@
}
private BjorneToken parseOperator() {
- int start = pos - 1;
- switch (prevCh()) {
+ int start = getPos() - 1;
+ switch (nextCh()) {
case '<':
switch (peekCh()) {
case '<':
@@ -458,26 +433,37 @@
throw new ShellFailureException("bad lexer state");
}
- private int nextCh() {
- return (pos >= len) ? EOS : chars[pos++];
+ private int nextCh() throws ShellFailureException {
+ try {
+ if (nextCh == INVALID) {
+ if (lastCh != EOS) {
+ lastCh = reader.read();
+ pos++;
+ }
+ } else {
+ lastCh = nextCh;
+ nextCh = INVALID;
+ pos++;
+ }
+ return lastCh;
+ } catch (IOException ex) {
+ throw new ShellFailureException("Unexpected exception", ex);
+ }
}
private int peekCh() {
- return (pos >= len) ? EOS : chars[pos];
- }
-
- private int prevCh() {
- if (pos <= 0) {
- throw new ShellFailureException("nextCh not called yet");
+ try {
+ if (nextCh == INVALID) {
+ nextCh = reader.read();
+ }
+ return nextCh;
+ } catch (IOException ex) {
+ throw new ShellFailureException("Unexpected exception", ex);
}
- return chars[pos - 1];
}
-
- private void backupCh() {
- if (pos == 0) {
- throw new ShellFailureException("cannot backup");
- }
- pos--;
+
+ private int getPos() {
+ return pos;
}
/**
Modified: trunk/shell/src/shell/org/jnode/shell/bjorne/SetBuiltin.java
===================================================================
--- trunk/shell/src/shell/org/jnode/shell/bjorne/SetBuiltin.java
2009-07-05 06:40:00 UTC (rev 5592)
+++ trunk/shell/src/shell/org/jnode/shell/bjorne/SetBuiltin.java
2009-07-06 14:33:04 UTC (rev 5593)
@@ -26,8 +26,8 @@
import org.jnode.shell.syntax.OptionalSyntax;
import org.jnode.shell.syntax.PowersetSyntax;
import org.jnode.shell.syntax.RepeatSyntax;
+import org.jnode.shell.syntax.SequenceSyntax;
import org.jnode.shell.syntax.StringArgument;
-import org.jnode.shell.syntax.SequenceSyntax;
import org.jnode.shell.syntax.SyntaxBundle;
import org.jnode.shell.syntax.VerbSyntax;
Modified: trunk/shell/src/shell/org/jnode/shell/bjorne/SourceBuiltin.java
===================================================================
--- trunk/shell/src/shell/org/jnode/shell/bjorne/SourceBuiltin.java
2009-07-05 06:40:00 UTC (rev 5592)
+++ trunk/shell/src/shell/org/jnode/shell/bjorne/SourceBuiltin.java
2009-07-06 14:33:04 UTC (rev 5593)
@@ -59,37 +59,27 @@
@Override
public void execute() throws Exception {
File file = argScript.getValue();
- long size = file.length();
- String commandStr = null;
- FileReader fin = null;
+ FileReader reader = null;
try {
- fin = new FileReader(file);
- if (size > 1000000) {
- // Since we are going to read the whole script into memory, we
- // need to set some limit on the script's file size ...
- getError().getPrintWriter().println("source: " + file + ":
file too big");
- exit(1);
+ reader = new FileReader(file);
+
+ // TODO ... implement args.
+ BjorneContext pc = getParentContext();
+ int rc = pc.getInterpreter().interpret(pc.getShell(), reader, "",
new String[0]);
+ if (rc != 0) {
+ exit(rc);
}
- char[] buffer = new char[(int) size];
- int nosRead = fin.read(buffer);
- commandStr = new String(buffer, 0, nosRead);
} catch (IOException ex) {
getError().getPrintWriter().println("source: " + file + ": " +
ex.getMessage());
exit(1);
} finally {
- if (fin != null) {
+ if (reader != null) {
try {
- fin.close();
+ reader.close();
} catch (IOException ex) {
/* blah */
}
}
}
- // TODO ... implement args.
- BjorneContext pc = getParentContext();
- int rc = pc.getInterpreter().interpret(pc.getShell(), commandStr,
null, true);
- if (rc != 0) {
- exit(rc);
- }
}
}
Modified: trunk/shell/src/test/org/jnode/test/shell/bjorne/BjorneParserTest.java
===================================================================
--- trunk/shell/src/test/org/jnode/test/shell/bjorne/BjorneParserTest.java
2009-07-05 06:40:00 UTC (rev 5592)
+++ trunk/shell/src/test/org/jnode/test/shell/bjorne/BjorneParserTest.java
2009-07-06 14:33:04 UTC (rev 5593)
@@ -20,6 +20,8 @@
package org.jnode.test.shell.bjorne;
+import java.io.StringReader;
+
import junit.framework.TestCase;
import org.jnode.shell.ShellException;
@@ -31,7 +33,7 @@
private static final boolean DEBUG = false;
public void testParser() throws ShellException {
- new BjorneParser(new BjorneTokenizer(""), null);
+ new BjorneParser(new BjorneTokenizer(""));
}
public void test1() throws ShellException {
@@ -147,7 +149,7 @@
}
private String doTest(String input) throws ShellException {
- BjorneParser p = new BjorneParser(new BjorneTokenizer(input, DEBUG),
null);
+ BjorneParser p = new BjorneParser(new BjorneTokenizer(new
StringReader(input), DEBUG));
String res = p.parse().toString();
if (DEBUG) {
System.err.println(res);
Modified:
trunk/shell/src/test/org/jnode/test/shell/bjorne/BjorneTokenizerTest.java
===================================================================
--- trunk/shell/src/test/org/jnode/test/shell/bjorne/BjorneTokenizerTest.java
2009-07-05 06:40:00 UTC (rev 5592)
+++ trunk/shell/src/test/org/jnode/test/shell/bjorne/BjorneTokenizerTest.java
2009-07-06 14:33:04 UTC (rev 5593)
@@ -67,26 +67,17 @@
import static org.jnode.shell.bjorne.BjorneToken.TOK_WORD;
import junit.framework.TestCase;
-import org.jnode.shell.IncompleteCommandException;
+import org.jnode.shell.ShellSyntaxException;
import org.jnode.shell.bjorne.BjorneToken;
import org.jnode.shell.bjorne.BjorneTokenizer;
public class BjorneTokenizerTest extends TestCase {
- public void testBjorneTokenizer() throws IncompleteCommandException {
+ public void testBjorneTokenizer() throws ShellSyntaxException {
new BjorneTokenizer("hello");
}
- public void testBjorneTokenizer2() {
- try {
- new BjorneTokenizer("hello\\");
- fail("no exception");
- } catch (IncompleteCommandException ex) {
- // expected
- }
- }
-
- public void testEmpty() throws IncompleteCommandException {
+ public void testEmpty() throws ShellSyntaxException {
BjorneTokenizer tokenizer = new BjorneTokenizer("");
BjorneToken token = tokenizer.peek();
assertEquals(TOK_END_OF_STREAM, token.getTokenType());
@@ -98,7 +89,7 @@
assertEquals(TOK_END_OF_STREAM, token.getTokenType());
}
- public void testNewline() throws IncompleteCommandException {
+ public void testNewline() throws ShellSyntaxException {
BjorneTokenizer tokenizer = new BjorneTokenizer("\n");
BjorneToken token = tokenizer.next();
assertEquals(TOK_END_OF_LINE, token.getTokenType());
@@ -106,7 +97,7 @@
assertEquals(TOK_END_OF_STREAM, token.getTokenType());
}
- public void testBlanksAndNewlines() throws IncompleteCommandException {
+ public void testBlanksAndNewlines() throws ShellSyntaxException {
BjorneTokenizer tokenizer = new BjorneTokenizer(" \n\t\n ");
BjorneToken token = tokenizer.next();
assertEquals(TOK_END_OF_LINE, token.getTokenType());
@@ -116,7 +107,7 @@
assertEquals(TOK_END_OF_STREAM, token.getTokenType());
}
- public void testComments() throws IncompleteCommandException {
+ public void testComments() throws ShellSyntaxException {
BjorneTokenizer tokenizer = new BjorneTokenizer(
"# comment\n #comment 2\n # comment # 3");
BjorneToken token = tokenizer.next();
@@ -127,7 +118,25 @@
assertEquals(TOK_END_OF_STREAM, token.getTokenType());
}
- public void testSymbols() throws IncompleteCommandException {
+ public void testContinuation() throws ShellSyntaxException {
+ BjorneTokenizer tokenizer = new BjorneTokenizer("hello\\\nthere");
+ BjorneToken token = tokenizer.next();
+ assertEquals(TOK_WORD, token.getTokenType());
+ assertEquals("hellothere", token.getText());
+ token = tokenizer.next();
+ assertEquals(TOK_END_OF_STREAM, token.getTokenType());
+ }
+
+ public void testBackslashAtEnd() throws ShellSyntaxException {
+ BjorneTokenizer tokenizer = new BjorneTokenizer("hello\\");
+ BjorneToken token = tokenizer.next();
+ assertEquals(TOK_WORD, token.getTokenType());
+ assertEquals("hello", token.getText());
+ token = tokenizer.next();
+ assertEquals(TOK_END_OF_STREAM, token.getTokenType());
+ }
+
+ public void testSymbols() throws ShellSyntaxException {
BjorneTokenizer tokenizer = new BjorneTokenizer("; | & < > ( )");
BjorneToken token = tokenizer.next();
assertEquals(TOK_SEMI, token.getTokenType());
@@ -147,9 +156,8 @@
assertEquals(TOK_END_OF_STREAM, token.getTokenType());
}
- public void testSymbols2() throws IncompleteCommandException {
- BjorneTokenizer tokenizer = new BjorneTokenizer(
- "; ;; | || & && < << > >>");
+ public void testSymbols2() throws ShellSyntaxException {
+ BjorneTokenizer tokenizer = new BjorneTokenizer("; ;; | || & && < << >
>>");
BjorneToken token = tokenizer.next();
assertEquals(TOK_SEMI, token.getTokenType());
token = tokenizer.next();
@@ -174,7 +182,7 @@
assertEquals(TOK_END_OF_STREAM, token.getTokenType());
}
- public void testSymbols3() throws IncompleteCommandException {
+ public void testSymbols3() throws ShellSyntaxException {
BjorneTokenizer tokenizer = new BjorneTokenizer(";;;|||&&&<<<>>>");
BjorneToken token = tokenizer.next();
assertEquals(TOK_DSEMI, token.getTokenType());
@@ -198,9 +206,8 @@
assertEquals(TOK_END_OF_STREAM, token.getTokenType());
}
- public void testSymbols4() throws IncompleteCommandException {
- BjorneTokenizer tokenizer = new BjorneTokenizer(
- "< << <<- <& <> > >> >| >&");
+ public void testSymbols4() throws ShellSyntaxException {
+ BjorneTokenizer tokenizer = new BjorneTokenizer("< << <<- <& <> > >>
>| >&");
BjorneToken token = tokenizer.next();
assertEquals(TOK_LESS, token.getTokenType());
token = tokenizer.next();
@@ -223,7 +230,7 @@
assertEquals(TOK_END_OF_STREAM, token.getTokenType());
}
- public void testWords() throws IncompleteCommandException {
+ public void testWords() throws ShellSyntaxException {
BjorneTokenizer tokenizer = new BjorneTokenizer("hello there");
BjorneToken token = tokenizer.next();
assertEquals(TOK_WORD, token.getTokenType());
@@ -235,9 +242,8 @@
assertEquals(TOK_END_OF_STREAM, token.getTokenType());
}
- public void testWords2() throws IncompleteCommandException {
- BjorneTokenizer tokenizer = new BjorneTokenizer(
- "hello\\ there\\\n friend");
+ public void testWords2() throws ShellSyntaxException {
+ BjorneTokenizer tokenizer = new BjorneTokenizer("hello\\ there\\\n
friend");
BjorneToken token = tokenizer.next();
assertEquals(TOK_WORD, token.getTokenType());
assertEquals("hello\\ there", token.getText());
@@ -266,7 +272,7 @@
assertEquals(TOK_END_OF_STREAM, token.getTokenType());
}
- public void testWords3() throws IncompleteCommandException {
+ public void testWords3() throws ShellSyntaxException {
BjorneTokenizer tokenizer = new BjorneTokenizer("'1 2' \"3 4\" `5 6`");
BjorneToken token = tokenizer.next();
assertEquals(TOK_WORD, token.getTokenType());
@@ -281,7 +287,7 @@
assertEquals(TOK_END_OF_STREAM, token.getTokenType());
}
- public void testWords4() throws IncompleteCommandException {
+ public void testWords4() throws ShellSyntaxException {
BjorneTokenizer tokenizer = new BjorneTokenizer("'1 \"2\"'
\"3\\\"4\"");
BjorneToken token = tokenizer.next();
assertEquals(TOK_WORD, token.getTokenType());
@@ -293,7 +299,7 @@
assertEquals(TOK_END_OF_STREAM, token.getTokenType());
}
- public void testWords5() throws IncompleteCommandException {
+ public void testWords5() throws ShellSyntaxException {
BjorneTokenizer tokenizer = new BjorneTokenizer("1<2>3&4;5|6)7");
BjorneToken token = tokenizer.next();
assertEquals(TOK_IO_NUMBER, token.getTokenType());
@@ -332,10 +338,9 @@
assertEquals(TOK_END_OF_STREAM, token.getTokenType());
}
- public void testRule1() throws IncompleteCommandException {
+ public void testRule1() throws ShellSyntaxException {
BjorneTokenizer tokenizer = new BjorneTokenizer(
- "if then else elif fi for done while until "
- + "case { } ! do in esac");
+ "if then else elif fi for done while until case { } ! do in
esac");
BjorneToken token = tokenizer.next(RULE_1_CONTEXT);
assertEquals(TOK_IF, token.getTokenType());
token = tokenizer.next(RULE_1_CONTEXT);
@@ -372,9 +377,8 @@
assertEquals(TOK_END_OF_STREAM, token.getTokenType());
}
- public void testRule5() throws IncompleteCommandException {
- BjorneTokenizer tokenizer = new BjorneTokenizer(
- "if a a1 9a a_b a,b AB A=b");
+ public void testRule5() throws ShellSyntaxException {
+ BjorneTokenizer tokenizer = new BjorneTokenizer("if a a1 9a a_b a,b AB
A=b");
BjorneToken token = tokenizer.next(RULE_5_CONTEXT);
assertEquals(TOK_NAME, token.getTokenType());
token = tokenizer.next(RULE_5_CONTEXT);
@@ -395,7 +399,7 @@
assertEquals(TOK_END_OF_STREAM, token.getTokenType());
}
- public void testRule6() throws IncompleteCommandException {
+ public void testRule6() throws ShellSyntaxException {
BjorneTokenizer tokenizer = new BjorneTokenizer("if in do");
BjorneToken token = tokenizer.next(RULE_6_CONTEXT);
assertEquals(TOK_WORD, token.getTokenType());
@@ -407,10 +411,9 @@
assertEquals(TOK_END_OF_STREAM, token.getTokenType());
}
- public void testRule7a() throws IncompleteCommandException {
+ public void testRule7a() throws ShellSyntaxException {
BjorneTokenizer tokenizer = new BjorneTokenizer(
- "if then else elif fi for done while until "
- + "case { } ! do in esac a= a=b 1a=b =c");
+ "if then else elif fi for done while until case { } ! do in
esac a= a=b 1a=b =c");
BjorneToken token = tokenizer.next(RULE_7a_CONTEXT);
assertEquals(TOK_IF, token.getTokenType());
token = tokenizer.next(RULE_7a_CONTEXT);
@@ -455,10 +458,9 @@
assertEquals(TOK_END_OF_STREAM, token.getTokenType());
}
- public void testRule7b() throws IncompleteCommandException {
+ public void testRule7b() throws ShellSyntaxException {
BjorneTokenizer tokenizer = new BjorneTokenizer(
- "if then else elif fi for done while until "
- + "case { } ! do in esac a= a=b 1a=b =c");
+ "if then else elif fi for done while until case { } ! do in
esac a= a=b 1a=b =c");
BjorneToken token = tokenizer.next(RULE_7b_CONTEXT);
assertEquals(TOK_WORD, token.getTokenType());
token = tokenizer.next(RULE_7b_CONTEXT);
@@ -503,10 +505,9 @@
assertEquals(TOK_END_OF_STREAM, token.getTokenType());
}
- public void testRule8() throws IncompleteCommandException {
+ public void testRule8() throws ShellSyntaxException {
BjorneTokenizer tokenizer = new BjorneTokenizer(
- "if then else elif fi for done while until "
- + "case { } ! do in esac a a_b a= a=b 1a=b =c");
+ "if then else elif fi for done while until case { } ! do in
esac a a_b a= a=b 1a=b =c");
BjorneToken token = tokenizer.next(RULE_8_CONTEXT);
assertEquals(TOK_IF, token.getTokenType());
token = tokenizer.next(RULE_8_CONTEXT);
@@ -555,7 +556,7 @@
assertEquals(TOK_END_OF_STREAM, token.getTokenType());
}
- public void testRegress() throws IncompleteCommandException {
+ public void testRegress() throws ShellSyntaxException {
BjorneTokenizer tokenizer = new BjorneTokenizer("ls -l");
BjorneToken token = tokenizer.peek(RULE_7a_CONTEXT);
assertEquals(TOK_WORD, token.getTokenType());
This was sent by the SourceForge.net collaborative development platform, the
world's largest Open Source development site.
------------------------------------------------------------------------------
_______________________________________________
Jnode-svn-commits mailing list
Jnode-svn-commits@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/jnode-svn-commits
