This one includes a parser that spits out InventoryItems.
-Chris
On 7/15/07, Christopher Omega <[EMAIL PROTECTED]> wrote:
Hey all,
Just finished something neat - a very rudimentary notecard parser :-)
Right now it just splits the raw text into Scopes, Statements and Text.
Lemme know what you think, I dont have much experience with parsing.
-Chris
--
"Anyone that would give up a little liberty for a little security, deserves
neither and loses both." -Ben Franklin
using System.Collections.Generic;
using System.Text;
using System.Text.RegularExpressions;
using System.IO;
using System;
namespace libsecondlife
{
public class ItemInfo
{
public int EmbeddedItemsParserVersion
{
get { return 1; }
}
private char _TextChar;
public char TextChar
{
get { return _TextChar; }
set { _TextChar = value; }
}
private InventoryItem _Item;
public InventoryItem Item
{
get { return _Item; }
set { _Item = value; }
}
private Logger Log;
private static readonly Dictionary<string, AssetType> AssetTypeMap =
new Dictionary<string, AssetType>();
private static readonly Dictionary<string, InventoryType>
InventoryTypeMap = new Dictionary<string, InventoryType>();
static ItemInfo() {
AssetTypeMap.Add("animatn", AssetType.Animation);
AssetTypeMap.Add("clothing", AssetType.Clothing);
AssetTypeMap.Add("object", AssetType.Object);
AssetTypeMap.Add("texture", AssetType.Texture);
AssetTypeMap.Add("sound", AssetType.Sound);
AssetTypeMap.Add("bodypart", AssetType.Bodypart);
AssetTypeMap.Add("gesture", AssetType.Gesture);
AssetTypeMap.Add("lsltext", AssetType.LSLText);
AssetTypeMap.Add("landmark", AssetType.Landmark);
AssetTypeMap.Add("notecard", AssetType.Notecard);
InventoryTypeMap.Add("sound", InventoryType.Sound);
InventoryTypeMap.Add("wearable", InventoryType.Wearable);
InventoryTypeMap.Add("gesture", InventoryType.Gesture);
InventoryTypeMap.Add("script", InventoryType.LSL);
InventoryTypeMap.Add("texture", InventoryType.Texture);
InventoryTypeMap.Add("landmark", InventoryType.Landmark);
InventoryTypeMap.Add("notecard", InventoryType.Notecard);
InventoryTypeMap.Add("object", InventoryType.Object);
InventoryTypeMap.Add("animation", InventoryType.Animation);
InventoryTypeMap.Add("snapshot", InventoryType.Snapshot);
}
private static string AssetTypeToString(AssetType type)
{
string assetTypeName = null;
foreach (string key in AssetTypeMap.Keys)
{
if (AssetTypeMap[key] == type)
{
assetTypeName = key;
break;
}
}
if (assetTypeName == null)
throw new Exception("Unmapped AssetType: " + type);
return assetTypeName;
}
private static string InventoryTypeToString(InventoryType type)
{
string invTypeName = null;
foreach (string key in InventoryTypeMap.Keys)
{
if (InventoryTypeMap[key] == type)
{
invTypeName = key;
break;
}
}
if (invTypeName == null)
throw new Exception("Unmapped InventoryType: " + type);
return invTypeName;
}
public ItemInfo(Logger log)
{
Log = log;
}
public NotecardScope GenerateScope()
{
NotecardScope outerScope = new NotecardScope();
// TODO: Extchar.
//scope.Children.Add(new NotecardStatement("ext", "char index ??"));
outerScope.Children.Add(new NotecardStatement("inv_item", "0"));
NotecardScope infoScope = new NotecardScope();
infoScope.Children.Add(new NotecardStatement("item_id",
Item.UUID.ToString()));
infoScope.Children.Add(new NotecardStatement("parent_id",
Item.ParentUUID.ToString()));
infoScope.Children.Add(new NotecardStatement("permissions", "0"));
NotecardScope permsScope = new NotecardScope();
permsScope.Children.Add(new NotecardStatement("base_mask",
((int)Item.Permissions.BaseMask).ToString("x"))); // constant length?
permsScope.Children.Add(new NotecardStatement("owner_mask",
((int)Item.Permissions.OwnerMask).ToString("x"))); // constant length?
permsScope.Children.Add(new NotecardStatement("group_mask",
((int)Item.Permissions.GroupMask).ToString("x"))); // constant length?
permsScope.Children.Add(new NotecardStatement("everyone_mask",
((int)Item.Permissions.EveryoneMask).ToString("x"))); // constant length?
permsScope.Children.Add(new NotecardStatement("creator_id",
LLUUID.Zero.ToString())); // FIXME
permsScope.Children.Add(new NotecardStatement("owner_id",
Item.OwnerID.ToString()));
permsScope.Children.Add(new NotecardStatement("last_owner_id",
LLUUID.Zero.ToString())); // FIXME
permsScope.Children.Add(new NotecardStatement("group_id",
Item.GroupID.ToString()));
infoScope.Children.Add(permsScope);
infoScope.Children.Add(new NotecardStatement("asset_id",
Item.AssetUUID.ToString()));
infoScope.Children.Add(new NotecardStatement("type",
AssetTypeToString(Item.AssetType)));
infoScope.Children.Add(new NotecardStatement("inv_type",
InventoryTypeToString(Item.InventoryType)));
infoScope.Children.Add(new NotecardStatement("flags",
Item.Flags.ToString("x"))); // constant length?
infoScope.Children.Add(new NotecardStatement("sale_info", "0"));
NotecardScope saleScope = new NotecardScope();
saleScope.Children.Add(new NotecardStatement("sale_type",
Item.SaleType.ToString().ToLower())); // FIXME
saleScope.Children.Add(new NotecardStatement("sale_price",
Item.SalePrice.ToString()));
saleScope.Children.Add(new NotecardStatement("perm_mask",
((int)Item.Permissions.NextOwnerMask).ToString("x"))); // constant length?
infoScope.Children.Add(saleScope);
infoScope.Children.Add(new NotecardStatement("name", Item.Name +
"|"));
infoScope.Children.Add(new NotecardStatement("desc",
Item.Description + "|"));
infoScope.Children.Add(new NotecardStatement("creation_date",
Helpers.DateTimeToUnixTime(Item.CreationDate).ToString()));
outerScope.Children.Add(infoScope);
return outerScope;
}
public void Parse(NotecardScope scope)
{
NotecardStatement extChar;
scope.GetStatement("ext", out extChar);
// TODO: Use extChar.Parameters[2] to get char.
// TextChar = ??
NotecardStatement invStmt;
int index = scope.GetStatement("inv_item", out invStmt); // What's
this one's parameter for?
if (invStmt == null)
throw new ArgumentException("Scope does not contain inventory
item data.");
InventoryItem item = null;
NotecardScope itemDataScope = scope.Children[index + 1] as
NotecardScope;
NotecardStatement scopeStatement = null;
foreach (NotecardElement el in itemDataScope.Children)
{
if (el is NotecardStatement)
{
NotecardStatement stmt = el as NotecardStatement;
switch (stmt.Name)
{
case "item_id":
item = new InventoryItem(new
LLUUID(stmt.Parameter));
item.Permissions = new Permissions();
break;
case "parent_id":
item.ParentUUID = new LLUUID(stmt.Parameter);
break;
case "permissions":
scopeStatement = stmt;
break;
case "asset_id":
item.AssetUUID = new LLUUID(stmt.Parameter);
break;
case "type":
item.AssetType = AssetTypeMap[stmt.Parameter];
break;
case "inv_type":
item.InventoryType =
InventoryTypeMap[stmt.Parameter];
break;
case "flags":
item.Flags = uint.Parse(stmt.Parameter);
break;
case "sale_info":
scopeStatement = stmt;
break;
case "name":
item.Name = stmt.Parameter.Substring(0,
stmt.Parameter.LastIndexOf('|'));
break;
case "desc":
item.Description = stmt.Parameter.Substring(0,
stmt.Parameter.LastIndexOf('|'));
break;
case "creation_date":
item.CreationDate =
Helpers.UnixTimeToDateTime(uint.Parse(stmt.Parameter));
break;
default:
Log.Log(string.Format("Unhandled item statement:
'{0}'", stmt.ToString()), Helpers.LogLevel.Warning);
break;
}
}
else if (el is NotecardScope && scopeStatement != null)
{
NotecardScope innerScope = el as NotecardScope;
if (scopeStatement.Name == "sale_info")
{
foreach (NotecardElement ele in innerScope.Children)
{
if (ele is NotecardStatement)
{
NotecardStatement stmt = ele as
NotecardStatement;
switch (stmt.Name)
{
case "sale_type":
item.SaleType =
(SaleType)Enum.Parse(typeof(SaleType), stmt.Parameter, true); // FIXME?
break;
case "sale_price":
item.SalePrice =
int.Parse(stmt.Parameter);
break;
case "perm_mask":
item.Permissions.NextOwnerMask =
(PermissionMask)int.Parse(stmt.Parameter,
System.Globalization.NumberStyles.HexNumber);
break;
default:
Log.Log(string.Format("Unhandled item
sale statement: '{0}'", ele.ToString()), Helpers.LogLevel.Warning);
break;
}
}
else
{
Log.Log(string.Format("Unexpected sale info
scope: '{0}'", ele.ToString()), Helpers.LogLevel.Warning);
}
}
}
else if (scopeStatement.Name == "permissions")
{
foreach (NotecardElement ele in innerScope.Children)
{
if (ele is NotecardStatement)
{
NotecardStatement stmt = ele as
NotecardStatement;
switch (stmt.Name)
{
case "base_mask":
item.Permissions.BaseMask =
(PermissionMask)int.Parse(stmt.Parameter,
System.Globalization.NumberStyles.HexNumber);
break;
case "owner_mask":
item.Permissions.OwnerMask =
(PermissionMask)int.Parse(stmt.Parameter,
System.Globalization.NumberStyles.HexNumber);
break;
case "group_mask":
item.Permissions.GroupMask =
(PermissionMask)int.Parse(stmt.Parameter,
System.Globalization.NumberStyles.HexNumber);
break;
case "everyone_mask":
item.Permissions.EveryoneMask =
(PermissionMask)int.Parse(stmt.Parameter,
System.Globalization.NumberStyles.HexNumber);
break;
case "creator_id":
// FIXME: Can we do something with this?
break;
case "owner_id":
item.OwnerID = new
LLUUID(stmt.Parameter);
break;
case "last_owner_id":
// FIXME: Can we do something with this?
break;
case "group_id":
item.GroupID = new
LLUUID(stmt.Parameter);
break;
default:
Log.Log(string.Format("Unhandled item
permissions statement: '{0}'", ele.ToString()), Helpers.LogLevel.Warning);
break;
}
}
else
{
Log.Log(string.Format("Unexpected permission
scope: '{0}'", ele.ToString()), Helpers.LogLevel.Warning);
}
}
}
scopeStatement = null;
}
else
{
Log.Log(string.Format("Unexpected item scope: '{0}'",
el.ToString()), Helpers.LogLevel.Warning);
}
}
Item = item;
}
}
public class NotecardParser
{
public int ParserVersion
{
get { return 2; }
}
private int TextVersion;
private NotecardRoot Root;
private NotecardScope RootContents;
private Logger Log;
public NotecardParser(NotecardRoot root, Logger log)
{
Root = root;
Log = log;
NotecardStatement rootStatement;
root.GetStatement("Linden", out rootStatement);
if (rootStatement == null || rootStatement.Parameters.Length < 3)
throw new FormatException("Parameter is not a Linden
notecard.");
if (rootStatement.Parameters[1] == "version")
{
int version;
if (int.TryParse(rootStatement.Parameters[2], out version))
{
TextVersion = version;
if (TextVersion < ParserVersion)
log.Log(string.Format("Notecard is old version {0},
current parser version is {1}.", version, ParserVersion),
Helpers.LogLevel.Warning);
}
else
{
throw new FormatException("Unable to parse notecard
version.");
}
}
else
{
throw new FormatException("Notecard version not specified.");
}
RootContents = (NotecardScope)root.Children[1];
}
public ItemInfo[] ParseItems()
{
ItemInfo[] items = null;
NotecardStatement embeddedItems;
int index = RootContents.GetStatement("LLEmbeddedItems", out
embeddedItems);
if (embeddedItems == null)
{
items = new ItemInfo[0];
return items;
}
NotecardScope itemsScope = RootContents.Children[index + 1] as
NotecardScope;
NotecardStatement count;
itemsScope.GetStatement("count", out count);
items = new ItemInfo[int.Parse(count.Parameter)];
int itemIndex = 0;
foreach (NotecardElement el in itemsScope.Children)
{
if (el is NotecardScope)
{
NotecardScope itemScope = el as NotecardScope;
ItemInfo info = new ItemInfo(Log);
info.Parse(itemScope);
items[itemIndex] = info;
++itemIndex;
}
}
return items;
}
}
#region Lexing
public class TextSource
{
public char this[int off]
{
get { return backingArray[start + off]; }
set { backingArray[start + off] = value; }
}
public string this[int off, int len]
{
get
{
return new string(backingArray, off + start, len);
}
}
public int Length
{
get { return backingArray.Length - start; }
}
private int start;
private char[] backingArray;
public TextSource(char[] src)
{
backingArray = src;
}
public TextSource(string src)
: this(src.ToCharArray()) { }
public string Consume(int len)
{
string ret = new string(backingArray, start, len);
start += len;
return ret;
}
public char Consume()
{
char ret = backingArray[start];
++start;
return ret;
}
public string ConsumeUntil(char[] chars)
{
StringBuilder builder = new StringBuilder();
int newStart;
for (newStart = start; newStart < backingArray.Length; ++newStart)
{
if (Array.IndexOf(chars, backingArray[newStart]) != -1)
{
start = newStart;
return builder.ToString();
}
else
{
builder.Append(backingArray[newStart]);
}
}
throw new FormatException(string.Format("ConsumeUntil could not
find a match in {0}", chars));
}
public string ConsumeUntil(char endChar)
{
StringBuilder builder = new StringBuilder();
int newStart;
for (newStart = start; newStart < backingArray.Length; ++newStart)
{
if (backingArray[newStart] == endChar)
{
start = newStart;
return builder.ToString();
}
else
{
builder.Append(backingArray[newStart]);
}
}
throw new FormatException(string.Format("ConsumeUntil could not
find '{0}' in source.", endChar));
}
public void TrimStart()
{
int newStart = start;
while (char.IsWhiteSpace(backingArray[newStart]))
newStart++;
start = newStart;
}
}
public abstract class NotecardElement
{
public const char LLNewline = '\r';
public abstract string ToString(int depth);
public override string ToString()
{
return ToString(0);
}
}
public class NotecardRoot : NotecardScope
{
public NotecardRoot(TextSource raw)
: base(raw) { }
public NotecardRoot() { }
public override string ToString(int depth)
{
StringBuilder builder = new StringBuilder();
foreach (NotecardElement child in Children)
{
builder.Append(child.ToString(0));
builder.Append(LLNewline);
}
return builder.ToString();
}
}
public class NotecardScope : NotecardElement
{
public List<NotecardElement> Children = new List<NotecardElement>();
public NotecardScope(TextSource raw)
{
Parse(raw);
}
public NotecardScope() { }
public void Parse(TextSource raw)
{
while (raw.Length > 0 && raw[0] != '}')
{
if (!char.IsWhiteSpace(raw[0]))
{
if (raw[0] != '{')
{
NotecardStatement statement = new
NotecardStatement(raw);
Children.Add(statement);
if (statement.Name == "Text")
{
if (statement.Parameters[0] == "length")
{
int txtLen = int.Parse(statement.Parameters[1]);
Children.Add(new NotecardText(raw, txtLen));
}
}
}
else
{
raw.Consume();// Consume the scope's beginning bracket.
Children.Add(new NotecardScope(raw));
}
}
else
{
raw.Consume();
}
}
if (raw.Length > 0)
raw.Consume(); // Consume the end bracket.
}
public int GetStatement(string name, out NotecardStatement statement)
{
for (int i = 0; i < Children.Count; ++i)
{
if (Children[i] is NotecardStatement)
{
NotecardStatement stmt = Children[i] as NotecardStatement;
if (stmt.Name == name)
{
statement = stmt;
return i;
}
}
}
statement = null;
return -1;
}
public override string ToString(int depth)
{
StringBuilder tabs = new StringBuilder(depth);
for (int i = 0; i < depth; ++i)
tabs.Append('\t');
string prefix = tabs.ToString();
StringBuilder builder = new StringBuilder();
builder.Append(tabs).Append('{').Append(LLNewline);
foreach (NotecardElement child in Children)
{
builder.Append(child.ToString(depth + 1));
builder.Append(LLNewline);
}
builder.Append(tabs).Append('}');
return builder.ToString();
}
}
public class NotecardStatement : NotecardElement
{
public string Name;
public string Parameter;
public string[] Parameters
{
get { return Parameter.Split(new char[] { ' ', '\t' },
StringSplitOptions.RemoveEmptyEntries); }
}
public NotecardStatement(TextSource raw)
{
Parse(raw);
}
public NotecardStatement(string name, string parameter)
{
Name = name;
Parameter = parameter;
}
public void Parse(TextSource raw)
{
raw.TrimStart();
// Parse name:
Name = raw.ConsumeUntil(new char[] { '\t', ' ' });
// Parse parameter:
raw.TrimStart();
Parameter = raw.ConsumeUntil(LLNewline);
raw.Consume(); // Consume the newline.
}
public override string ToString(int depth)
{
StringBuilder tabs = new StringBuilder(depth);
for (int i = 0; i < depth; ++i)
tabs.Append('\t');
string prefix = tabs.ToString();
return string.Format("{0}{1}\t{2}", tabs, Name, Parameter);
}
}
public class NotecardText : NotecardElement
{
public string Text;
public NotecardText(string text)
{
Text = text;
}
public NotecardText(TextSource raw, int length)
{
Parse(raw, length);
}
public void Parse(TextSource raw, int length)
{
Text = raw.Consume(length);
}
public override string ToString(int depth)
{
return Text;
}
}
public class NotecardLexerTest
{
public const char LLNewline = '\n';
public static void Main(string[] args)
{
FileStream reader = File.OpenRead(args[0]);
byte[] bytes = new byte[reader.Length];
reader.Read(bytes, 0, (int)reader.Length);
// WEIRDNESS:
// version 2 notecards use 4-byte placeholders for inventory items,
// UTF8 encoding will eat all four bytes and spit out only 1 or 2
chars.
// this messes up the text's length, it no longer matches up with
the "Text Length"
// metadata. They must be parsed using an ASCII encoding. Version 1
notecards,
// on the other hand, get screwed up by the ASCII decoder. Version
1 uses 1-byte
// placeholders, and oddly, decoding with ASCII decoder results in
a string that is
// too long. They must be parsed using the UTF8 encoding.
string text = Encoding.UTF8.GetString(bytes, 0, bytes.Length - 1);
Console.WriteLine("Read {0}", args[0]);
Console.WriteLine(text);
Console.WriteLine("Parsing...");
NotecardRoot root = new NotecardRoot();
DateTime start = DateTime.Now;
root.Parse(new TextSource(text));
DateTime stop = DateTime.Now;
Console.WriteLine("Lexed in {0} ms",
stop.Subtract(start).TotalMilliseconds);
string outputfile = args[0] + ".parsed.txt";
if (File.Exists(outputfile))
File.Delete(outputfile);
StreamWriter writer = File.CreateText(outputfile);
writer.WriteLine(root.ToString());
writer.Close();
Logger log = new Logger("Foo Bar");
log.LogNames = false;
NotecardParser parser = new NotecardParser(root, log);
ItemInfo[] items = parser.ParseItems();
foreach (ItemInfo info in items) {
InventoryItem item = info.Item;
Console.WriteLine("Item data:");
Console.WriteLine("\tType: {0}", item.InventoryType);
Console.WriteLine("\tName: {0}", item.Name);
Console.WriteLine("\tDesc: {0}", item.Description);
Console.WriteLine("\tItemID: {0}", item.UUID);
Console.WriteLine("\tParent: {0}", item.ParentUUID);
Console.WriteLine("\tOwner: {0}", item.OwnerID);
Console.WriteLine("\tAssetID: {0}", item.AssetUUID);
Console.WriteLine("\tAsset Type:{0}", item.AssetType);
Console.WriteLine("\tSale Price: {0}", item.SalePrice);
Console.WriteLine("\tSale Type: {0}", item.SaleType);
Console.WriteLine("\tPermissions: {0}", item.Permissions);
Console.WriteLine("\tGroup ID: {0}", item.GroupID);
Console.WriteLine("\tCreation Date: {0}", item.CreationDate);
}
}
}
#endregion
}
using System;
using System.Collections.Generic;
using System.Text;
namespace libsecondlife
{
public class Logger
{
private bool _LogNames;
public bool LogNames
{
get { return _LogNames; }
set { _LogNames = value; }
}
private bool _Debug;
public bool Debug
{
get { return _Debug; }
set { _Debug = value; }
}
private string _Name;
public string Name
{
get { return _Name; }
set { _Name = value; }
}
/// <summary>
/// Callback used for client apps to receive log messages from
/// libsecondlife
/// </summary>
/// <param name="message"></param>
/// <param name="level"></param>
public delegate void LogCallback(string message, Helpers.LogLevel
level);
/// <summary>Triggered whenever a message is logged.
/// If this is left null, log messages will go to
/// the console</summary>
public event LogCallback OnLogMessage;
public Logger(string name)
{
Name = name;
}
/// <summary>
/// Send a log message to the debugging output system
/// </summary>
/// <param name="message">The log message</param>
/// <param name="level">The severity of the log entry</param>
public void Log(string message, Helpers.LogLevel level)
{
if (level == Helpers.LogLevel.Debug && !Debug) return;
if (OnLogMessage != null)
{
try { OnLogMessage(message, level); }
catch (Exception e) { Console.WriteLine(e.ToString()); }
}
else
{
if (LogNames)
Console.WriteLine("{0} [{1}]: {2}",
level.ToString().ToUpper(), Name, message);
else
Console.WriteLine("{0}: {1}", level.ToString().ToUpper(),
message);
}
}
/// <summary>
/// If the library is compiled with DEBUG defined, and SecondLife.Debug
/// is true, either an event will be fired for the debug message or
/// it will be written to the console
/// </summary>
/// <param name="message">The debug message</param>
[System.Diagnostics.Conditional("DEBUG")]
public void DebugLog(string message)
{
if (Debug)
{
if (OnLogMessage != null)
{
try { OnLogMessage(message, Helpers.LogLevel.Debug); }
catch (Exception e) { Console.WriteLine(e.ToString()); }
}
else
{
if (LogNames)
Console.WriteLine("DEBUG [{0}]: {1}", Name, message);
else
Console.WriteLine("DEBUG: {0}", message);
}
}
}
/// <summary>
///
/// </summary>
/// <param name="message"></param>
/// <param name="level"></param>
public static void LogStatic(string message, Helpers.LogLevel level)
{
}
/// <summary>
///
/// </summary>
/// <param name="message"></param>
/// <param name="level"></param>
/// <param name="exception"></param>
public static void LogStatic(string message, Helpers.LogLevel level,
Exception exception)
{
}
/// <summary>
///
/// </summary>
/// <param name="message"></param>
[System.Diagnostics.Conditional("DEBUG")]
public static void DebugLogStatic(string message)
{
}
}
}
_______________________________________________
libsl-dev mailing list
libsl-dev@opensecondlife.org
http://opensecondlife.org/cgi-bin/mailman/listinfo/libsl-dev
