On 22 August 2010 18:25, Michael Van Canneyt wrote:
>
> Why ? To the compiler there is no difference between EOL and whitespace.
>
> If you want to generate it with the exact layout, then, yes.
> But for profiling I don't see the need.
Maybe to the compiler it doesn't matter, but to a human trying to
debug their profiler to see if they inserted code in the correct
place, or to see why code doesn't compile any more after the profiler
modified the code - such "human readable code" is rather important.
:)
If no NewLine characters are inserted into the newly generated units,
a compiler error on line 1 doesn't help the developer much. Line 1
could be 50k bytes or more long.
Lazarus can't open the file (error message says it doesn't look like a
text file), other editor show one line, but clips text because they
can't display such a long single line of text. The profiler becomes a
near perfect obfuscater. :-)
>
> Well, I still fail to see why you need the tab and EOL.
Well, does it hurt having them tokenized?
If anybody other than me comes up with a new idea/way of using the
fcl-passrc parser on tokenizer - maybe they would like to rebuild
source code at some point. Without tokens telling you where EOL used
to be or TAB characters used to be (though the latter is not to
important), it does limit it's functionality.
Attached is a file processed with the profiler and fcl-passrc that
doesn't use tkLineEnding or tkTab. It's rather hard to read and fix
compiler errors introduced by the profile inserting code.
I don't see why introducing such new tokens is bad? It can only make
the tokenizer and parser more useful in the long term.
--
Regards,
- Graeme -
_______________________________________________
fpGUI - a cross-platform Free Pascal GUI toolkit
http://opensoft.homeip.net/fpgui/
unit fpputils; interface uses fpprof, pscanner, PasTree, SysUtils, Classes, FPPWriter; const FPPROF_EXT = '.fpprof'; type TPasToken = record token: TToken; value: string; end; PPasToken = ^TPasToken; TPasTokenList = class(TObject) FFileName: string; private FList: TFPList; function GetList(index: integer): TPasToken; procedure SetFileName(const AValue: string); procedure SetList(AIndex: integer; const AValue: TPasToken); public constructor Create; destructor Destroy; override; function Count: integer; function ParseSource(AFileName: string): boolean; procedure Add(AToken: TToken; AValue: string); procedure Clear; procedure Insert(APos: integer; AToken: TToken; AValue: string); procedure SaveToFile(const AFileName: string); property FileName: string read FFileName write SetFileName; property List[index: integer]: TPasToken read GetList write SetList; default; end; TModTokenProc = procedure(AFileName: string; tokenlist: TPasTokenList); procedure FileSearch(SearchDir: string; ExtensionMask: string; var FileList: TStrings; Recursive: boolean = false); procedure InsertProfilingCode(FileList: TStrings; ModTokenProc: TModTokenProc); procedure RemoveProfilingCodeFromFile(const FileName: string); procedure RemoveProfilingCode(FileList: TStrings); implementation procedure FileSearch(SearchDir: string; ExtensionMask: string; var FileList: TStrings; Recursive: boolean = false); var Info : TSearchRec; ExtensionList: TStrings; begin fpprof_entry_profile;
SearchDir := IncludeTrailingPathDelimiter(SearchDir); ExtensionList := TStringList.Create; ExtensionList.Delimiter := ';'; ExtensionList.DelimitedText := ExtensionMask; if FindFirst(SearchDir+AllFilesMask, faAnyFile and faDirectory, Info) = 0 then begin repeat if Recursive then if ((Info.Attr and faDirectory) = faDirectory) and (Info.Name <> '.') and (Info.Name <> '..')then FileSearch(SearchDir + Info.Name, ExtensionMask, FileList, Recursive); if ExtensionList.IndexOf(ExtractFileExt(Info.Name)) <> -1 then begin if FileList.IndexOf(SearchDir + Info.Name) = -1 then FileList.Add(SearchDir + Info.Name); end; until FindNext(Info)<>0; end; FindClose(Info); ExtensionList.Free; fpprof_exit_profile;
end; procedure InsertProfilingCode(FileList: TStrings; ModTokenProc: TModTokenProc); var i: integer; PasTokenList: TPasTokenList; Success: boolean; writer: TFPPWriter; begin fpprof_entry_profile;
FileList.SaveToFile('./file_processed_list.txt'); PasTokenList := TPasTokenList.Create; writer := TFPPWriter.Create; writer.CreateIgnored; for i := 0 to FileList.Count - 1 do begin write('insert: ', FileList[i]); try Success := PasTokenList.ParseSource(FileList[i]); Success := Success and RenameFile(FileList[i], FileList[i] + FPPROF_EXT); if Assigned(ModTokenProc) then ModTokenProc(FileList[i], PasTokenList); PasTokenList.SaveToFile(FileList[i]); if Success then writeln(' .......... OK') except writeln(' .......... FAIL'); writer.AddIgnoredFile(FileList[i]); end; PasTokenList.Clear; end; writer.Save; writer.Free; PasTokenList.Free; FileList.Free; fpprof_exit_profile;
end; procedure RemoveProfilingCodeFromFile(const FileName: string); var NewFileName: string; begin fpprof_entry_profile;
NewFileName := Copy(FileName, 1, Length(FileName) - Length(FPPROF_EXT)); write('revert: ', NewFileName); DeleteFile(NewFileName); if RenameFile(FileName, NewFileName) then writeln(' .......... OK') else writeln(' .......... FAIL'); fpprof_exit_profile;
end; procedure RemoveProfilingCode(FileList: TStrings); var i: integer; begin fpprof_entry_profile;
for i := 0 to FileList.Count - 1 do RemoveProfilingCodeFromFile(FileList[i]); fpprof_exit_profile;
end; function TPasTokenList.GetList(index: integer): TPasToken; begin fpprof_entry_profile;
Result := TPasToken(FList[index]^); fpprof_exit_profile;
end; procedure TPasTokenList.SetFileName(const AValue: string); begin fpprof_entry_profile;
if FFileName=AValue then exit; FFileName:=AValue; fpprof_exit_profile;
end; procedure TPasTokenList.SetList(AIndex: integer; const AValue: TPasToken); begin fpprof_entry_profile;
FList.Add(@AValue); fpprof_exit_profile;
end; constructor TPasTokenList.Create; begin fpprof_entry_profile;
FList := TFPList.Create; fpprof_exit_profile;
end; destructor TPasTokenList.Destroy; begin fpprof_entry_profile;
Clear; FList.Free; inherited Destroy; fpprof_exit_profile;
end; procedure TPasTokenList.Clear; var i: integer; begin fpprof_entry_profile;
for i := 0 to FList.Count - 1 do Dispose(PPasToken(FList[i])); FList.Clear; fpprof_exit_profile;
end; procedure TPasTokenList.Insert(APos: integer; AToken: TToken; AValue: string); var pt: ^TPasToken; begin fpprof_entry_profile;
New(pt); pt^.token := AToken; pt^.Value := AValue; FList.Insert(APos, pt); fpprof_exit_profile;
end; procedure TPasTokenList.SaveToFile(const AFileName: string); var t: text; i: integer; begin fpprof_entry_profile;
assign(t, AFileName); rewrite(t); for i := FList.Count-1 downto 0 do begin case TPasToken(FList[i]^).token of tkWhitespace: write(t, ' '); tkString: write(t, TPasToken(FList[i]^).value ); tkBraceOpen: write(t, '('); tkBraceClose: write(t, ')'); tkMul: write(t, '*'); tkPlus: write(t, '+'); tkComma: write(t, ','); tkMinus: write(t, '-'); tkDot: write(t, '.'); tkDivision: write(t, '/'); tkColon: write(t, ':'); tkSemicolon: write(t, ';'); tkLessThan: write(t, '<'); tkEqual: write(t, '='); tkGreaterThan: write(t, '>'); tkAt: write(t, '@'); tkSquaredBraceOpen: write(t, '['); tkSquaredBraceClose: write(t, ']'); tkCaret: write(t, '^'); tkDotDot: write(t, '..'); tkAssign: write(t, ':='); tkNotEqual: write(t, '<>'); tkLessEqualThan: write(t, '<='); tkGreaterEqualThan: write(t, '>='); tkPower: write(t, '**'); tkSymmetricalDifference: write(t, '><'); tkLineEnding: writeln(t, LineEnding); tkTab: writeln(t, #9); else if TPasToken(FList[i]^).token <> tkComment then write(t, TPasToken(FList[i]^).value); end; end; close(t); fpprof_exit_profile;
end; function TPasTokenList.ParseSource(AFileName: string): boolean; var pas: TPascalScanner; fr: TFileResolver; index: integer; token: TToken; begin fpprof_entry_profile;
Result := False; fr := TFileResolver.Create; pas := TPascalScanner.Create(fr); try pas.OpenFile(AFileName); index := 0; repeat inc(index); token := pas.FetchToken; Add(token, pas.CurTokenString); until pas.CurToken = tkEOF; Result := True; finally pas.Free; fr.Free; end; fpprof_exit_profile;
end; procedure TPasTokenList.Add(AToken: TToken; AValue: string); begin fpprof_entry_profile;
Insert(0, AToken, AValue); fpprof_exit_profile;
end; function TPasTokenList.Count: integer; begin fpprof_entry_profile;
Result := FList.Count; fpprof_exit_profile;
end; end. _______________________________________________
fpc-devel maillist - fpc-devel@lists.freepascal.org
http://lists.freepascal.org/mailman/listinfo/fpc-devel
