Author: toad
Date: 2005-12-07 19:03:04 +0000 (Wed, 07 Dec 2005)
New Revision: 7686
Added:
trunk/freenet/src/freenet/support/OutputStreamLogger.java
Modified:
trunk/freenet/src/freenet/node/Node.java
trunk/freenet/src/freenet/node/TextModeClientInterface.java
trunk/freenet/src/freenet/node/Version.java
trunk/freenet/src/freenet/support/FileLoggerHook.java
Log:
285: (mandatory because of continuous log compression)
Continual log compression: we write the compressed log at the same time as the
uncompressed log.
We keep the last two uncompressed logs (latest and previous) only.
Move logs/ to logs-<port>/
If we redirect stdout/stderr to log, we log it properly via the log queue,
rather than just writing it directly.
Modified: trunk/freenet/src/freenet/node/Node.java
===================================================================
--- trunk/freenet/src/freenet/node/Node.java 2005-12-07 17:25:08 UTC (rev
7685)
+++ trunk/freenet/src/freenet/node/Node.java 2005-12-07 19:03:04 UTC (rev
7686)
@@ -163,6 +163,7 @@
final RequestStarter requestStarter;
final RequestThrottle insertThrottle;
final RequestStarter insertStarter;
+ final File downloadDir;
// Client stuff that needs to be configged - FIXME
static final int MAX_ARCHIVE_HANDLERS = 200; // don't take up much RAM...
FIXME
@@ -278,13 +279,13 @@
int port = Integer.parseInt(args[0]);
System.out.println("Port number: "+port);
- new File("logs").mkdir();
- FileLoggerHook logger = new FileLoggerHook(true,
"logs/freenet-"+port+".log", "d (c, t, p): m", "MMM dd, yyyy HH:mm:ss:SSS",
Logger.MINOR, false, true);
+ File logDir = new File("logs-"+port);
+ logDir.mkdir();
+ FileLoggerHook logger = new FileLoggerHook(true, new File(logDir,
"freenet-"+port).getAbsolutePath(), "d (c, t, p): m", "MMM dd, yyyy
HH:mm:ss:SSS", Logger.MINOR, false, true);
logger.setInterval("5MINUTES");
Logger.setupChain();
Logger.globalSetThreshold(Logger.MINOR);
Logger.globalAddHook(logger);
- logger.setUseNativeGzip(true);
logger.start();
Logger.normal(Node.class, "Creating node...");
Yarrow yarrow = new Yarrow();
@@ -316,6 +317,7 @@
if(prefix == null) prefix = "";
filenamesPrefix = prefix;
this.overrideIPAddress = overrideIP;
+ downloadDir = new File("downloads");
try {
datastore = new
BaseFreenetStore(prefix+"freenet-"+portNumber,16384); // 512MB
} catch (FileNotFoundException e1) {
Modified: trunk/freenet/src/freenet/node/TextModeClientInterface.java
===================================================================
--- trunk/freenet/src/freenet/node/TextModeClientInterface.java 2005-12-07
17:25:08 UTC (rev 7685)
+++ trunk/freenet/src/freenet/node/TextModeClientInterface.java 2005-12-07
19:03:04 UTC (rev 7686)
@@ -46,6 +46,7 @@
final Hashtable streams;
final RequestStarterClient requestStarterClient;
final RequestStarterClient insertStarterClient;
+ final File downloadsDir;
TextModeClientInterface(Node n) {
this.n = n;
@@ -56,6 +57,7 @@
new Thread(this, "Text mode client interface").start();
this.requestStarterClient =
n.makeStarterClient(RequestStarter.INTERACTIVE_PRIORITY_CLASS, (short)0, false);
this.insertStarterClient =
n.makeStarterClient(RequestStarter.INTERACTIVE_PRIORITY_CLASS, (short)0, true);
+ this.downloadsDir = n.downloadDir;
}
/**
@@ -175,13 +177,14 @@
if(ext != null && !ext.equals(""))
fnam += "." + ext;
}
- if(new File(fnam).exists()) {
+ File f = new File(downloadsDir, fnam);
+ if(f.exists()) {
System.out.println("File exists already: "+fnam);
fnam = "freenet-"+System.currentTimeMillis()+"-"+fnam;
}
FileOutputStream fos = null;
try {
- fos = new FileOutputStream(fnam);
+ fos = new FileOutputStream(f);
BucketTools.copyTo(data, fos, Long.MAX_VALUE);
fos.close();
System.out.println("Written to "+fnam);
Modified: trunk/freenet/src/freenet/node/Version.java
===================================================================
--- trunk/freenet/src/freenet/node/Version.java 2005-12-07 17:25:08 UTC (rev
7685)
+++ trunk/freenet/src/freenet/node/Version.java 2005-12-07 19:03:04 UTC (rev
7686)
@@ -20,10 +20,10 @@
public static final String protocolVersion = "1.0";
/** The build number of the current revision */
- public static final int buildNumber = 284;
+ public static final int buildNumber = 285;
/** Oldest build of Fred we will talk to */
- public static final int lastGoodBuild = 282;
+ public static final int lastGoodBuild = 285;
/** The highest reported build of fred */
public static int highestSeenBuild = buildNumber;
Modified: trunk/freenet/src/freenet/support/FileLoggerHook.java
===================================================================
--- trunk/freenet/src/freenet/support/FileLoggerHook.java 2005-12-07
17:25:08 UTC (rev 7685)
+++ trunk/freenet/src/freenet/support/FileLoggerHook.java 2005-12-07
19:03:04 UTC (rev 7686)
@@ -5,7 +5,6 @@
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
-import java.io.InputStream;
import java.io.OutputStream;
import java.io.PrintStream;
import java.io.PrintWriter;
@@ -19,6 +18,7 @@
import java.util.StringTokenizer;
import java.util.TimeZone;
import java.util.Vector;
+import java.util.zip.GZIPOutputStream;
/**
* Converted the old StandardLogger to Ian's loggerhook interface.
@@ -60,17 +60,18 @@
private int[] fmt;
private String[] str;
- /**
- * The printstream the Logger writes too.
- */
- public PrintStream lout;
+ /** Stream to write data to (compressed if rotate is on) */
+ protected OutputStream logStream;
+ /** Other stream to write data to (may be null) */
+ protected OutputStream altLogStream;
- protected OutputStream uout;
-
protected final boolean logOverwrite;
/* Base filename for rotating logs */
protected String baseFilename = null;
+
+ protected File latestFilename;
+ protected File previousFilename;
/* Whether to redirect stdout */
protected boolean redirectStdOut = false;
@@ -86,18 +87,12 @@
protected int MAX_LIST_SIZE = 100000;
protected long MAX_LIST_BYTES = 10 * (1 << 20);
- protected boolean useNativeGzip = false;
-
// FIXME: should reimplement LinkedList with minimal locking
public void setMaxListLength(int len) {
MAX_LIST_SIZE = len;
}
- public void setUseNativeGzip(boolean x) {
- useNativeGzip = x;
- }
-
public void setMaxListBytes(long len) {
MAX_LIST_BYTES = len;
}
@@ -136,7 +131,7 @@
"invalid interval " + intervalName);
}
- protected String getHourLogName(Calendar c) {
+ protected String getHourLogName(Calendar c, boolean compressed) {
StringBuffer buf = new StringBuffer(50);
buf.append(baseFilename).append('-');
buf.append(c.get(Calendar.YEAR)).append('-');
@@ -149,6 +144,8 @@
buf.append('-');
pad2digits(buf, c.get(Calendar.MINUTE));
}
+ buf.append(".log");
+ if(compressed) buf.append(".gz");
return buf.toString();
}
@@ -173,6 +170,8 @@
GregorianCalendar gc = null;
String filename = null;
if (baseFilename != null) {
+ latestFilename = new
File(baseFilename+"-latest.log");
+ previousFilename = new
File(baseFilename+"-previous.log");
gc = new GregorianCalendar();
switch (INTERVAL) {
case Calendar.YEAR :
@@ -194,16 +193,15 @@
int x = gc.get(INTERVAL);
gc.set(INTERVAL, (x /
INTERVAL_MULTIPLIER) * INTERVAL_MULTIPLIER);
}
- filename = getHourLogName(gc);
- uout = openNewLogFile(filename);
+ filename = getHourLogName(gc, true);
+ logStream = openNewLogFile(new File(filename),
true);
+ if(latestFilename != null) {
+ altLogStream =
openNewLogFile(latestFilename, false);
+ }
+ System.err.println("Created log files");
gc.add(INTERVAL, INTERVAL_MULTIPLIER);
nextHour = gc.getTimeInMillis();
- lout = new PrintStream(uout);
}
- if (redirectStdOut)
- System.setOut(lout);
- if (redirectStdErr)
- System.setErr(lout);
while (true) {
try {
thisTime = System.currentTimeMillis();
@@ -211,44 +209,49 @@
if (thisTime > nextHour) {
// Switch logs
try {
- uout.flush();
+
logStream.flush();
+ if(altLogStream
!= null) altLogStream.flush();
} catch (IOException e)
{
System.err.println(
"Flushing on change caught " + e);
}
String oldFilename =
filename;
- filename =
getHourLogName(gc);
- OutputStream os =
openNewLogFile(filename);
- OutputStream oldUout =
uout;
- uout = os;
- lout = new
PrintStream(uout);
- if (redirectStdOut)
-
System.setOut(lout);
- if (redirectStdErr)
-
System.setErr(lout);
+ // Rotate primary log
stream
+ filename =
getHourLogName(gc, true);
try {
- oldUout.close();
+
logStream.close();
} catch (IOException e)
{
System.err.println(
-
"Closing on change caught " + e);
+
"Closing on change caught " + e);
}
+ logStream =
openNewLogFile(new File(filename), true);
+ if(latestFilename !=
null) {
+ try {
+
altLogStream.close();
+ } catch
(IOException e) {
+
System.err.println(
+
"Closing alt on change caught " + e);
+ }
+
if(previousFilename != null) {
+
previousFilename.delete();
+
latestFilename.renameTo(previousFilename);
+
latestFilename.delete();
+ } else {
+
latestFilename.delete();
+ }
+ altLogStream =
openNewLogFile(latestFilename, false);
+ }
+
System.err.println("Rotated log files: "+filename);
//System.err.println("Almost rotated");
- if (useNativeGzip) {
-
CompressorThread ct =
- new
CompressorThread(oldFilename);
- ct.start();
- // Don't care
about result
- } // FIXME: implement a
portable default compressor
gc.add(INTERVAL,
INTERVAL_MULTIPLIER);
nextHour =
gc.getTimeInMillis();
//System.err.println("Rotated");
}
}
if(list.size() == 0) {
- if (uout != null) {
- myWrite(uout, null);
- } else
- lout.flush();
+ myWrite(logStream, null);
+ if(altLogStream != null)
+ myWrite(altLogStream, null);
}
synchronized (list) {
while (list.size() == 0) {
@@ -264,11 +267,9 @@
o = list.removeFirst();
listBytes -= (((byte[])
o).length + 16);
}
- if (uout != null) {
- myWrite(uout, ((byte[]) o));
- } else {
- lout.print(new String((byte[])
o));
- }
+ myWrite(logStream, ((byte[]) o));
+ if(altLogStream != null)
+ myWrite(altLogStream, (byte[]) o);
} catch (OutOfMemoryError e) {
// FIXME
//freenet.node.Main.dumpInterestingObjects();
@@ -291,9 +292,9 @@
boolean thrown = false;
try {
if (b != null)
- uout.write(b);
+ os.write(b);
else
- uout.flush();
+ os.flush();
} catch (IOException e) {
System.err.println(
"Exception writing to log: "
@@ -315,13 +316,16 @@
}
}
- protected OutputStream openNewLogFile(String filename) {
+ protected OutputStream openNewLogFile(File filename, boolean
compress) {
while (true) {
long sleepTime = 1000;
try {
- System.err.println("Creating new
logfile " + filename);
OutputStream o = new
FileOutputStream(filename, !logOverwrite);
- return new BufferedOutputStream(o,
32768);
+ o = new BufferedOutputStream(o, 32768);
+ if(compress) {
+ o = new GZIPOutputStream(o);
+ }
+ return o;
} catch (IOException e) {
System.err.println(
"Could not create FOS " +
filename + ": " + e);
@@ -342,67 +346,6 @@
private Date myDate = new Date();
- class CompressorThread extends Thread {
- protected String filename;
-
- public CompressorThread(String fn) {
- super("Logfile compressor thread");
- this.filename = fn;
- }
-
- public void run() {
- synchronized (runningCompressorsSync) {
- runningCompressors++;
- if (runningCompressors > 3) { // try to absorb
a spike...
- System.err.println(
- "Too many compressors ("
- + runningCompressors
- + ") running!");
- runningCompressors--;
- new File(filename).delete();
- return;
- }
- }
- try {
- //System.err.println("Starting gzip " +
filename);
- Process r =
- Runtime.getRuntime().exec(
- new String[] { "nice", "gzip",
filename });
- //System.err.println("Started gzip " +
filename);
- InputStream is = r.getInputStream();
- InputStream es = r.getErrorStream();
- while (true) {
- byte[] buf = new byte[1024];
- try {
- is.read(buf, 0, buf.length);
- } catch (IOException e) {
- }
- try {
- es.read(buf, 0, buf.length);
- } catch (IOException e) {
- }
- try {
- if(r.exitValue() != 0) {
- new
File(filename).delete();
- }
- break;
- } catch (IllegalThreadStateException e)
{
- }
- }
- //System.err.println("Finished gzip " +
filename);
- is.close();
- es.close();
- } catch (IOException e) {
- System.err.println("Cannot compress old logfile
" + filename);
- e.printStackTrace(System.err);
- } finally {
- synchronized (runningCompressorsSync) {
- runningCompressors--;
- }
- }
- }
- }
-
/**
* Create a Logger to append to the given file. If the file does not
exist
* it will be created.
@@ -474,7 +417,7 @@
String dfmt,
int threshold) {
this(new PrintStream(os), fmt, dfmt, threshold, true);
- uout = os;
+ logStream = os;
}
public FileLoggerHook(
@@ -483,7 +426,7 @@
String dfmt,
String threshold) {
this(new PrintStream(os), fmt, dfmt,
priorityOf(threshold), true);
- uout = os;
+ logStream = os;
}
/**
@@ -505,10 +448,14 @@
int threshold,
boolean overwrite) {
this(fmt, dfmt, threshold, overwrite);
- lout = stream;
+ logStream = stream;
}
public void start() {
+ if(redirectStdOut)
+ System.setOut(new PrintStream(new
OutputStreamLogger(Logger.NORMAL, "Stdout: ")));
+ if(redirectStdErr)
+ System.setErr(new PrintStream(new
OutputStreamLogger(Logger.ERROR, "Stderr: ")));
WriterThread wt = new WriterThread();
//wt.setDaemon(true);
CloserThread ct = new CloserThread();
@@ -531,12 +478,9 @@
if (!assumeWorking)
checkStdStreams();
if (rotate) {
- lout = null;
this.baseFilename = baseFilename;
} else {
- lout =
- new PrintStream(
- new FileOutputStream(baseFilename,
!logOverwrite));
+ logStream = new FileOutputStream(baseFilename,
!logOverwrite);
}
}
@@ -733,10 +677,6 @@
return ((2 * ERROR) - 1) & ~(threshold - 1);
}
- public PrintStream getStream() {
- return lout;
- }
-
public void close() {
closed = true;
}
Added: trunk/freenet/src/freenet/support/OutputStreamLogger.java
===================================================================
--- trunk/freenet/src/freenet/support/OutputStreamLogger.java 2005-12-07
17:25:08 UTC (rev 7685)
+++ trunk/freenet/src/freenet/support/OutputStreamLogger.java 2005-12-07
19:03:04 UTC (rev 7686)
@@ -0,0 +1,26 @@
+package freenet.support;
+
+import java.io.OutputStream;
+
+public class OutputStreamLogger extends OutputStream {
+
+ final int prio;
+ final String prefix;
+
+ public OutputStreamLogger(int prio, String prefix) {
+ this.prio = prio;
+ this.prefix = prefix;
+ }
+
+ public void write(int b) {
+ Logger.logStatic(this, prefix+(char)b, prio);
+ }
+
+ public void write(byte[] buf, int offset, int length) {
+ Logger.logStatic(this, prefix+new String(buf, offset, length),
prio);
+ }
+
+ public void write(byte[] buf) {
+ write(buf, 0, buf.length);
+ }
+}
