Hi guys,
First of all I like to say I don't read all the posts here. So maybe
someone has already posted a patch... If not; here is one :)
Earlier I posted a problem I encountered while serving 'large' FLV
files and files with the same filenames in different paths.
The problem is the cache mechanism and the new ByteBuffer
implementation. As I wrote then, maybe it's just not a good idea to
'cache' the whole FLV in heap mem... maybe it's an idea to leave the
caching to the OS (unless it's a small and often played file, like a
banner ;)).
To fix the problems I patched the FLV and FLVReader class so it
doesn't cache anymore and it doesn't 'allocate' the MINA ByteBuffer.
Furthermore I added a file-based caching mechanism for the KeyFrame
Meta-info (FLVReader) so it doesn't 'scan' the whole file every time.
The patch is tested by playing a 300MB FLV via a bandwidth capped NFS
share, it plays well and it uses only 7MB from the 16MB heap (-Xmx=32m
and -Xms=16m).
Since you guys did all the great work I whould like to share the patch
and hope you can use it (or at least the idea).
Greetz,
Martin Schipper
p.s. I patched upon SVN version 1535
p.s.2 Maybe attachments don't work in this list so I'll also put them
online at: http://www.dbcorp.nl/red5/
------------------------------------------------------------------------
Index: src/org/red5/io/flv/impl/FLV.java
===================================================================
--- src/org/red5/io/flv/impl/FLV.java (revision 1535)
+++ src/org/red5/io/flv/impl/FLV.java (working copy)
@@ -2,21 +2,21 @@
/*
* RED5 Open Source Flash Server - http://www.osflash.org/red5
- *
+ *
* Copyright (c) 2006 by respective authors (see below). All rights reserved.
- *
- * This library is free software; you can redistribute it and/or modify it under the
- * terms of the GNU Lesser General Public License as published by the Free Software
- * Foundation; either version 2.1 of the License, or (at your option) any later
- * version.
- *
- * This library is distributed in the hope that it will be useful, but WITHOUT ANY
- * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+ *
+ * This library is free software; you can redistribute it and/or modify it
under the
+ * terms of the GNU Lesser General Public License as published by the Free
Software
+ * Foundation; either version 2.1 of the License, or (at your option) any later
+ * version.
+ *
+ * This library is distributed in the hope that it will be useful, but WITHOUT
ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
FOR A
* PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
details.
- *
- * You should have received a copy of the GNU Lesser General Public License along
- * with this library; if not, write to the Free Software Foundation, Inc.,
- * 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ *
+ * You should have received a copy of the GNU Lesser General Public License
along
+ * with this library; if not, write to the Free Software Foundation, Inc.,
+ * 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*/
import java.io.File;
@@ -41,18 +41,19 @@
/**
* A FLVImpl implements the FLV api
- *
+ *
* @author The Red5 Project ([email protected])
* @author Dominick Accattato ([EMAIL PROTECTED])
* @author Luke Hubbard, Codegent Ltd ([EMAIL PROTECTED])
* @author Paul Gregoire, ([EMAIL PROTECTED])
+ * @author Martin Schipper ([EMAIL PROTECTED])
*/
public class FLV implements IFLV {
protected static Log log = LogFactory.getLog(FLV.class.getName());
- private static ICacheStore cache;
-
+ private static ICacheStore cache;
+
private File file;
private boolean generateMetadata;
@@ -60,12 +61,12 @@
private IMetaService metaService;
/**
- * Default constructor, used by Spring so that parameters
+ * Default constructor, used by Spring so that parameters
* may be injected.
*/
public FLV() {
}
-
+
public FLV(File file) {
this(file, false);
}
@@ -77,16 +78,16 @@
/**
* Sets the cache implementation to be used.
- *
+ *
* @param cache
*/
public void setCache(ICacheStore cache) {
FLV.cache = cache;
}
-
+
/*
* (non-Javadoc)
- *
+ *
* @see org.red5.io.flv.FLV#hasMetaData()
*/
public boolean hasMetaData() {
@@ -96,7 +97,7 @@
/*
* (non-Javadoc)
- *
+ *
* @see org.red5.io.flv.FLV#getMetaData()
*/
public IMetaData getMetaData() throws FileNotFoundException {
@@ -106,7 +107,7 @@
/*
* (non-Javadoc)
- *
+ *
* @see org.red5.io.flv.FLV#hasKeyFrameData()
*/
public boolean hasKeyFrameData() {
@@ -116,7 +117,7 @@
/*
* (non-Javadoc)
- *
+ *
* @see org.red5.io.flv.FLV#setKeyFrameData(java.util.Map)
*/
public void setKeyFrameData(Map keyframedata) {
@@ -124,7 +125,7 @@
/*
* (non-Javadoc)
- *
+ *
* @see org.red5.io.flv.FLV#getKeyFrameData()
*/
public Map getKeyFrameData() {
@@ -134,7 +135,7 @@
/*
* (non-Javadoc)
- *
+ *
* @see org.red5.io.flv.FLV#refreshHeaders()
*/
public void refreshHeaders() throws IOException {
@@ -144,7 +145,7 @@
/*
* (non-Javadoc)
- *
+ *
* @see org.red5.io.flv.FLV#flushHeaders()
*/
public void flushHeaders() throws IOException {
@@ -154,25 +155,36 @@
/*
* (non-Javadoc)
- *
+ *
* @see org.red5.io.flv.FLV#reader()
*/
public ITagReader getReader() throws IOException {
FLVReader reader = null;
- ByteBuffer fileData = null;
- String fileName = file.getName();
+ //ByteBuffer fileData = null;
+ //String fileName = file.getName();
- ICacheable ic = cache.get(fileName);
+ /** TODO: YYY: I disabled the cache because it consumes all our
heap mem.
+ * Besides; it doesn't work (filename/pathname) and I run Linux
and load
+ * the FLV's from an NFS share, Linux caches the file itself.
+ * Maybe it really gains performance if it is used for small
files which
+ * are served very often (like banners).
+ */
+ //ICacheable ic = cache.get(fileName);
+ // look in the cache before reading the file from the disk
+ //if (null == ic || (null == ic.getByteBuffer())) {
- // look in the cache before reading the file from the disk
- if (null == ic || (null == ic.getByteBuffer())) {
if (file.exists()) {
if (log.isDebugEnabled()) {
log.debug("File size: " +
file.length());
}
- reader = new FLVReader(new
FileInputStream(file), generateMetadata);
+ reader = new FLVReader(new
FileInputStream(file), generateMetadata, file);
+
+
+ /* YYY: Disabled.
+ *
// get a ref to the mapped byte buffer
fileData = reader.getFileData();
+
// offer the uncached file to the cache
if (cache.offer(fileName, new
CacheableImpl(fileData))) {
if (log.isDebugEnabled()) {
@@ -183,20 +195,24 @@
log.debug("Item will not be cached:
" + fileName);
}
}
+ */
+
} else {
log.info("Creating new file: " + file);
file.createNewFile();
}
+ /* YYY: Disabled.
} else {
fileData = ByteBuffer.wrap(ic.getBytes());
reader = new FLVReader(fileData, generateMetadata);
}
+ */
return reader;
}
/*
* (non-Javadoc)
- *
+ *
* @see org.red5.io.flv.FLV#readerFromNearestKeyFrame(int)
*/
public ITagReader readerFromNearestKeyFrame(int seekPoint) {
@@ -206,7 +222,7 @@
/*
* (non-Javadoc)
- *
+ *
* @see org.red5.io.flv.FLV#writer()
*/
public ITagWriter getWriter() throws IOException {
@@ -246,7 +262,7 @@
/*
* (non-Javadoc)
- *
+ *
* @see org.red5.io.flv.FLV#writerFromNearestKeyFrame(int)
*/
public ITagWriter writerFromNearestKeyFrame(int seekPoint) {
Index: src/org/red5/io/flv/impl/FLVReader.java
===================================================================
--- src/org/red5/io/flv/impl/FLVReader.java (revision 1535)
+++ src/org/red5/io/flv/impl/FLVReader.java (working copy)
@@ -2,24 +2,28 @@
/*
* RED5 Open Source Flash Server - http://www.osflash.org/red5
- *
+ *
* Copyright (c) 2006 by respective authors (see below). All rights reserved.
- *
- * This library is free software; you can redistribute it and/or modify it under the
- * terms of the GNU Lesser General Public License as published by the Free Software
- * Foundation; either version 2.1 of the License, or (at your option) any later
- * version.
- *
- * This library is distributed in the hope that it will be useful, but WITHOUT ANY
- * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+ *
+ * This library is free software; you can redistribute it and/or modify it
under the
+ * terms of the GNU Lesser General Public License as published by the Free
Software
+ * Foundation; either version 2.1 of the License, or (at your option) any later
+ * version.
+ *
+ * This library is distributed in the hope that it will be useful, but WITHOUT
ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
FOR A
* PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
details.
- *
- * You should have received a copy of the GNU Lesser General Public License along
- * with this library; if not, write to the Free Software Foundation, Inc.,
- * 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ *
+ * You should have received a copy of the GNU Lesser General Public License
along
+ * with this library; if not, write to the Free Software Foundation, Inc.,
+ * 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*/
+import java.io.File;
import java.io.FileInputStream;
+import java.io.ObjectInputStream;
+import java.io.FileOutputStream;
+import java.io.ObjectOutputStream;
import java.io.IOException;
import java.nio.ByteOrder;
import java.nio.MappedByteBuffer;
@@ -47,12 +51,16 @@
* @author Dominick Accattato ([EMAIL PROTECTED])
* @author Luke Hubbard, Codegent Ltd ([EMAIL PROTECTED])
* @author Paul Gregoire, ([EMAIL PROTECTED])
+ * @author Martin Schipper ([EMAIL PROTECTED])
*/
public class FLVReader implements IoConstants, ITagReader,
IKeyFrameDataAnalyzer {
private static Log log = LogFactory.getLog(FLVReader.class.getName());
+ /** TODO: YYY: added file ref for 'keyframeMeta-cache' */
+ private File file = null;
+
private FileInputStream fis;
private FileChannel channel;
@@ -86,14 +94,78 @@
/** Buffer type / style to use **/
private static String bufferType = "auto"; //Default
-
+
FLVReader() {}
-
+
public FLVReader(FileInputStream f) {
this(f, false);
}
+ /**
+ * TODO: YYY: Added an extra constructor so I have the file reference.
+ */
+ public FLVReader(FileInputStream f, boolean generateMetadata, File
file_) {
+ file = file_;
+ boolean dontNeedToSave = this.loadKeyFramesCache();
+ this.init(f, generateMetadata);
+ if (!dontNeedToSave) this.saveKeyFramesCache();
+ }
+
+ /**
+ * TODO: YYY: load and save the 'keyframeMeta' so we don't have to scan
+ * the whole file everytime we load it.
+ * TODO: YYY: add a 'file-version' option so we are sure to have the
correct
+ * 'keyframeMeta' for the given file.
+ */
+ public synchronized boolean loadKeyFramesCache() {
+ if (file == null) return false;
+ log.debug("Check for KeyFramesCache");
+ try {
+ File kfCacheFile = new File(file.getPath()+".kfcache");
+ if (kfCacheFile.isFile() && kfCacheFile.canRead()) {
+ ObjectInputStream in = new
ObjectInputStream(new FileInputStream(kfCacheFile));
+ //keyframeMeta = (KeyFrameMeta) in.readObject();
+ HashMap cacheData = (HashMap) in.readObject();
+ keyframeMeta = (KeyFrameMeta)
cacheData.get("keyframeMeta");
+ posTimeMap = (HashMap<Long, Long>)
cacheData.get("posTimeMap");
+ posTagMap = (HashMap<Long, Integer>)
cacheData.get("posTagMap");
+ in.close();
+ log.debug("Loaded KeyFramesCache for file
"+file.getPath());
+ return true;
+ }
+ } catch (Exception e) { e.printStackTrace(); /* ignore */ }
+ return false;
+ }
+ public boolean saveKeyFramesCache() {
+ if (file == null) return false;
+ log.debug("Save KeyFramesCache");
+ try {
+ File kfCacheFile = new
File(file.getPath()+".Red5cache");
+ if (kfCacheFile.isFile() && kfCacheFile.canRead()) {
+ log.debug("already know this file");
+ return true;
+ }
+ if (kfCacheFile.createNewFile() &&
kfCacheFile.canWrite()) {
+ ObjectOutputStream out = new
ObjectOutputStream(new FileOutputStream(kfCacheFile));
+ HashMap cacheData = new HashMap();
+ cacheData.put("keyframeMeta", keyframeMeta);
+ cacheData.put("posTimeMap", posTimeMap);
+ cacheData.put("posTagMap", posTagMap);
+ out.writeObject(cacheData);
+ out.close();
+ log.debug("Saved KeyFramesCache for file
"+file.getPath());
+ return true;
+ }
+ } catch (Exception e) { e.printStackTrace(); /* ignore */ }
+ return false;
+ }
+
+ /** TODO: YYY: renamed the constructor to init */
public FLVReader(FileInputStream f, boolean generateMetadata) {
+ this.init(f,generateMetadata);
+ }
+
+ public void init (FileInputStream f, boolean generateMetadata) {
this.fis = f;
this.generateMetadata = generateMetadata;
channel = fis.getChannel();
@@ -102,9 +174,13 @@
.size());
mappedFile.order(ByteOrder.BIG_ENDIAN);
if (log.isDebugEnabled()) {
- log.debug("Mapped file capacity: " +
mappedFile.capacity() + " Channel size: " + channel.size());
+ log.debug("Mapped file capacity: " + mappedFile.capacity() + "
Channel size: " + channel.size() + " remaining: " + mappedFile.remaining());
}
- switch (bufferType.hashCode()) {
+
+ /** TODO: YYY: Don't allocate a fixed bytebuffer, it
consumes all our heap mem.
+ * I serve 300MB flv's, it just won't work for me..
+ */
+ /*switch (bufferType.hashCode()) {
case 3198444: //heap
//Get a heap buffer from buffer pool
in =
ByteBuffer.allocate(mappedFile.capacity(), false);
@@ -117,13 +193,17 @@
default:
//Let MINA choose
in =
ByteBuffer.allocate(mappedFile.capacity());
- }
+ }*/
//drop in the file
- in.put(mappedFile);
+ //in.put(mappedFile);
//prepare the buffer for access
- in.flip();
+ //in.flip();
+
+ /** TODO: YYY: Just wrap the NIO ByteBuffer in a nice
MINA bytebuffer. */
+ in = ByteBuffer.wrap(mappedFile);
+
if (log.isDebugEnabled()) {
- log.debug("Direct buffer: " + in.isDirect() + " Read only:
" + in.isReadOnly() + " Pooled: " + in.isPooled());
+ log.debug("Buffer limit: " + in.limit() + "Direct buffer: " +
in.isDirect() + " Read only: " + in.isReadOnly() + " Pooled: " + in.isPooled());
}
} catch (IOException e) {
log.error("FLVReader :: FLVReader ::>\n", e);
@@ -142,7 +222,7 @@
/**
* Accepts mapped file bytes to construct internal members.
- *
+ *
* @param mappedFile
* @param generateMetadata
*/
@@ -169,7 +249,7 @@
/**
* Returns the file buffer.
- *
+ *
* @return file bytes
*/
public ByteBuffer getFileData() {
@@ -191,7 +271,7 @@
/*
* (non-Javadoc)
- *
+ *
* @see org.red5.io.flv.Reader#getFLV()
*/
public IStreamableFile getFile() {
@@ -201,7 +281,7 @@
/*
* (non-Javadoc)
- *
+ *
* @see org.red5.io.flv.Reader#getOffset()
*/
public int getOffset() {
@@ -211,7 +291,7 @@
/*
* (non-Javadoc)
- *
+ *
* @see org.red5.io.flv.Reader#getBytesRead()
*/
synchronized public long getBytesRead() {
@@ -224,10 +304,11 @@
/*
* (non-Javadoc)
- *
+ *
* @see org.red5.io.flv.Reader#hasMoreTags()
*/
synchronized public boolean hasMoreTags() {
+ //log.debug("hasMoreTags : remaining : "+in.remaining());
return in.remaining() > 4;
}
@@ -271,7 +352,7 @@
/*
* (non-Javadoc)
- *
+ *
* @see org.red5.io.flv.Reader#readTag()
*/
synchronized public ITag readTag() {
@@ -291,7 +372,7 @@
ByteBuffer body = ByteBuffer.allocate(tag.getBodySize());
final int limit = in.limit();
- // XXX Paul: this assists in 'properly' handling damaged FLV
files
+ // XXX Paul: this assists in 'properly' handling damaged FLV
files
int newPosition = in.position() + tag.getBodySize();
if (newPosition <= limit) {
in.limit(newPosition);
@@ -307,7 +388,7 @@
/*
* (non-Javadoc)
- *
+ *
* @see org.red5.io.flv.Reader#close()
*/
synchronized public void close() {
@@ -335,6 +416,8 @@
return keyframeMeta;
}
+ log.debug("analyzeKeyFrames");
+
List<Integer> positionList = new ArrayList<Integer>();
List<Integer> timestampList = new ArrayList<Integer>();
int origPos = in.position();
@@ -391,6 +474,7 @@
keyframeMeta = new KeyFrameMeta();
posTimeMap = new HashMap<Long, Long>();
+
keyframeMeta.positions = new int[positionList.size()];
keyframeMeta.timestamps = new int[timestampList.size()];
for (int i = 0; i < keyframeMeta.positions.length; i++) {
@@ -400,6 +484,7 @@
.get(i));
}
return keyframeMeta;
+
}
synchronized public void position(long pos) {
------------------------------------------------------------------------
_______________________________________________
Red5 mailing list
[email protected]
http://osflash.org/mailman/listinfo/red5_osflash.org
