[OPENMEETINGS-1770] ODF import is fixed
Project: http://git-wip-us.apache.org/repos/asf/openmeetings/repo Commit: http://git-wip-us.apache.org/repos/asf/openmeetings/commit/f9eef4df Tree: http://git-wip-us.apache.org/repos/asf/openmeetings/tree/f9eef4df Diff: http://git-wip-us.apache.org/repos/asf/openmeetings/diff/f9eef4df Branch: refs/heads/4.0.x Commit: f9eef4dfd6eeb452541489a23c708edb63bc1da3 Parents: 45a9507 Author: Maxim Solodovnik <[email protected]> Authored: Fri Nov 24 18:19:48 2017 +0700 Committer: Maxim Solodovnik <[email protected]> Committed: Fri Nov 24 18:19:48 2017 +0700 ---------------------------------------------------------------------- .../core/data/file/FileProcessor.java | 5 +- .../src/main/assembly/components/all.xml | 126 ++++++++++++++++++- openmeetings-util/pom.xml | 5 + .../apache/openmeetings/util/StoredFile.java | 30 +++-- .../openmeetings/util/TestStoredFile.java | 16 +++ .../src/test/resources/ODFtest.odt | Bin 0 -> 8363 bytes .../src/test/resources/ODStest.ods | Bin 0 -> 7339 bytes .../web/room/sidebar/UploadDialog.java | 3 - .../openmeetings/util/TestStoredFile.java | 1 - 9 files changed, 167 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/openmeetings/blob/f9eef4df/openmeetings-core/src/main/java/org/apache/openmeetings/core/data/file/FileProcessor.java ---------------------------------------------------------------------- diff --git a/openmeetings-core/src/main/java/org/apache/openmeetings/core/data/file/FileProcessor.java b/openmeetings-core/src/main/java/org/apache/openmeetings/core/data/file/FileProcessor.java index f530323..e3fca3e 100644 --- a/openmeetings-core/src/main/java/org/apache/openmeetings/core/data/file/FileProcessor.java +++ b/openmeetings-core/src/main/java/org/apache/openmeetings/core/data/file/FileProcessor.java @@ -36,6 +36,7 @@ import org.apache.openmeetings.db.entity.file.FileItem; import org.apache.openmeetings.util.StoredFile; import org.apache.openmeetings.util.process.ProcessResult; import org.apache.openmeetings.util.process.ProcessResultList; +import org.apache.tika.exception.UnsupportedFormatException; import org.red5.logging.Red5LoggerFactory; import org.slf4j.Logger; import org.springframework.beans.factory.annotation.Autowired; @@ -82,15 +83,13 @@ public class FileProcessor { } else if (sf.isPdf() || sf.isOffice()) { f.setType(Type.Presentation); } else { - logs.add(new ProcessResult("The file type cannot be converted :: " + f.getName())); - return logs; + throw new UnsupportedFormatException("The file type cannot be converted :: " + f.getName()); } f.setHash(hash); processFile(f, sf, temp, logs); } catch (Exception e) { log.debug("Error while processing the file", e); - logs.add(new ProcessResult("Unexpected exception: " + e.getMessage())); throw e; } finally { if (temp != null && temp.exists() && temp.isFile()) { http://git-wip-us.apache.org/repos/asf/openmeetings/blob/f9eef4df/openmeetings-server/src/main/assembly/components/all.xml ---------------------------------------------------------------------- diff --git a/openmeetings-server/src/main/assembly/components/all.xml b/openmeetings-server/src/main/assembly/components/all.xml index d8a13d8..629312e 100644 --- a/openmeetings-server/src/main/assembly/components/all.xml +++ b/openmeetings-server/src/main/assembly/components/all.xml @@ -36,6 +36,18 @@ <exclude>lib/commons-io*.jar</exclude> <exclude>lib/commons-lang3*</exclude> <exclude>lib/spring*.jar</exclude> + <!-- Tika related jars are in lib folder + <exclude>lib/apache-mime4j*</exclude> + <exclude>lib/boilerpipe*</exclude> + <exclude>lib/jempbox*</exclude> + <exclude>lib/juniversalchardet*</exclude> + <exclude>lib/mchange-commons*</exclude> + <exclude>lib/metadata-extractor*</exclude> + <exclude>lib/pdfbox*</exclude> + <exclude>lib/rome*</exclude> + <exclude>lib/tagsoup*</exclude> + <exclude>lib/tika*</exclude> + end of Tika libs --> <exclude>plugins/websocket*.jar</exclude> <exclude>plugins/tomcat-embed*.jar</exclude> <!-- added for Red5 lib substitution --> <exclude>plugins/ecj*.jar</exclude> <!-- added for Red5 lib substitution --> @@ -111,7 +123,10 @@ <exclude>WEB-INF/classes/META-INF/**</exclude> <exclude>WEB-INF/lib/asm*</exclude> <exclude>WEB-INF/lib/aopalliance*</exclude> + <exclude>WEB-INF/lib/bcprov-*</exclude> <exclude>WEB-INF/lib/cglib-nodep*</exclude> + <exclude>WEB-INF/lib/jcl-over-slf4j*</exclude> + <exclude>WEB-INF/lib/jul-to-slf4j*</exclude> <exclude>WEB-INF/lib/slf4j-*</exclude> <exclude>WEB-INF/lib/ecj-*</exclude> <!-- added for Red5 lib substitution --> <exclude>WEB-INF/lib/tomcat-*</exclude> @@ -123,7 +138,69 @@ <exclude>WEB-INF/lib/commons-codec*</exclude> <exclude>WEB-INF/lib/commons-collections-*</exclude> <exclude>WEB-INF/lib/commons-lang3*</exclude> - <exclude>WEB-INF/lib/jcl-over-slf4j*</exclude> + <exclude>WEB-INF/lib/commons-compress*</exclude> + <exclude>WEB-INF/lib/httpclient*</exclude> + <exclude>WEB-INF/lib/httpcore*</exclude> + <exclude>WEB-INF/lib/xmpcore*</exclude> + <exclude>WEB-INF/lib/quartz*</exclude> + <!-- Tika related jars are in lib folder --> + <exclude>WEB-INF/lib/apache-mime4j*</exclude> + <exclude>WEB-INF/lib/boilerpipe*</exclude> + <exclude>WEB-INF/lib/ehcache-core*</exclude> + <exclude>WEB-INF/lib/jempbox*</exclude> + <exclude>WEB-INF/lib/juniversalchardet*</exclude> + <exclude>WEB-INF/lib/mchange-commons*</exclude> + <exclude>WEB-INF/lib/metadata-extractor*</exclude> + <exclude>WEB-INF/lib/pdfbox*</exclude> + <exclude>WEB-INF/lib/rome*</exclude> + <exclude>WEB-INF/lib/tagsoup*</exclude> + <exclude>WEB-INF/lib/tika*</exclude> + <exclude>WEB-INF/lib/animal-sniffer-annotations*</exclude> + <exclude>WEB-INF/lib/bcmail-*</exclude> + <exclude>WEB-INF/lib/bcpkix-*</exclude> + <exclude>WEB-INF/lib/bzip2-*</exclude> + <exclude>WEB-INF/lib/cdm-*</exclude> + <exclude>WEB-INF/lib/commons-csv-*</exclude> + <exclude>WEB-INF/lib/commons-exec-*</exclude> + <exclude>WEB-INF/lib/commons-vfs2-*</exclude> + <exclude>WEB-INF/lib/curvesapi-*</exclude> + <exclude>WEB-INF/lib/error_prone_annotations-*</exclude> + <exclude>WEB-INF/lib/geoapi-*</exclude> + <exclude>WEB-INF/lib/grib-*</exclude> + <exclude>WEB-INF/lib/guava-*</exclude> + <exclude>WEB-INF/lib/httpmime-*</exclude> + <exclude>WEB-INF/lib/httpservices-*</exclude> + <exclude>WEB-INF/lib/isoparser-*</exclude> + <exclude>WEB-INF/lib/j2objc-annotations-*</exclude> + <exclude>WEB-INF/lib/jackcess-*</exclude> + <exclude>WEB-INF/lib/jackson-core-*</exclude> + <exclude>WEB-INF/lib/java-libpst-*</exclude> + <exclude>WEB-INF/lib/jcip-annotations-*</exclude> + <exclude>WEB-INF/lib/jcommander-*</exclude> + <exclude>WEB-INF/lib/jdom2-*</exclude> + <exclude>WEB-INF/lib/jhighlight-*</exclude> + <exclude>WEB-INF/lib/jmatio-*</exclude> + <exclude>WEB-INF/lib/jna-*</exclude> + <exclude>WEB-INF/lib/joda-time-*</exclude> + <exclude>WEB-INF/lib/json-*</exclude> + <exclude>WEB-INF/lib/json-simple-*</exclude> + <exclude>WEB-INF/lib/jsoup-*</exclude> + <exclude>WEB-INF/lib/jsr-275-*</exclude> + <exclude>WEB-INF/lib/jsr305-*</exclude> + <exclude>WEB-INF/lib/junrar-*</exclude> + <exclude>WEB-INF/lib/maven-scm-*</exclude> + <exclude>WEB-INF/lib/netcdf4-*</exclude> + <exclude>WEB-INF/lib/opennlp-tools-*</exclude> + <exclude>WEB-INF/lib/plexus-utils-*</exclude> + <exclude>WEB-INF/lib/poi-*</exclude> + <exclude>WEB-INF/lib/protobuf-java-*</exclude> + <exclude>WEB-INF/lib/regexp-*</exclude> + <exclude>WEB-INF/lib/sentiment-analysis-parser-*</exclude> + <exclude>WEB-INF/lib/sis-*</exclude> + <exclude>WEB-INF/lib/udunits-*</exclude> + <exclude>WEB-INF/lib/vorbis-java-*</exclude> + <exclude>WEB-INF/lib/xmlbeans-*</exclude> + <exclude>WEB-INF/lib/xz-*</exclude> </excludes> </fileSet> <fileSet> @@ -143,6 +220,53 @@ <include>commons-fileupload*</include> <include>commons-io*</include> <include>commons-lang3*</include> + <!-- Tika related jars are in lib folder --> + <include>tika-*</include> + <include>bcmail-*</include> + <include>bcpkix-*</include> + <include>bzip2-*</include> + <include>cdm-*</include> + <include>commons-csv-*</include> + <include>commons-exec-*</include> + <include>commons-vfs2-*</include> + <include>curvesapi-*</include> + <include>error_prone_annotations-*</include> + <include>geoapi-*</include> + <include>grib-*</include> + <include>guava-*</include> + <include>httpmime-*</include> + <include>httpservices-*</include> + <include>isoparser-*</include> + <include>j2objc-annotations-*</include> + <include>jackcess-*</include> + <include>jackson-core-*</include> + <include>java-libpst-*</include> + <include>jcip-annotations-*</include> + <include>jcommander-*</include> + <include>jdom2-*</include> + <include>jhighlight-*</include> + <include>jmatio-*</include> + <include>jna-*</include> + <include>joda-time-*</include> + <include>json-*</include> + <include>json-simple-*</include> + <include>jsoup-*</include> + <include>jsr-275-*</include> + <include>jsr305-*</include> + <include>junrar-*</include> + <include>maven-scm-*</include> + <include>netcdf4-*</include> + <include>opennlp-tools-*</include> + <include>plexus-utils-*</include> + <include>poi-*</include> + <include>protobuf-java-*</include> + <include>regexp-*</include> + <include>sentiment-analysis-parser-*</include> + <include>sis-*</include> + <include>udunits-*</include> + <include>vorbis-java-*</include> + <include>xmlbeans-*</include> + <include>xz-*</include> </includes> </fileSet> <fileSet> http://git-wip-us.apache.org/repos/asf/openmeetings/blob/f9eef4df/openmeetings-util/pom.xml ---------------------------------------------------------------------- diff --git a/openmeetings-util/pom.xml b/openmeetings-util/pom.xml index 0a93366..a639c2f 100644 --- a/openmeetings-util/pom.xml +++ b/openmeetings-util/pom.xml @@ -123,5 +123,10 @@ <groupId>org.apache.wicket</groupId> <artifactId>wicket-extensions</artifactId> </dependency> + <dependency> + <groupId>org.apache.tika</groupId> + <artifactId>tika-parsers</artifactId> + <version>1.16</version> + </dependency> </dependencies> </project> http://git-wip-us.apache.org/repos/asf/openmeetings/blob/f9eef4df/openmeetings-util/src/main/java/org/apache/openmeetings/util/StoredFile.java ---------------------------------------------------------------------- diff --git a/openmeetings-util/src/main/java/org/apache/openmeetings/util/StoredFile.java b/openmeetings-util/src/main/java/org/apache/openmeetings/util/StoredFile.java index 117fd5a..6a7b3dc 100644 --- a/openmeetings-util/src/main/java/org/apache/openmeetings/util/StoredFile.java +++ b/openmeetings-util/src/main/java/org/apache/openmeetings/util/StoredFile.java @@ -34,7 +34,9 @@ import java.util.HashSet; import java.util.LinkedHashSet; import java.util.Set; -import org.apache.tika.Tika; +import org.apache.tika.config.TikaConfig; +import org.apache.tika.exception.TikaException; +import org.apache.tika.io.TikaInputStream; import org.apache.tika.metadata.Metadata; import org.apache.tika.mime.MediaType; import org.apache.wicket.util.string.Strings; @@ -57,6 +59,7 @@ public class StoredFile { private static final Set<MediaType> CHART_TYPES = new HashSet<>(); private static final Set<MediaType> AS_IS_TYPES = new HashSet<>(Arrays.asList(MIME_JPG)); private static final String ACCEPT_STRING; + private static TikaConfig tika; static { Set<MediaType> types = new LinkedHashSet<>(); types.addAll(CONVERT_TYPES); @@ -71,6 +74,12 @@ public class StoredFile { sb.append(',').append(mt.toString()); } ACCEPT_STRING = sb.toString(); + try { + tika = new TikaConfig(); + } catch (IOException | TikaException e) { + log.error("Unexpected exception while initializing TIKA", e); + throw new RuntimeException(e); + } } private String name; @@ -95,21 +104,20 @@ public class StoredFile { } } - private void init(String name, String ext, InputStream is) { - if (Strings.isEmpty(ext)) { - int idx = name.lastIndexOf('.'); - this.name = idx < 0 ? name : name.substring(0, idx); - this.ext = getFileExt(name); + private void init(String _name, String _ext, InputStream is) { + if (Strings.isEmpty(_ext)) { + int idx = _name.lastIndexOf('.'); + name = idx < 0 ? _name : _name.substring(0, idx); + ext = getFileExt(_name); } else { - this.name = name; - this.ext = ext.toLowerCase(); + name = _name; + ext = _ext.toLowerCase(); } - Tika tika = new Tika(); Metadata md = new Metadata(); md.add(RESOURCE_NAME_KEY, String.format(FILE_NAME_FMT, name, ext)); try { - mime = MediaType.parse(tika.detect(is, md)); - } catch (Exception e) { + mime = tika.getDetector().detect(is == null ? null : TikaInputStream.get(is), md); + } catch (Throwable e) { mime = null; log.error("Unexpected exception while detecting mime type", e); } http://git-wip-us.apache.org/repos/asf/openmeetings/blob/f9eef4df/openmeetings-util/src/test/java/org/apache/openmeetings/util/TestStoredFile.java ---------------------------------------------------------------------- diff --git a/openmeetings-util/src/test/java/org/apache/openmeetings/util/TestStoredFile.java b/openmeetings-util/src/test/java/org/apache/openmeetings/util/TestStoredFile.java index ac74054..7e278a6 100644 --- a/openmeetings-util/src/test/java/org/apache/openmeetings/util/TestStoredFile.java +++ b/openmeetings-util/src/test/java/org/apache/openmeetings/util/TestStoredFile.java @@ -21,6 +21,7 @@ package org.apache.openmeetings.util; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import java.io.IOException; import java.io.InputStream; import org.junit.Test; @@ -76,4 +77,19 @@ public class TestStoredFile { assertFalse(String.format("Files of type '%s' should NOT be treated as Video", ext), sf.isVideo()); } } + + private static void fileOfficeTest(String path) throws IOException { + try (InputStream is = TestStoredFile.class.getResourceAsStream(path)) { + StoredFile sf = new StoredFile(path, is); + assertTrue(String.format("Files of type '%s' should be treated as Convertible", sf.getExt()), sf.isOffice()); + assertFalse(String.format("Files of type '%s' should NOT be treated as Video", sf.getExt()), sf.isVideo()); + } + } + + @Test + public void testOffice1() throws IOException { + for (String path : new String[] {"/ODFtest.odt", "/ODFtest.ods"}) { + fileOfficeTest(path); + } + } } http://git-wip-us.apache.org/repos/asf/openmeetings/blob/f9eef4df/openmeetings-util/src/test/resources/ODFtest.odt ---------------------------------------------------------------------- diff --git a/openmeetings-util/src/test/resources/ODFtest.odt b/openmeetings-util/src/test/resources/ODFtest.odt new file mode 100644 index 0000000..611dcdf Binary files /dev/null and b/openmeetings-util/src/test/resources/ODFtest.odt differ http://git-wip-us.apache.org/repos/asf/openmeetings/blob/f9eef4df/openmeetings-util/src/test/resources/ODStest.ods ---------------------------------------------------------------------- diff --git a/openmeetings-util/src/test/resources/ODStest.ods b/openmeetings-util/src/test/resources/ODStest.ods new file mode 100644 index 0000000..753246f Binary files /dev/null and b/openmeetings-util/src/test/resources/ODStest.ods differ http://git-wip-us.apache.org/repos/asf/openmeetings/blob/f9eef4df/openmeetings-web/src/main/java/org/apache/openmeetings/web/room/sidebar/UploadDialog.java ---------------------------------------------------------------------- diff --git a/openmeetings-web/src/main/java/org/apache/openmeetings/web/room/sidebar/UploadDialog.java b/openmeetings-web/src/main/java/org/apache/openmeetings/web/room/sidebar/UploadDialog.java index 19148bf..ae1884a 100644 --- a/openmeetings-web/src/main/java/org/apache/openmeetings/web/room/sidebar/UploadDialog.java +++ b/openmeetings-web/src/main/java/org/apache/openmeetings/web/room/sidebar/UploadDialog.java @@ -30,13 +30,11 @@ import org.apache.openmeetings.core.data.file.FileProcessor; import org.apache.openmeetings.db.dao.file.FileItemLogDao; import org.apache.openmeetings.db.entity.file.BaseFileItem; import org.apache.openmeetings.db.entity.file.FileItem; -import org.apache.openmeetings.util.StoredFile; import org.apache.openmeetings.util.process.ProcessResult; import org.apache.openmeetings.util.process.ProcessResultList; import org.apache.openmeetings.web.app.Application; import org.apache.openmeetings.web.room.RoomPanel; import org.apache.openmeetings.web.util.upload.BootstrapFileUploadBehavior; -import org.apache.wicket.AttributeModifier; import org.apache.wicket.ajax.AjaxRequestTarget; import org.apache.wicket.ajax.form.AjaxFormSubmitBehavior; import org.apache.wicket.ajax.form.OnChangeAjaxBehavior; @@ -118,7 +116,6 @@ public class UploadDialog extends AbstractFormDialog<String> { return new ArrayList<>(); } })); - uploadField.add(AttributeModifier.append("accept", StoredFile.getAcceptAttr())); Form<String> nameForm = new Form<>("name-form"); fileName = new HiddenField<>("name", Model.of("")); fileName.add(new AjaxFormSubmitBehavior(nameForm, "change") { http://git-wip-us.apache.org/repos/asf/openmeetings/blob/f9eef4df/openmeetings-web/src/test/java/org/apache/openmeetings/util/TestStoredFile.java ---------------------------------------------------------------------- diff --git a/openmeetings-web/src/test/java/org/apache/openmeetings/util/TestStoredFile.java b/openmeetings-web/src/test/java/org/apache/openmeetings/util/TestStoredFile.java index 7091eff..14e97e0 100644 --- a/openmeetings-web/src/test/java/org/apache/openmeetings/util/TestStoredFile.java +++ b/openmeetings-web/src/test/java/org/apache/openmeetings/util/TestStoredFile.java @@ -26,7 +26,6 @@ import java.io.FileNotFoundException; import java.io.IOException; import org.apache.openmeetings.AbstractJUnitDefaults; -import org.apache.openmeetings.util.StoredFile; import org.junit.Test; public class TestStoredFile extends AbstractJUnitDefaults {
