Filippo Giunchedi has submitted this change and it was merged.

Change subject: Upgrade to 0.1.21
......................................................................


Upgrade to 0.1.21

Change-Id: I25b6f9ce199159196e9c0cc92804c4e6926828a5
---
M debian/40-wikimedia.conf
M debian/changelog
M debian/control
M debian/rules
M setup.py
M tests/integration/__init__.py
A tests/integration/originals/Jeremy_Bentham.pdf
A tests/integration/originals/Television.svg
A tests/integration/test_https_loader.py
A tests/integration/test_huge_video.py
D tests/integration/test_rsvg.py
M tests/integration/test_types.py
M tests/integration/test_vips.py
A tests/integration/test_vips_https_loader.py
A tests/integration/thumbnails/200px-Television.svg.png
A tests/integration/thumbnails/320px-seek=0-Borsch_01.webm.jpg
A tests/integration/thumbnails/page19-400px-Jeremy_Bentham.pdf.jpg
A tests/integration/thumbnails/page259-400px-Il_cavallarizzo.djvu.jpg
A tests/integration/thumbnails/page440-400px-Zibaldone_di_pensieri_V.djvu.jpg
M wikimedia_thumbor/engine/__init__.py
M wikimedia_thumbor/engine/djvu/djvu.py
M wikimedia_thumbor/engine/ghostscript/ghostscript.py
M wikimedia_thumbor/engine/imagemagick/imagemagick.py
M wikimedia_thumbor/engine/proxy/proxy.py
M wikimedia_thumbor/engine/svg/svg.py
M wikimedia_thumbor/engine/vips/vips.py
M wikimedia_thumbor/engine/xcf/xcf.py
M wikimedia_thumbor/exiftool_runner/__init__.py
A wikimedia_thumbor/filter/format/__init__.py
A wikimedia_thumbor/filter/format/format.py
M wikimedia_thumbor/handler/images/images.py
A wikimedia_thumbor/loader/https/__init__.py
M wikimedia_thumbor/loader/video/__init__.py
M wikimedia_thumbor/result_storage/swift/swift.py
M wikimedia_thumbor/storage/request/request.py
35 files changed, 647 insertions(+), 347 deletions(-)

Approvals:
  Filippo Giunchedi: Looks good to me, approved
  jenkins-bot: Verified



diff --git a/debian/40-wikimedia.conf b/debian/40-wikimedia.conf
index 4c34cf4..e0ea4c5 100644
--- a/debian/40-wikimedia.conf
+++ b/debian/40-wikimedia.conf
@@ -1,20 +1,6 @@
-CHROMA_SUBSAMPLING = '4:2:0'
-QUALITY_LOW = 40
-DEFAULT_FILTERS_JPEG = 'conditional_sharpen(0.0,0.8,1.0,0.0,0.85)'
-
-COMMUNITY_EXTENSIONS = [
-    'wikimedia_thumbor.handler.multi',
-    'wikimedia_thumbor.handler.images'
-]
-
 EXIFTOOL_PATH = '/usr/bin/exiftool'
-EXIF_FIELDS_TO_KEEP = [ 'Artist', 'Copyright', 'Description' ]
 
-SUBPROCESS_USE_TIMEOUT = True
 SUBPROCESS_TIMEOUT_PATH = '/usr/bin/timeout'
-SUBPROCESS_TIMEOUT = 60
-
-VIPS_ENGINE_MIN_PIXELS = 10000000
 
 RSVG_CONVERT_PATH = '/usr/bin/rsvg-convert'
 
@@ -27,19 +13,3 @@
 GHOSTSCRIPT_PATH = '/usr/bin/gs'
 
 VIPS_PATH = '/usr/bin/vips'
-
-PROXY_ENGINE_ENGINES = [
-    ('wikimedia_thumbor.engine.svg', ['svg']),
-    ('wikimedia_thumbor.engine.xcf', ['xcf']),
-    ('wikimedia_thumbor.engine.djvu', ['djvu']),
-    ('wikimedia_thumbor.engine.vips', ['tiff', 'png']),
-    ('wikimedia_thumbor.engine.tiff', ['tiff']),
-    ('wikimedia_thumbor.engine.ghostscript', ['pdf']),
-    ('wikimedia_thumbor.engine.imagemagick', ['jpg', 'png']),
-]
-
-PROXY_LOADER_LOADERS = [
-    'wikimedia_thumbor.loader.video'
-]
-
-SLOW_PROCESSING_LIMIT = 30000
diff --git a/debian/changelog b/debian/changelog
index f693ba7..de5b7c9 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,16 @@
+python-thumbor-wikimedia (0.1.21-1) jessie-wikimedia; urgency=low
+
+  * New upstream release
+  * debian/control
+    - remove dependency on python-djvu, python-gi, python-cairosvg, python-lxml
+  * debian/rules
+    - ignore new tests that require network connectivity
+  * debian/40-wikimedia.conf
+    - remove all config except paths, it makes more sense to define the
+      rest in puppet
+
+ -- Gilles Dubuc <gil...@wikimedia.org>  Tue, 20 Sep 2016 10:16:00 +0000
+
 python-thumbor-wikimedia (0.1.19-1) jessie-wikimedia; urgency=low
 
   * New upstream release
diff --git a/debian/control b/debian/control
index e62441d..bc50c86 100644
--- a/debian/control
+++ b/debian/control
@@ -13,11 +13,7 @@
                librsvg2-bin,
                libvips-tools,
                python-all,
-               python-cairosvg,
-               python-djvu,
-               python-gi,
                python-libthumbor (>= 1.3.2),
-               python-lxml,
                python-nose,
                python-pyssim,
                python-setuptools,
@@ -31,7 +27,7 @@
 
 Package: python-thumbor-wikimedia
 Architecture: all
-Depends: ${misc:Depends}, ${python:Depends}, djvulibre-bin, ffmpeg, 
ghostscript, gifsicle, libimage-exiftool-perl, librsvg2-bin, libvips-tools, 
python-cairosvg, python-djvu, python-gi, python-libthumbor (>= 1.3.2), 
python-lxml, python-swiftclient (>= 2.5.0), python-thumbor-community-core, 
python-wand, thumbor (>= 6.0.1), xcftools
+Depends: ${misc:Depends}, ${python:Depends}, djvulibre-bin, ffmpeg, 
ghostscript, gifsicle, libimage-exiftool-perl, librsvg2-bin, libvips-tools, 
python-libthumbor (>= 1.3.2), python-swiftclient (>= 2.5.0), 
python-thumbor-community-core, python-wand, thumbor (>= 6.0.1), xcftools
 Suggests: cgroup-tools
 Provides: ${python:Provides}
 Description: Thumbor wikimedia extensions
diff --git a/debian/rules b/debian/rules
index c659663..bc5ee54 100755
--- a/debian/rules
+++ b/debian/rules
@@ -3,7 +3,7 @@
 export PYBUILD_TEST_NOSE=1
 export PYBUILD_BEFORE_TEST=cp -R tests {build_dir}
 export PYBUILD_AFTER_TEST=rm -rf {build_dir}/tests
-export PYBUILD_TEST_ARGS=-s tests/ --ignore-files=test_proxy_loader.py
+export PYBUILD_TEST_ARGS=-s tests/ 
--ignore-files='test_proxy_loader.py|test_huge_video.py|test_https_loader.py|test_vips_https_loader.py'
 
 %:
        dh $@ --with python2 --buildsystem=pybuild
diff --git a/setup.py b/setup.py
index 575b4d9..8f80e64 100644
--- a/setup.py
+++ b/setup.py
@@ -11,7 +11,7 @@
 
 setup(
     name='wikimedia_thumbor',
-    version='0.1.19',
+    version='0.1.21',
     url='https://phabricator.wikimedia.org/diffusion/THMBREXT/',
     license='MIT',
     author='Gilles Dubuc, Wikimedia Foundation',
@@ -21,11 +21,7 @@
     zip_safe=False,
     platforms='any',
     install_requires=[
-        'cairosvg',
-        'gi',
         'libthumbor>=1.3.2',
-        'lxml',
-        'python-djvulibre',
         'python-swiftclient',
         'thumbor>=6.0.1',
         'wand'
diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py
index 5cbf1d3..648028a 100644
--- a/tests/integration/__init__.py
+++ b/tests/integration/__init__.py
@@ -1,4 +1,3 @@
-import logging
 import platform
 import os.path
 
@@ -45,6 +44,8 @@
         cfg.FFPROBE_PATH = which('ffprobe')
         cfg.XCF2PNG_PATH = which('xcf2png')
         cfg.GHOSTSCRIPT_PATH = which('gs')
+        cfg.DDJVU_PATH = which('ddjvu')
+        cfg.RSVG_CONVERT_PATH = which('rsvg-convert')
         timeout = which(
             'gtimeout' if platform.system() == 'Darwin' else 'timeout'
         )
@@ -84,11 +85,11 @@
 
         cfg.FILTERS = [
             'wikimedia_thumbor.filter.conditional_sharpen',
+            'wikimedia_thumbor.filter.format',
             'wikimedia_thumbor.filter.lang',
             'wikimedia_thumbor.filter.page',
             'wikimedia_thumbor.filter.crop',
             'wikimedia_thumbor.filter.flip',
-            'thumbor.filters.format',
             'thumbor.filters.quality',
             'thumbor.filters.rotate'
         ]
diff --git a/tests/integration/originals/Jeremy_Bentham.pdf 
b/tests/integration/originals/Jeremy_Bentham.pdf
new file mode 100644
index 0000000..3170d57
--- /dev/null
+++ b/tests/integration/originals/Jeremy_Bentham.pdf
Binary files differ
diff --git a/tests/integration/originals/Television.svg 
b/tests/integration/originals/Television.svg
new file mode 100644
index 0000000..1316dc0
--- /dev/null
+++ b/tests/integration/originals/Television.svg
@@ -0,0 +1,112 @@
+<?xml version="1.0" encoding="iso-8859-1"?>
+<!-- Generator: Adobe Illustrator 10.0.3, SVG Export Plug-In . SVG Version: 
3.0.0 Build 77)  -->
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.0//EN"    
"http://www.w3.org/TR/2001/REC-SVG-20010904/DTD/svg10.dtd"; [
+       <!ENTITY ns_flows "http://ns.adobe.com/Flows/1.0/";>
+       <!ENTITY ns_svg "http://www.w3.org/2000/svg";>
+       <!ENTITY ns_xlink "http://www.w3.org/1999/xlink";>
+]>
+<svg  xmlns="&ns_svg;" xmlns:xlink="&ns_xlink;" 
xmlns:a="http://ns.adobe.com/AdobeSVGViewerExtensions/3.0/";
+        width="128" height="128" viewBox="0 0 128 128" overflow="visible" 
enable-background="new 0 0 128 128" xml:space="preserve">
+       <g id="Layer_1">
+               <g>
+                       <g>
+                               <path fill="#333333" 
d="M118.442,15.26c-1.162,1.161-3.046,1.161-4.207,0s-1.161-3.045,0-4.206c1.161-1.162,3.045-1.162,4.207,0
+                                       
C119.604,12.215,119.604,14.099,118.442,15.26z"/>
+                               <path fill="#333333" 
d="M86.631,44.443l-1.577-1.577l29.708-29.709l1.577,1.577L86.631,44.443z"/>
+                       </g>
+                       <g>
+                               <path fill="#333333" 
d="M89.872,3.186c0,1.643-1.333,2.976-2.975,2.976s-2.974-1.333-2.974-2.976
+                                       
c0-1.642,1.332-2.973,2.974-2.973C88.54,0.213,89.872,1.544,89.872,3.186z"/>
+                               <path fill="#333333" 
d="M88.014,46.316h-2.231V4.302h2.231V46.316z"/>
+                       </g>
+                       <path fill="#333333" 
d="M112.619,125.717c0,1.144-0.919,2.07-2.052,2.07H10.736c-1.133,0-2.05-0.927-2.05-2.07l0.478-84.611
+                               
c0-1.145,0.918-2.068,2.05-2.068h99.832c1.132,0,2.051,0.924,2.051,2.068L112.619,125.717z"/>
+                       <linearGradient id="XMLID_1_" 
gradientUnits="userSpaceOnUse" x1="60.8911" y1="29.6812" x2="60.8911" 
y2="99.7845">
+                               <stop  offset="0" style="stop-color:#FFFFFF"/>
+                               <stop  offset="0.3678" 
style="stop-color:#9D9D9D"/>
+                               <stop  offset="0.8052" 
style="stop-color:#2D2D2D"/>
+                               <stop  offset="1" style="stop-color:#000000"/>
+                               <a:midPointStop  offset="0" 
style="stop-color:#FFFFFF"/>
+                               <a:midPointStop  offset="0.479" 
style="stop-color:#FFFFFF"/>
+                               <a:midPointStop  offset="1" 
style="stop-color:#000000"/>
+                       </linearGradient>
+                       <path opacity="0.81" fill="url(#XMLID_1_)"  
a:adobe-blending-mode="screen" d="M110.616,124.004
+                               
c0,1.086-0.882,1.968-1.968,1.968H12.656c-1.087,0-1.968-0.882-1.968-1.968l0.478-81.006c0-1.088,0.882-1.969,1.969-1.969h95.991
+                               
c1.086,0,1.969,0.881,1.969,1.969L110.616,124.004z"/>
+                       <g>
+                               <linearGradient id="XMLID_2_" 
gradientUnits="userSpaceOnUse" x1="61.5024" y1="18.5396" x2="61.5024" 
y2="110.5096">
+                                       <stop  offset="0" 
style="stop-color:#000000"/>
+                                       <stop  offset="1" 
style="stop-color:#B5B5B5"/>
+                                       <a:midPointStop  offset="0" 
style="stop-color:#000000"/>
+                                       <a:midPointStop  offset="0.5" 
style="stop-color:#000000"/>
+                                       <a:midPointStop  offset="1" 
style="stop-color:#B5B5B5"/>
+                               </linearGradient>
+                               <path fill="url(#XMLID_2_)" 
d="M102.646,107.807c0,0.934-0.753,1.687-1.685,1.687h-78.92c-0.931,0-1.684-0.753-1.684-1.687
+                                       
V48.068c0-0.931,0.753-1.685,1.684-1.685h78.92c0.932,0,1.685,0.754,1.685,1.685V107.807z"/>
+                               
+                                       <radialGradient id="XMLID_3_" 
cx="58.7769" cy="122.3877" r="52.3604" fx="58.7769" fy="122.3877" 
gradientUnits="userSpaceOnUse">
+                                       <stop  offset="0" 
style="stop-color:#B8BBDA"/>
+                                       <stop  offset="1" 
style="stop-color:#0D2D7D"/>
+                                       <a:midPointStop  offset="0" 
style="stop-color:#B8BBDA"/>
+                                       <a:midPointStop  offset="0.5" 
style="stop-color:#B8BBDA"/>
+                                       <a:midPointStop  offset="1" 
style="stop-color:#0D2D7D"/>
+                               </radialGradient>
+                               <path fill="url(#XMLID_3_)" 
d="M100.328,106.127c0,0.876-0.711,1.587-1.589,1.587H24.266c-0.878,0-1.589-0.711-1.589-1.587
+                                       
V49.75c0-0.877,0.711-1.589,1.589-1.589h74.474c0.878,0,1.589,0.712,1.589,1.589V106.127z"/>
+                       </g>
+                       
+                               <radialGradient id="XMLID_4_" cx="36.2217" 
cy="60.2163" r="57.7388" fx="36.2217" fy="60.2163" 
gradientTransform="matrix(1.0211 0 0 0.9264 27.8623 -8.8409)" 
gradientUnits="userSpaceOnUse">
+                               <stop  offset="0" style="stop-color:#B8BBDA"/>
+                               <stop  offset="1" style="stop-color:#0D2D7D"/>
+                               <a:midPointStop  offset="0" 
style="stop-color:#B8BBDA"/>
+                               <a:midPointStop  offset="0.5" 
style="stop-color:#B8BBDA"/>
+                               <a:midPointStop  offset="1" 
style="stop-color:#0D2D7D"/>
+                       </radialGradient>
+                       <path opacity="0.78" fill="url(#XMLID_4_)" 
d="M23.57,59.344c0,11.951,17.039,21.64,38.059,21.64
+                               
c21.018,0,38.057-9.688,38.057-21.64c0-3.993-1.909-7.729-5.227-10.94H28.798C25.479,51.614,23.57,55.351,23.57,59.344z"/>
+                       <g>
+                               <linearGradient id="XMLID_5_" 
gradientUnits="userSpaceOnUse" x1="98.6885" y1="125.7139" x2="98.6885" 
y2="118.7914">
+                                       <stop  offset="0" 
style="stop-color:#FFFFFF"/>
+                                       <stop  offset="0.8483" 
style="stop-color:#00CD00"/>
+                                       <stop  offset="1" 
style="stop-color:#00BD00"/>
+                                       <a:midPointStop  offset="0" 
style="stop-color:#FFFFFF"/>
+                                       <a:midPointStop  offset="0.5" 
style="stop-color:#FFFFFF"/>
+                                       <a:midPointStop  offset="0.8483" 
style="stop-color:#00CD00"/>
+                                       <a:midPointStop  offset="0.5" 
style="stop-color:#00CD00"/>
+                                       <a:midPointStop  offset="1" 
style="stop-color:#00BD00"/>
+                               </linearGradient>
+                               <path fill="url(#XMLID_5_)" 
d="M102.397,118.534c0,2.048-1.659,3.708-3.708,3.708s-3.71-1.66-3.71-3.708
+                                       
c0-2.049,1.661-3.71,3.71-3.71S102.397,116.485,102.397,118.534z"/>
+                               <linearGradient id="XMLID_6_" 
gradientUnits="userSpaceOnUse" x1="94.1318" y1="111.6963" x2="96.9031" 
y2="115.8532">
+                                       <stop  offset="0.0112" 
style="stop-color:#009500"/>
+                                       <stop  offset="0.2978" 
style="stop-color:#009000"/>
+                                       <stop  offset="1" 
style="stop-color:#005B00"/>
+                                       <a:midPointStop  offset="0.0112" 
style="stop-color:#009500"/>
+                                       <a:midPointStop  offset="0.5" 
style="stop-color:#009500"/>
+                                       <a:midPointStop  offset="0.2978" 
style="stop-color:#009000"/>
+                                       <a:midPointStop  offset="0.5" 
style="stop-color:#009000"/>
+                                       <a:midPointStop  offset="1" 
style="stop-color:#005B00"/>
+                               </linearGradient>
+                               <path fill="url(#XMLID_6_)" 
d="M94.804,118.534c0,2.142,1.742,3.884,3.886,3.884c2.143,0,3.886-1.742,3.886-3.884
+                                       
c0-2.144-1.743-3.888-3.886-3.888C96.546,114.646,94.804,116.391,94.804,118.534z 
M95.157,118.534
+                                       
c0-1.949,1.585-3.533,3.532-3.533s3.532,1.584,3.532,3.533c0,1.946-1.585,3.532-3.532,3.532S95.157,120.48,95.157,118.534z"/>
+                               <linearGradient id="XMLID_7_" 
gradientUnits="userSpaceOnUse" x1="98.5879" y1="115.1816" x2="98.5879" 
y2="119.1344">
+                                       <stop  offset="0" 
style="stop-color:#FFFFFF"/>
+                                       <stop  offset="0.1147" 
style="stop-color:#E9FAE9"/>
+                                       <stop  offset="0.3539" 
style="stop-color:#B0EDB0"/>
+                                       <stop  offset="0.6936" 
style="stop-color:#57D957"/>
+                                       <stop  offset="1" 
style="stop-color:#00C500"/>
+                                       <a:midPointStop  offset="0" 
style="stop-color:#FFFFFF"/>
+                                       <a:midPointStop  offset="0.5424" 
style="stop-color:#FFFFFF"/>
+                                       <a:midPointStop  offset="1" 
style="stop-color:#00C500"/>
+                               </linearGradient>
+                               <path fill="url(#XMLID_7_)" 
d="M98.706,118.675c1.291-0.954,2.505-1.275,3.038-1.378c-0.513-1.155-1.671-1.961-3.016-1.961
+                                       
c-1.82,0-3.297,1.475-3.297,3.295c0,0.287,0.036,0.563,0.104,0.828C96.07,119.559,97.377,119.657,98.706,118.675z"/>
+                       </g>
+                       <path opacity="0.81" fill="#666666" 
d="M77.539,120.482h-36.61v6.461h36.61V120.482z"/>
+                       <path 
d="M77.539,120.109H40.556v7.205h37.356v-7.205H77.539z 
M77.169,120.854c0,0.619,0,5.097,0,5.719
+                               
c-0.722,0-35.148,0-35.869,0c0-0.622,0-5.1,0-5.719C42.021,120.854,76.447,120.854,77.169,120.854z"/>
+               </g>
+               <path fill="none" d="M128,128H0V0h128V128z"/>
+       </g>
+</svg>
diff --git a/tests/integration/test_https_loader.py 
b/tests/integration/test_https_loader.py
new file mode 100644
index 0000000..c4dea8e
--- /dev/null
+++ b/tests/integration/test_https_loader.py
@@ -0,0 +1,106 @@
+import platform
+
+from . import WikimediaTestCase
+
+
+class WikimediaHttpsLoaderTest(WikimediaTestCase):
+    def get_config(self):
+        cfg = super(WikimediaHttpsLoaderTest, self).get_config()
+        cfg.LOADER = 'wikimedia_thumbor.loader.proxy'
+        cfg.HTTP_LOADER_MAX_BODY_SIZE = 1024*1024*1024  # 1GB
+        cfg.PROXY_LOADER_LOADERS = [
+            'wikimedia_thumbor.loader.video',
+            'wikimedia_thumbor.loader.https'
+        ]
+
+        return cfg
+
+    def test_huge_djvu(self):
+        # We have to host this on testwiki because thumbor's default
+        # handler doesn't like commas and the original has one in its title
+        self.run_and_check_ssim_and_size(
+            'unsafe/400x/filters:page(440)/https://upload.wikimedia.org/'
+            + 'wikipedia/commons/e/ef/Zibaldone_di_pensieri_V.djvu',
+            'page440-400px-Zibaldone_di_pensieri_V.djvu.jpg',
+            # Mediawiki generates incorrect dimensions in this test case
+            # resulting in soft djvu thumbs
+            0.71,
+            1.2
+        )
+
+    def test_jpg(self):
+        self.run_and_check_ssim_and_size(
+            ('unsafe/400x/filters:conditional_sharpen(0.0,0.8,1.0,0.0,0.85)/'
+                'https://upload.wikimedia.org/wikipedia/commons/'
+                + '6/6d/Christophe_Henner_-_June_2016.jpg'),
+            '400px-Christophe_Henner_-_June_2016.jpg',
+            0.98,
+            1.0
+        )
+
+    def test_png(self):
+        self.run_and_check_ssim_and_size(
+            'unsafe/400x/https://upload.wikimedia.org/wikipedia/commons/'
+            + 'thumb/d/d6/1Mcolors.png/600px-1Mcolors.png',
+            '400px-1Mcolors.png',
+            0.99,
+            1.0
+        )
+
+    def test_tiff(self):
+        self.run_and_check_ssim_and_size(
+            'unsafe/400x/https://upload.wikimedia.org/wikipedia/commons/'
+            + '0/0e/0729.tiff',
+            'lossy-page1-400px-0729.tiff.jpg',
+            0.96,
+            1.0
+        )
+
+    def test_multipage_tiff(self):
+        self.run_and_check_ssim_and_size(
+            'unsafe/400x/filters:page(3)/https://upload.wikimedia.org/'
+            + 'wikipedia/commons/8/87/All_that_jazz.tif',
+            'lossy-page3-400px-All_that_jazz.tif.jpg',
+            0.99,
+            1.0
+        )
+
+    def test_multipage_tiff_with_out_of_bounds_page(self):
+        self.run_and_check_ssim_and_size(
+            'unsafe/400x/filters:page(500)/https://upload.wikimedia.org/'
+            + 'wikipedia/commons/8/87/All_that_jazz.tif',
+            'lossy-page1-400px-All_that_jazz.tif.jpg',
+            0.99,
+            1.0
+        )
+
+    def test_svg(self):
+        self.run_and_check_ssim_and_size(
+            'unsafe/200x/filters:lang(fr):format(png)/https://'
+            + 'upload.wikimedia.org/wikipedia/commons/3/39/Speech_bubbles.svg',
+            'langfr-200px-Speech_bubbles.svg.png',
+            (0.6 if platform.system() == 'Darwin' else 0.99),
+            1.1
+        )
+
+    def test_pdf(self):
+        self.run_and_check_ssim_and_size(
+            'unsafe/400x/filters:page(19)/https://upload.wikimedia.org/'
+            + 'wikipedia/commons/d/dc/Jeremy_Bentham%2C_A_Fragment_on_'
+            + 'Government_(1891).pdf',
+            'page19-400px-Jeremy_Bentham.pdf.jpg',
+            0.96,
+            1.0
+        )
+
+    def test_xcf(self):
+        self.run_and_check_ssim_and_size(
+            'unsafe/400x/filters:format(png)/https://upload.wikimedia.org/'
+            + 'wikipedia/commons/8/86/Janus.xcf',
+            '400px-Janus.xcf.png',
+            # Compression/sharpening artifacts explain the SSIM difference, but
+            # it's impossible to say when eyeballing if one if higher quality
+            # than the other
+            0.92,
+            1.01
+        )
diff --git a/tests/integration/test_huge_video.py 
b/tests/integration/test_huge_video.py
new file mode 100644
index 0000000..c648aa4
--- /dev/null
+++ b/tests/integration/test_huge_video.py
@@ -0,0 +1,18 @@
+from . import WikimediaTestCase
+
+
+class WikimediaHugeVideoTest(WikimediaTestCase):
+    def get_config(self):
+        cfg = super(WikimediaHugeVideoTest, self).get_config()
+        cfg.LOADER = 'wikimedia_thumbor.loader.video'
+
+        return cfg
+
+    def test_webm_with_fallback_seek(self):
+        self.run_and_check_ssim_and_size(
+            'unsafe/320x/filters:page(82)/https://upload.wikimedia.org/'
+            + 'wikipedia/commons/a/ab/Borsch_01.webm',
+            '320px-seek=0-Borsch_01.webm.jpg',
+            0.98,
+            1.0
+        )
diff --git a/tests/integration/test_rsvg.py b/tests/integration/test_rsvg.py
deleted file mode 100644
index 832fd26..0000000
--- a/tests/integration/test_rsvg.py
+++ /dev/null
@@ -1,21 +0,0 @@
-import platform
-
-from thumbor.utils import which
-
-from . import WikimediaTestCase
-
-
-class WikimediaRSVGTest(WikimediaTestCase):
-    def get_config(self):
-        cfg = super(WikimediaRSVGTest, self).get_config()
-        cfg.RSVG_CONVERT_PATH = which('rsvg-convert')
-        return cfg
-
-    def test_rsvg(self):
-        self.run_and_check_ssim_and_size(
-            'unsafe/200x/filters:lang(fr):format(png)/Speech_bubbles.svg',
-            'langfr-200px-Speech_bubbles.svg.png',
-            # Low score on OS X due to font differences
-            (0.6 if platform.system() == 'Darwin' else 0.99),
-            1.00
-        )
diff --git a/tests/integration/test_types.py b/tests/integration/test_types.py
index c9d687b..aa070e0 100644
--- a/tests/integration/test_types.py
+++ b/tests/integration/test_types.py
@@ -1,3 +1,5 @@
+import platform
+
 from . import WikimediaTestCase
 
 
@@ -34,7 +36,7 @@
     def test_djvu_with_out_of_bounds_page(self):
         self.run_and_check_ssim_and_size(
             'unsafe/400x/filters:page(500)/Il_cavallarizzo.djvu',
-            'page1-400px-Il_cavallarizzo.djvu.jpg',
+            'page259-400px-Il_cavallarizzo.djvu.jpg',
             # Mediawiki generates incorrect dimensions in this test case
             # resulting in soft djvu thumbs
             0.87,
@@ -45,8 +47,14 @@
         self.run_and_check_ssim_and_size(
             'unsafe/200x/filters:lang(fr):format(png)/Speech_bubbles.svg',
             'langfr-200px-Speech_bubbles.svg.png',
-            # Low score due to font differences
-            0.76,
+            (0.6 if platform.system() == 'Darwin' else 0.99),
+            1.1
+        )
+        self.run_and_check_ssim_and_size(
+            'unsafe/200x/filters:format(png)/Television.svg',
+            '200px-Television.svg.png',
+            # This file is only there to test SVG syntax
+            0.36,
             1.1
         )
 
@@ -54,7 +62,17 @@
         self.run_and_check_ssim_and_size(
             'unsafe/400x/filters:page(3)/Internationalisation.pdf',
             'page3-400px-Internationalisation.pdf.jpg',
-            0.95,
+            # Low score because framing is slightly different, and includes
+            # more content in the Thumbor case
+            0.87,
+            1.0
+        )
+
+    def test_pdf2(self):
+        self.run_and_check_ssim_and_size(
+            'unsafe/400x/filters:page(19)/Jeremy_Bentham.pdf',
+            'page19-400px-Jeremy_Bentham.pdf.jpg',
+            0.96,
             1.0
         )
 
@@ -62,7 +80,9 @@
         self.run_and_check_ssim_and_size(
             'unsafe/400x/Internationalisation.pdf',
             'page1-400px-Internationalisation.pdf.jpg',
-            0.95,
+            # Low score because framing is slightly different, and includes
+            # more content in the Thumbor case
+            0.86,
             1.0
         )
 
@@ -70,7 +90,9 @@
         self.run_and_check_ssim_and_size(
             'unsafe/400x/filters:page(500)/Internationalisation.pdf',
             'page1-400px-Internationalisation.pdf.jpg',
-            0.95,
+            # Low score because framing is slightly different, and includes
+            # more content in the Thumbor case
+            0.86,
             1.0
         )
 
diff --git a/tests/integration/test_vips.py b/tests/integration/test_vips.py
index 17604f7..df4982e 100644
--- a/tests/integration/test_vips.py
+++ b/tests/integration/test_vips.py
@@ -10,15 +10,15 @@
 
     def test_tiff(self):
         self.run_and_check_ssim_and_size(
-            'unsafe/400x/0729.tiff',
+            'unsafe/400x/filters:format(jpg)/0729.tiff',
             'lossy-page1-400px-0729.tiff.jpg',
-            0.96,
+            0.95,
             1.0
         )
 
     def test_multipage_tiff(self):
         self.run_and_check_ssim_and_size(
-            'unsafe/400x/filters:page(3)/All_that_jazz.tif',
+            'unsafe/400x/filters:format(jpg):page(3)/All_that_jazz.tif',
             'lossy-page3-400px-All_that_jazz.tif.jpg',
             0.98,
             1.0
@@ -26,7 +26,7 @@
 
     def test_multipage_tiff_without_page_filter(self):
         self.run_and_check_ssim_and_size(
-            'unsafe/400x/All_that_jazz.tif',
+            'unsafe/400x/filters:format(jpg)/All_that_jazz.tif',
             'lossy-page1-400px-All_that_jazz.tif.jpg',
             0.98,
             1.0
@@ -34,7 +34,7 @@
 
     def test_multipage_tiff_with_out_of_bounds_page(self):
         self.run_and_check_ssim_and_size(
-            'unsafe/400x/filters:page(500)/All_that_jazz.tif',
+            'unsafe/400x/filters:format(jpg):page(500)/All_that_jazz.tif',
             'lossy-page1-400px-All_that_jazz.tif.jpg',
             0.98,
             1.0
diff --git a/tests/integration/test_vips_https_loader.py 
b/tests/integration/test_vips_https_loader.py
new file mode 100644
index 0000000..cb0ee32
--- /dev/null
+++ b/tests/integration/test_vips_https_loader.py
@@ -0,0 +1,60 @@
+from . import WikimediaTestCase
+
+
+class WikimediaVipsHttpsLoaderTest(WikimediaTestCase):
+    def get_config(self):
+        cfg = super(WikimediaVipsHttpsLoaderTest, self).get_config()
+        cfg.VIPS_ENGINE_MIN_PIXELS = 0
+        cfg.LOADER = 'wikimedia_thumbor.loader.proxy'
+        cfg.HTTP_LOADER_MAX_BODY_SIZE = 1024*1024*1024  # 1GB
+        cfg.PROXY_LOADER_LOADERS = [
+            'wikimedia_thumbor.loader.video',
+            'wikimedia_thumbor.loader.https'
+        ]
+
+        return cfg
+
+    def test_tiff(self):
+        self.run_and_check_ssim_and_size(
+            'unsafe/400x/filters:format(jpg)/https://upload.wikimedia.org/'
+            + 'wikipedia/commons/0/0e/0729.tiff',
+            'lossy-page1-400px-0729.tiff.jpg',
+            0.95,
+            1.0
+        )
+
+    def test_multipage_tiff(self):
+        self.run_and_check_ssim_and_size(
+            'unsafe/400x/filters:format(jpg):page(3)/https://'
+            + 'upload.wikimedia.org/wikipedia/commons/8/87/All_that_jazz.tif',
+            'lossy-page3-400px-All_that_jazz.tif.jpg',
+            0.98,
+            1.0
+        )
+
+    def test_multipage_tiff_without_page_filter(self):
+        self.run_and_check_ssim_and_size(
+            'unsafe/400x/filters:format(jpg)/https://upload.wikimedia.org/'
+            + 'wikipedia/commons/8/87/All_that_jazz.tif',
+            'lossy-page1-400px-All_that_jazz.tif.jpg',
+            0.98,
+            1.0
+        )
+
+    def test_multipage_tiff_with_out_of_bounds_page(self):
+        self.run_and_check_ssim_and_size(
+            'unsafe/400x/filters:format(jpg):page(500)/https://'
+            + 'upload.wikimedia.org/wikipedia/commons/8/87/All_that_jazz.tif',
+            'lossy-page1-400px-All_that_jazz.tif.jpg',
+            0.98,
+            1.0
+        )
+
+    def test_png(self):
+        self.run_and_check_ssim_and_size(
+            'unsafe/400x/https://upload.wikimedia.org/wikipedia/commons/c/cf'
+            + '/WorldMap-A_non-Frame.png',
+            '400px-WorldMap-A_non-Frame.png',
+            0.94,
+            1.0
+        )
diff --git a/tests/integration/thumbnails/200px-Television.svg.png 
b/tests/integration/thumbnails/200px-Television.svg.png
new file mode 100644
index 0000000..1634680
--- /dev/null
+++ b/tests/integration/thumbnails/200px-Television.svg.png
Binary files differ
diff --git a/tests/integration/thumbnails/320px-seek=0-Borsch_01.webm.jpg 
b/tests/integration/thumbnails/320px-seek=0-Borsch_01.webm.jpg
new file mode 100644
index 0000000..3826c1e
--- /dev/null
+++ b/tests/integration/thumbnails/320px-seek=0-Borsch_01.webm.jpg
Binary files differ
diff --git a/tests/integration/thumbnails/page19-400px-Jeremy_Bentham.pdf.jpg 
b/tests/integration/thumbnails/page19-400px-Jeremy_Bentham.pdf.jpg
new file mode 100644
index 0000000..cd7a5ea
--- /dev/null
+++ b/tests/integration/thumbnails/page19-400px-Jeremy_Bentham.pdf.jpg
Binary files differ
diff --git 
a/tests/integration/thumbnails/page259-400px-Il_cavallarizzo.djvu.jpg 
b/tests/integration/thumbnails/page259-400px-Il_cavallarizzo.djvu.jpg
new file mode 100644
index 0000000..b054a4d
--- /dev/null
+++ b/tests/integration/thumbnails/page259-400px-Il_cavallarizzo.djvu.jpg
Binary files differ
diff --git 
a/tests/integration/thumbnails/page440-400px-Zibaldone_di_pensieri_V.djvu.jpg 
b/tests/integration/thumbnails/page440-400px-Zibaldone_di_pensieri_V.djvu.jpg
new file mode 100644
index 0000000..4a7fcda
--- /dev/null
+++ 
b/tests/integration/thumbnails/page440-400px-Zibaldone_di_pensieri_V.djvu.jpg
Binary files differ
diff --git a/wikimedia_thumbor/engine/__init__.py 
b/wikimedia_thumbor/engine/__init__.py
index 5c696ea..d96413d 100644
--- a/wikimedia_thumbor/engine/__init__.py
+++ b/wikimedia_thumbor/engine/__init__.py
@@ -73,6 +73,13 @@
         return super(BaseWikimediaEngine, self).read(extension, quality)
 
     def prepare_source(self, buffer):
+        if hasattr(self.context, 'wikimedia_original_file'):
+            logger.debug('[BWE] Found source file in context')
+            self.source = self.context.wikimedia_original_file.name
+            del self.context.wikimedia_original_file
+            return
+
+        logger.debug('[BWE] Create source file from buffer')
         self.source = os.path.join(self.temp_dir, 'source_file')
 
         with open(self.source, 'w') as source:
@@ -85,7 +92,7 @@
     def cleanup(self):  # pragma: no cover
         shutil.rmtree(self.temp_dir, True)
 
-    def command(self, command, env=None):
+    def command(self, command, env=None, clean_on_error=True):
         returncode, stderr, stdout = ShellRunner.command(
             command,
             self.context,
@@ -93,7 +100,8 @@
         )
 
         if returncode != 0:
-            self.cleanup_source()
+            if clean_on_error:
+                self.cleanup_source()
             raise CommandError(
                 command,
                 stdout,
diff --git a/wikimedia_thumbor/engine/djvu/djvu.py 
b/wikimedia_thumbor/engine/djvu/djvu.py
index a75d50e..634808b 100644
--- a/wikimedia_thumbor/engine/djvu/djvu.py
+++ b/wikimedia_thumbor/engine/djvu/djvu.py
@@ -11,11 +11,6 @@
 
 # DjVu engine
 
-# Otherwise Python thinks that djvu is the local module
-from __future__ import absolute_import
-
-import djvu.decode
-
 from wikimedia_thumbor.engine import BaseWikimediaEngine
 
 
@@ -33,48 +28,23 @@
 
 
 class Engine(BaseWikimediaEngine):
-    pixel_format = djvu.decode.PixelFormatRgb('RGB')
-    pixel_format.rows_top_to_bottom = 1
-    pixel_format.y_top_to_bottom = 1
-
     def create_image(self, buffer):
         self.original_buffer = buffer
-        # Unfortunately the djvu python bindings don't support reading
-        # file contents from memory, nor from a fifo. Which means we have to
-        # store the input in a temporary file
         self.prepare_source(buffer)
 
         try:
-            page = self.context.request.page - 1
+            page = self.context.request.page
         except AttributeError:
-            page = 0
+            page = 1
 
-        context = djvu.decode.Context()
-        file_uri = djvu.decode.FileURI(self.source)
-        document = context.new_document(file_uri, cache=False)
-        document.decoding_job.wait()
+        command = [
+            self.context.config.DDJVU_PATH,
+            '-format=ppm',
+            '-page=%d' % page,
+            self.source,
+            '-'
+        ]
 
-        # If we try to read a page out of bounds, use the first page
-        if page < 0 or page > len(document.pages) - 1:
-            page = 0
-
-        document_page = document.pages[page]
-        page_job = document_page.decode(wait=True)
-
-        width, height = page_job.size
-        rect = (0, 0, width, height)
-
-        data = page_job.render(
-            djvu.decode.RENDER_COLOR,
-            rect,
-            rect,
-            self.pixel_format
-        )
-
-        # PBM is a very simple file format, which ImageMagick can consume
-        ppm = 'P6 %d %d 255\n' % (width, height)
-        ppm += data
-
-        self.cleanup_source()
+        ppm = self.command(command)
 
         return super(Engine, self).create_image(ppm)
diff --git a/wikimedia_thumbor/engine/ghostscript/ghostscript.py 
b/wikimedia_thumbor/engine/ghostscript/ghostscript.py
index c7ebb68..39c9862 100644
--- a/wikimedia_thumbor/engine/ghostscript/ghostscript.py
+++ b/wikimedia_thumbor/engine/ghostscript/ghostscript.py
@@ -30,17 +30,19 @@
         except AttributeError:
             page = 1
 
-        png = self.get_png_for_page(buffer, page)
+        jpg = self.get_jpg_for_page(buffer, page)
 
         # GS is being unhelpful and outputting that error to stdout
         # with a 0 exit status
         error = 'No pages will be processed (FirstPage > LastPage)'
-        if len(png) < 200 and png.find(error) != -1:
-            png = self.get_png_for_page(buffer, 1)
+        if len(jpg) < 200 and jpg.find(error) != -1:
+            jpg = self.get_jpg_for_page(buffer, 1)
 
-        return super(Engine, self).create_image(png)
+        self.extension = '.jpg'
 
-    def get_png_for_page(self, buffer, page):
+        return super(Engine, self).create_image(jpg)
+
+    def get_jpg_for_page(self, buffer, page):
         self.prepare_source(buffer)
 
         # We use the command and not the python bindings because those can't
@@ -51,7 +53,8 @@
         # file for the destination.
         command = [
             self.context.config.GHOSTSCRIPT_PATH,
-            "-sDEVICE=png16m",
+            "-sDEVICE=jpeg",
+            "-dJPEG=90",
             "-sOutputFile=%stdout",
             "-dFirstPage=%d" % page,
             "-dLastPage=%d" % page,
diff --git a/wikimedia_thumbor/engine/imagemagick/imagemagick.py 
b/wikimedia_thumbor/engine/imagemagick/imagemagick.py
index 0c84c0d..60ad698 100644
--- a/wikimedia_thumbor/engine/imagemagick/imagemagick.py
+++ b/wikimedia_thumbor/engine/imagemagick/imagemagick.py
@@ -98,6 +98,14 @@
     exiftool = ExiftoolRunner()
 
     def create_image(self, buffer):
+        # This should be enough for now, if memory blows up on huge files we
+        # can could use an mmap here
+        if hasattr(self.context, 'wikimedia_original_file'):
+            fname = self.context.wikimedia_original_file.name
+            with open(fname, 'r') as content_file:
+                buffer = content_file.read()
+            ShellRunner.rm_f(fname)
+
         self.im_original_buffer = buffer
         self.exif = {}
 
@@ -118,7 +126,7 @@
 
     def jpeg_size(self, im, exif_image_size):
         buffer_size = exif_image_size.split('x')
-        buffer_size = [ float(x) for x in buffer_size ]
+        buffer_size = [float(x) for x in buffer_size]
         buffer_ratio = buffer_size[0] / buffer_size[1]
 
         width = float(self.context.request.width)
@@ -262,7 +270,8 @@
         return result
 
     def crop(self, crop_left, crop_top, crop_right, crop_bottom):
-        logger.debug('[IM] crop: %r %r %r %r' % (
+        logger.debug(
+            '[IM] crop: %r %r %r %r' % (
                 crop_left,
                 crop_top,
                 crop_right,
@@ -349,7 +358,3 @@
     @property
     def size(self):
         return self.image.size
-
-    def cleanup(self):  # pragma: no cover
-        logger.debug('[IM] cleanup')
-        Engine.exiftool.cleanup()
diff --git a/wikimedia_thumbor/engine/proxy/proxy.py 
b/wikimedia_thumbor/engine/proxy/proxy.py
index f18b334..4cebe7d 100644
--- a/wikimedia_thumbor/engine/proxy/proxy.py
+++ b/wikimedia_thumbor/engine/proxy/proxy.py
@@ -51,7 +51,10 @@
         if self.lcl['selected_engine'] is not None:
             return self.lcl['selected_engine']
 
-        ext = self.lcl['extension'].lstrip('.')
+        if self.lcl['extension'] is None:
+            ext = None
+        else:
+            ext = self.lcl['extension'].lstrip('.')
 
         logger.debug('[Proxy] Looking for a %s engine' % ext)
 
diff --git a/wikimedia_thumbor/engine/svg/svg.py 
b/wikimedia_thumbor/engine/svg/svg.py
index 572ece2..698d159 100644
--- a/wikimedia_thumbor/engine/svg/svg.py
+++ b/wikimedia_thumbor/engine/svg/svg.py
@@ -11,44 +11,26 @@
 
 # SVG engine
 
-import cairosvg
-import locale
-import StringIO
-
-from thumbor.utils import logger
-
 from wikimedia_thumbor.engine import BaseWikimediaEngine
+
+BaseWikimediaEngine.add_format(
+    'image/svg+xml',
+    '.svg',
+    lambda buffer: Engine.is_svg(buffer)
+)
 
 
 class Engine(BaseWikimediaEngine):
+    @classmethod
+    def is_svg(cls, buffer):
+        # Quite wide, but it's better to let rsvg give a file a shot
+        # rather than bail without trying
+        return (buffer.startswith('<?xml') and
+                'http://www.w3.org/2000/svg' in buffer[:1024])
+
     def create_image(self, buffer):
         self.original_buffer = buffer
 
-        if hasattr(self.context.config, 'RSVG_CONVERT_PATH'):
-            logger.debug('[SVG] Converting with rsvg')
-            png = self.create_image_with_rsvg(buffer)
-        else:
-            logger.debug('[SVG] Converting with cairosvg')
-            png = self.create_image_with_cairosvg(buffer)
-
-        return super(Engine, self).create_image(png)
-
-    def create_image_with_cairosvg(self, buffer):
-        cairosvg.features.LOCALE = 'en_US'
-
-        if hasattr(self.context.request, 'lang'):
-            request_locale = self.context.request.lang.upper()
-            normalized_locale = locale.normalize(request_locale)
-            cairosvg.features.LOCALE = normalized_locale
-
-        output = StringIO.StringIO()
-
-        return cairosvg.svg2png(
-            bytestring=buffer,
-            dpi=self.context.config.SVG_DPI
-        )
-
-    def create_image_with_rsvg(self, buffer):
         self.prepare_source(buffer)
 
         command = [
@@ -68,7 +50,9 @@
         if hasattr(self.context.request, 'lang'):
             env = {'LANG': self.context.request.lang.upper()}
 
-        return self.command(command, env)
+        png = self.command(command, env)
+
+        return super(Engine, self).create_image(png)
 
     # Disable this method in BaseEngine, do the conversion in create_image
     # instead
diff --git a/wikimedia_thumbor/engine/vips/vips.py 
b/wikimedia_thumbor/engine/vips/vips.py
index cc995b0..e3cb80a 100644
--- a/wikimedia_thumbor/engine/vips/vips.py
+++ b/wikimedia_thumbor/engine/vips/vips.py
@@ -11,7 +11,6 @@
 
 # VIPS engine
 
-import logging
 import math
 import os
 
@@ -21,33 +20,6 @@
 from wikimedia_thumbor.engine import CommandError
 from wikimedia_thumbor.shell_runner import ShellRunner  # noqa
 
-
-use_command_line = True
-
-try:  # pragma: no cover
-    import gi
-    if hasattr(gi, 'require_version'):
-        logger.debug('[VIPS] gi found')
-        try:
-            gi.require_version('Vips', '8.0')
-            logging.disable(logging.DEBUG)
-            from gi.repository import Vips
-            logging.disable(logging.NOTSET)
-            logger.debug('[VIPS] VIPS found in gi repository')
-            use_command_line = False
-        except ImportError:
-            logger.debug('[VIPS] VIPS not found in gi repository')
-        except ValueError:
-            logger.debug('[VIPS] VIPS 8.0+ not found in gi repository')
-    else:
-        logger.debug('[VIPS] Wrong gi found (not PyGObject)')
-except ImportError:  # pragma: no cover
-    logger.debug('[VIPS] gi not found')
-
-if use_command_line:
-    logger.debug('[VIPS] Will use command line')
-else:  # pragma: no cover
-    logger.debug('[VIPS] Will use bindings')
 
 BaseWikimediaEngine.add_format(
     'image/tiff',
@@ -69,11 +41,18 @@
             '-s'
         ]
 
-        stdout = Engine.exiftool.command(
-            pre=command,
-            context=self.context,
-            buffer=buffer
-        )
+        if hasattr(self.context, 'wikimedia_original_file'):
+            stdout = Engine.exiftool.command(
+                pre=command,
+                context=self.context,
+                input_temp_file=self.context.wikimedia_original_file
+            )
+        else:
+            stdout = Engine.exiftool.command(
+                pre=command,
+                context=self.context,
+                buffer=buffer
+            )
 
         size = stdout.strip().split('x')
 
@@ -90,13 +69,17 @@
 
         return False
 
+    def target_format(self):
+        if self.context.request.format:
+            return '.%s' % self.context.request.format
+
+        return self.context.request.extension
+
     def create_image(self, buffer):
-        try:
-            original_ext = self.context.request.extension
-        except AttributeError:  # pragma: no cover
-            # If there is no extension in the request, it means that we
-            # are serving a cached result. In which case no VIPS processing
-            # is required.
+        # If there is no extension in the request, it means that we
+        # are serving a cached result. In which case no VIPS processing
+        # is required.
+        if not hasattr(self.context.request, 'extension'):
             return super(Engine, self).create_image(buffer)
 
         self.original_buffer = buffer
@@ -107,37 +90,14 @@
             float(self.context.request.width)
         ))
 
-        if use_command_line:
-            result = self.shrink_with_command(buffer, shrink_factor)
-        else:  # pragma: no cover
-            result = self.shrink_with_bindings(buffer, shrink_factor)
+        result = self.shrink(buffer, shrink_factor)
 
-        self.extension = original_ext
+        self.extension = self.target_format()
+        logger.debug('[VIPS] Setting extension to: %s' % self.extension)
 
         return super(Engine, self).create_image(result)
 
-    def shrink_with_bindings(self, buffer, shrink_factor):  # pragma: no cover
-        logger.debug('[VIPS] Shrinking with bindings')
-        logging.disable(logging.DEBUG)
-
-        if gi and hasattr(gi, 'overrides'):
-            exceptions = (AttributeError, gi.overrides.Vips.Error)
-        else:
-            exceptions = AttributeError
-
-        try:
-            page = self.context.request.page - 1
-            source = Vips.Image.new_from_buffer(buffer, 'page=%d' % page)
-        except exceptions:
-            source = Vips.Image.new_from_buffer(buffer, '')
-
-        source = source.shrink(shrink_factor, shrink_factor)
-        result = source.write_to_buffer('.png')
-
-        logging.disable(logging.NOTSET)
-        return result
-
-    def shrink_with_command(self, buffer, shrink_factor):
+    def shrink(self, buffer, shrink_factor):
         logger.debug('[VIPS] Shrinking with command')
         self.prepare_source(buffer)
 
@@ -150,17 +110,24 @@
             source = self.source
 
         try:
-            return self.shrink_with_command_for_page(source, shrink_factor)
+            return self.shrink_for_page(source, shrink_factor)
         except CommandError:
             # The page is probably out of bounds, try again without
             # specifying a page
-            self.prepare_source(buffer)
             source = self.source
 
-        return self.shrink_with_command_for_page(source, shrink_factor)
+        try:
+            return self.shrink_for_page(source, shrink_factor)
+        except CommandError as e:
+            # Now that we've failed twice in a row, we cleanup the source
+            self.cleanup_source()
+            raise e
 
-    def shrink_with_command_for_page(self, source, shrink_factor):
-        destination = os.path.join(self.temp_dir, 'vips_result.png')
+    def shrink_for_page(self, source, shrink_factor):
+        destination = os.path.join(
+            self.temp_dir,
+            'vips_result%s' % self.target_format()
+        )
 
         command = [
             self.context.config.VIPS_PATH,
@@ -172,7 +139,7 @@
         ]
 
         try:
-            self.command(command)
+            self.command(command, clean_on_error=False)
         except CommandError as e:  # pragma: no cover
             ShellRunner.rm_f(destination)
             raise e
diff --git a/wikimedia_thumbor/engine/xcf/xcf.py 
b/wikimedia_thumbor/engine/xcf/xcf.py
index d918227..9686373 100644
--- a/wikimedia_thumbor/engine/xcf/xcf.py
+++ b/wikimedia_thumbor/engine/xcf/xcf.py
@@ -11,12 +11,7 @@
 
 # XCF engine
 
-import errno
-import os
-import time
-
 from wikimedia_thumbor.engine import BaseWikimediaEngine
-from wikimedia_thumbor.shell_runner import ShellRunner
 
 
 BaseWikimediaEngine.add_format(
@@ -29,98 +24,17 @@
 class Engine(BaseWikimediaEngine):
     def create_image(self, buffer):
         self.original_buffer = buffer
-        # xcf2png supports reading from a fifo instead of a file. It can't read
-        # from stdin, unfortunately, which is why we settled on the fifo
-        fifo = self.prepare_fifo()
+        self.prepare_source(buffer)
 
         # xcf2png doesn't exist in library form, only as an executable
         command = [
             self.context.config.XCF2PNG_PATH,
-            fifo,
+            self.source,
             '-o',
             '-'
         ]
 
-        png = self.command_fifo(command, buffer, fifo)
+        png = self.command(command)
+        self.extension = '.png '
 
         return super(Engine, self).create_image(png)
-
-    def popen_fifo(self, command, buffer, fifo):
-        """
-        Writes the buffer contents to the fifo and returns a subprocess for the
-        command.
-        """
-        proc = ShellRunner.popen(
-            command,
-            self.context
-        )
-
-        self.write_buffer_to_fifo(buffer, fifo)
-
-        return proc
-
-    def command_fifo(self, command, buffer, fifo):
-        stdout = ''
-
-        proc = self.popen_fifo(command, buffer, fifo)
-        stdout, stderr = proc.communicate()
-
-        ShellRunner.rm_f(fifo)
-
-        if proc.returncode != 0:  # pragma: no cover
-            raise Exception(
-                'CommandError',
-                command,
-                stdout,
-                stderr,
-                proc.returncode
-            )
-
-        return stdout
-
-    def prepare_fifo(self):
-        """
-        Creates a fifo and returns its path.
-        """
-        fifo = os.path.join(self.temp_dir, 'xcf_fifo')
-        os.mkfifo(fifo)
-
-        return fifo
-
-    def write_buffer_to_fifo(self, buffer, fifo):
-        """
-        Writes the contents of the buffer to the fifo.
-        """
-        while True:
-            try:
-                os_fifo = os.open(fifo, os.O_NONBLOCK | os.O_WRONLY)
-                break
-            except OSError as err:
-                # We wait until the consumer (xcf2png) starts reading the fifo
-                if err.errno == errno.ENXIO:
-                    time.sleep(.1)
-                    continue
-                else:  # pragma: no cover
-                    raise
-
-        # The consumer is reading, start writing the buffer contents to the
-        # fifo
-        seek = 0
-        length_written = 0
-
-        while seek == 0 or length_written:
-            seek += length_written
-
-            while True:
-                try:
-                    length_written = os.write(os_fifo, buffer[seek:])
-                    break
-                except OSError as err:
-                    if err.errno == errno.EAGAIN:
-                        time.sleep(.1)
-                        continue
-                    else:  # pragma: no cover
-                        os.close(os_fifo)
-                        raise
-
-        os.close(os_fifo)
diff --git a/wikimedia_thumbor/exiftool_runner/__init__.py 
b/wikimedia_thumbor/exiftool_runner/__init__.py
index a223f72..47f64fa 100644
--- a/wikimedia_thumbor/exiftool_runner/__init__.py
+++ b/wikimedia_thumbor/exiftool_runner/__init__.py
@@ -29,12 +29,20 @@
 
 class ExiftoolRunner:
     @classmethod
-    def command(cls, pre=[], post=[], context=None, buffer=''):
+    def command(
+        cls,
+        pre=[],
+        post=[],
+        context=None,
+        buffer='',
+        input_temp_file=None
+    ):
         start = datetime.datetime.now()
 
-        input_temp_file = NamedTemporaryFile()
-        input_temp_file.write(buffer)
-        input_temp_file.flush()
+        if not input_temp_file:
+            input_temp_file = NamedTemporaryFile()
+            input_temp_file.write(buffer)
+            input_temp_file.flush()
 
         command = [context.config.EXIFTOOL_PATH]
         command += pre
diff --git a/wikimedia_thumbor/filter/format/__init__.py 
b/wikimedia_thumbor/filter/format/__init__.py
new file mode 100644
index 0000000..fc97fa4
--- /dev/null
+++ b/wikimedia_thumbor/filter/format/__init__.py
@@ -0,0 +1,3 @@
+from .format import Filter
+
+__all__ = ['Filter']
diff --git a/wikimedia_thumbor/filter/format/format.py 
b/wikimedia_thumbor/filter/format/format.py
new file mode 100644
index 0000000..66d63f2
--- /dev/null
+++ b/wikimedia_thumbor/filter/format/format.py
@@ -0,0 +1,32 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+# thumbor imaging service
+# https://github.com/thumbor/thumbor/wiki
+
+# Licensed under the MIT license:
+# http://www.opensource.org/licenses/mit-license
+# Copyright (c) 2011 globo.com timeh...@corp.globo.com
+# Copyright (c) 2015 Wikimedia Foundation
+
+# Simply passes the format parameter
+# This is a fork of thumbor's format parameter, running in the PRE_LOAD phase
+
+from thumbor.filters import BaseFilter, filter_method, PHASE_PRE_LOAD
+from thumbor.utils import logger
+
+
+ALLOWED_FORMATS = ['png', 'jpeg', 'jpg', 'gif', 'webp']
+
+
+class Filter(BaseFilter):
+    phase = PHASE_PRE_LOAD
+
+    @filter_method(BaseFilter.String)
+    def format(self, format):
+        if format.lower() not in ALLOWED_FORMATS:
+            logger.debug('Format not allowed: %s' % format.lower())
+            self.context.request.format = None
+        else:
+            logger.debug('Format specified: %s' % format.lower())
+            self.context.request.format = format.lower()
\ No newline at end of file
diff --git a/wikimedia_thumbor/handler/images/images.py 
b/wikimedia_thumbor/handler/images/images.py
index b5cdb9d..f3b1185 100644
--- a/wikimedia_thumbor/handler/images/images.py
+++ b/wikimedia_thumbor/handler/images/images.py
@@ -16,6 +16,8 @@
 from thumbor.handlers.imaging import ImagingHandler
 from thumbor.utils import logger
 
+class TranslateError(Exception):
+    pass
 
 class ImagesHandler(ImagingHandler):
     @classmethod
@@ -84,6 +86,9 @@
         )
 
         translated = {'width': kw['width']}
+
+        if int(kw['width']) < 1:
+            raise TranslateError('Width requested must be at least 1')
 
         sharded_containers = []
 
@@ -193,7 +198,14 @@
 
     @gen.coroutine
     def check_image(self, kw):
-        translated_kw = self.translate(kw)
+        try:
+            translated_kw = self.translate(kw)
+        except TranslateError as e:
+            self._error(
+                400,
+                str(e)
+            )
+            return
 
         if self.context.config.MAX_ID_LENGTH > 0:
             # Check if an image with an uuid exists in storage
diff --git a/wikimedia_thumbor/loader/https/__init__.py 
b/wikimedia_thumbor/loader/https/__init__.py
new file mode 100644
index 0000000..3e88949
--- /dev/null
+++ b/wikimedia_thumbor/loader/https/__init__.py
@@ -0,0 +1,88 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+# thumbor imaging service
+# https://github.com/thumbor/thumbor/wiki
+
+# Licensed under the MIT license:
+# http://www.opensource.org/licenses/mit-license
+# Copyright (c) 2011 globo.com timeh...@corp.globo.com
+# Copyright (c) 2016 Wikimedia Foundation
+
+# Https loader. Unlike the stock Thumbor one, uses a streaming callback
+# and can define a higher body size limit than 100MB
+
+from functools import partial
+from tempfile import NamedTemporaryFile
+import tornado.simple_httpclient
+
+
+from thumbor.loaders import http_loader, https_loader
+from thumbor.utils import logger
+
+
+def should_run(url):  # pragma: no cover
+    return True
+
+
+def return_contents(response, url, callback, context, f):  # pragma: no cover
+    # We put the first kb of content into the response body, to let Thumbor's
+    # mime detection work
+    f.seek(0)
+    response._body = f.read(1024)
+    context.wikimedia_original_file = f
+    f.close()
+    return http_loader.return_contents(response, url, callback, context)
+
+
+def stream_contents(response, f):
+    f.write(response)
+
+
+def load_sync(context, url, callback):
+    logger.debug('[HTTPS] load_sync: %s' % url)
+    client = tornado.simple_httpclient.SimpleAsyncHTTPClient(
+        max_clients=context.config.HTTP_LOADER_MAX_CLIENTS,
+        max_body_size=context.config.HTTP_LOADER_MAX_BODY_SIZE
+    )
+
+    user_agent = None
+    if context.config.HTTP_LOADER_FORWARD_USER_AGENT:
+        if 'User-Agent' in context.request_handler.request.headers:
+            user_agent = context.request_handler.request.headers['User-Agent']
+    if user_agent is None:
+        user_agent = context.config.HTTP_LOADER_DEFAULT_USER_AGENT
+
+    f = NamedTemporaryFile(delete=False)
+
+    url = https_loader._normalize_url(url)
+    req = tornado.httpclient.HTTPRequest(
+        url=url,
+        connect_timeout=context.config.HTTP_LOADER_CONNECT_TIMEOUT,
+        request_timeout=context.config.HTTP_LOADER_REQUEST_TIMEOUT,
+        follow_redirects=context.config.HTTP_LOADER_FOLLOW_REDIRECTS,
+        max_redirects=context.config.HTTP_LOADER_MAX_REDIRECTS,
+        user_agent=user_agent,
+        proxy_host=encode(context.config.HTTP_LOADER_PROXY_HOST),
+        proxy_port=context.config.HTTP_LOADER_PROXY_PORT,
+        proxy_username=encode(context.config.HTTP_LOADER_PROXY_USERNAME),
+        proxy_password=encode(context.config.HTTP_LOADER_PROXY_PASSWORD),
+        ca_certs=encode(context.config.HTTP_LOADER_CA_CERTS),
+        client_key=encode(context.config.HTTP_LOADER_CLIENT_KEY),
+        client_cert=encode(context.config.HTTP_LOADER_CLIENT_CERT),
+        streaming_callback=partial(stream_contents, f=f)
+    )
+
+    client.fetch(
+        req, callback=partial(
+            return_contents,
+            url=url,
+            callback=callback,
+            context=context,
+            f=f
+        )
+    )
+
+
+def encode(string):
+    return None if string is None else string.encode('ascii')
diff --git a/wikimedia_thumbor/loader/video/__init__.py 
b/wikimedia_thumbor/loader/video/__init__.py
index e0ff639..66c2bb0 100644
--- a/wikimedia_thumbor/loader/video/__init__.py
+++ b/wikimedia_thumbor/loader/video/__init__.py
@@ -14,8 +14,10 @@
 # otherwise the first frame requested will be stored as the
 # original for subsequent requests
 
-from urllib import unquote
+import os
 from functools import partial
+from tempfile import NamedTemporaryFile
+from urllib import unquote
 
 from thumbor.loaders import LoaderResult
 from thumbor.utils import logger
@@ -103,6 +105,12 @@
     except AttributeError:
         seek = duration / 2
 
+    seek_and_screenshot(callback, context, unquoted_url, seek)
+
+
+def seek_and_screenshot(callback, context, unquoted_url, seek):
+    output_file = NamedTemporaryFile(delete=False)
+
     command = ShellRunner.wrap_command([
         context.config.FFMPEG_PATH,
         # Order is important, for fast seeking -ss has to come before -i
@@ -119,8 +127,8 @@
         'image2',
         '-nostats',
         '-loglevel',
-        'error',
-        '-'
+        'fatal',
+        output_file.name
     ], context)
 
     logger.debug('[Video] _parse_time: %r' % command)
@@ -130,28 +138,43 @@
         partial(
             _process_done,
             callback,
-            process
+            process,
+            context,
+            unquoted_url,
+            seek,
+            output_file
         )
     )
 
+def _process_done(
+        callback,
+        process,
+        context,
+        unquoted_url,
+        seek,
+        output_file,
+        status
+    ):
+    # If rendering the desired frame fails, attempt to render the
+    # first frame instead
+    if status != 0 and seek > 0:
+        seek_and_screenshot(callback, context, unquoted_url, 0)
+        return
 
-def _process_done(callback, process, status):
     result = LoaderResult()
 
     if status != 0:  # pragma: no cover
         result.successful = False
-        callback(result)
     else:
-        process.stdout.read_until_close(
-            partial(
-                _process_stdout,
-                callback
-            )
-        )
+        result.successful = True
+        result.buffer = output_file.read()
 
+    output_file.close()
 
-def _process_stdout(callback, stdout):
-    result = LoaderResult()
-    result.successful = True
-    result.buffer = stdout
+    try:
+        os.unlink(output_file.name)
+    except OSError as e:  # pragma: no cover
+        if e.errno != errno.ENOENT:
+            raise
+
     callback(result)
diff --git a/wikimedia_thumbor/result_storage/swift/swift.py 
b/wikimedia_thumbor/result_storage/swift/swift.py
index 8949cc1..c4487fd 100644
--- a/wikimedia_thumbor/result_storage/swift/swift.py
+++ b/wikimedia_thumbor/result_storage/swift/swift.py
@@ -81,7 +81,11 @@
 
     @return_future
     def get(self, callback):
-        logger.debug('[Swift] get')
+        logger.debug('[Swift] get: %r %r' % (
+                self.context.wikimedia_thumbnail_container,
+                self.context.wikimedia_path
+            )
+        )
 
         try:
             start = datetime.datetime.now()
@@ -102,6 +106,7 @@
                 'Swift-Time', duration
             )
 
+            logger.debug('[Swift] found')
             callback(data)
         # We want this to be exhaustive because not catching an exception here
         # would result in the request hanging indefinitely
@@ -109,6 +114,7 @@
             logging.disable(logging.NOTSET)
             # No need to log this one, it's expected behavior when the
             # requested object isn't there
+            logger.debug('[Swift] missing')
             callback(None)
         except Exception as e:
             logging.disable(logging.NOTSET)
diff --git a/wikimedia_thumbor/storage/request/request.py 
b/wikimedia_thumbor/storage/request/request.py
index 169c509..07f979a 100644
--- a/wikimedia_thumbor/storage/request/request.py
+++ b/wikimedia_thumbor/storage/request/request.py
@@ -37,8 +37,9 @@
     def get(self, path, callback):
         logger.debug("[REQUEST_STORAGE] get: %s" % path)
         try:
+            value = self.dict[path]
             logger.debug("[REQUEST_STORAGE] found")
-            callback(self.dict[path])
+            callback(value)
         except KeyError:
             logger.debug("[REQUEST_STORAGE] missing")
             callback(None)

-- 
To view, visit https://gerrit.wikimedia.org/r/311668
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: merged
Gerrit-Change-Id: I25b6f9ce199159196e9c0cc92804c4e6926828a5
Gerrit-PatchSet: 3
Gerrit-Project: operations/debs/python-thumbor-wikimedia
Gerrit-Branch: master
Gerrit-Owner: Gilles <gdu...@wikimedia.org>
Gerrit-Reviewer: Filippo Giunchedi <fgiunch...@wikimedia.org>
Gerrit-Reviewer: Gilles <gdu...@wikimedia.org>
Gerrit-Reviewer: jenkins-bot <>

_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

Reply via email to