BBlack has uploaded a new change for review.

  https://gerrit.wikimedia.org/r/311600

Change subject: upload storage size class experiment
......................................................................

upload storage size class experiment

This will probably need tweaking.  Uses cp1099 as first test host.

Bug: T145661
Change-Id: I024b54ed3ae1446743f1a3b15dd3842987a0fda9
---
A hieradata/hosts/cp1099.yaml
M modules/role/manifests/cache/upload.pp
M modules/varnish/templates/upload-backend.inc.vcl.erb
3 files changed, 72 insertions(+), 19 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/00/311600/1

diff --git a/hieradata/hosts/cp1099.yaml b/hieradata/hosts/cp1099.yaml
new file mode 100644
index 0000000..c02abf9
--- /dev/null
+++ b/hieradata/hosts/cp1099.yaml
@@ -0,0 +1 @@
+upload_storage_experiment: true
diff --git a/modules/role/manifests/cache/upload.pp 
b/modules/role/manifests/cache/upload.pp
index 826a269..77c6eba 100644
--- a/modules/role/manifests/cache/upload.pp
+++ b/modules/role/manifests/cache/upload.pp
@@ -81,28 +81,48 @@
         'pass_random'      => false,
     })
 
-    if ($::role::cache::2layer::varnish_version4) {
-        # The persistent storage backend is deprecated and buggy in Varnish 4.
-        # Use "file" instead. See T142810, T142848 and
-        # https://www.varnish-cache.org/docs/trunk/phk/persistent.html
-        $storage_backend = 'file'
-        $storage_mma_1 = ''
-        $storage_mma_2 = ''
+    $upload_storage_experiment = hiera('upload_storage_experiment', false);
+    if $upload_storage_experiment {
+        $sda = $::role::cache::2layer::storage_parts[0]
+        $sdb = $::role::cache::2layer::storage_parts[1]
+        $ssm = $::role::cache::2layer::storage_size * 2 * 1024
+        $bin0_size = floor($ssm * 0.03)
+        $bin1_size = floor($ssm * 0.20)
+        $bin2_size = floor($ssm * 0.43)
+        $bin3_size = floor($ssm * 0.30)
+        $bin4_size = floor($ssm * 0.04)
+        $upload_storage_args = join([
+            "-s bin0=file,/srv/${sda}/varnish.bin0,${bin0_size}M"
+            "-s bin1=file,/srv/${sdb}/varnish.bin1,${bin1_size}M"
+            "-s bin2=file,/srv/${sda}/varnish.bin2,${bin2_size}M"
+            "-s bin3=file,/srv/${sdb}/varnish.bin3,${bin3_size}M"
+            "-s bin4=file,/srv/${sda}/varnish.bin4,${bin4_size}M"
+        ], ' ')
     }
     else {
-        $storage_backend = 'persistent'
-        $storage_mma_1 = ",${::role::cache::2layer::mma[0]}"
-        $storage_mma_2 = ",${::role::cache::2layer::mma[1]}"
-    }
+        if ($::role::cache::2layer::varnish_version4) {
+            # The persistent storage backend is deprecated and buggy in 
Varnish 4.
+            # Use "file" instead. See T142810, T142848 and
+            # https://www.varnish-cache.org/docs/trunk/phk/persistent.html
+            $storage_backend = 'file'
+            $storage_mma_1 = ''
+            $storage_mma_2 = ''
+        }
+        else {
+            $storage_backend = 'persistent'
+            $storage_mma_1 = ",${::role::cache::2layer::mma[0]}"
+            $storage_mma_2 = ",${::role::cache::2layer::mma[1]}"
+        }
 
-    $storage_size_bigobj = floor($::role::cache::2layer::storage_size / 6)
-    $storage_size_up = $::role::cache::2layer::storage_size - 
$storage_size_bigobj
-    $upload_storage_args = join([
-        "-s 
main1=${storage_backend},/srv/${::role::cache::2layer::storage_parts[0]}/varnish.main1,${storage_size_up}G${storage_mma_1}",
-        "-s 
main2=${storage_backend},/srv/${::role::cache::2layer::storage_parts[1]}/varnish.main2,${storage_size_up}G${storage_mma_2}",
-        "-s 
bigobj1=file,/srv/${::role::cache::2layer::storage_parts[0]}/varnish.bigobj1,${storage_size_bigobj}G",
-        "-s 
bigobj2=file,/srv/${::role::cache::2layer::storage_parts[1]}/varnish.bigobj2,${storage_size_bigobj}G",
-    ], ' ')
+        $storage_size_bigobj = floor($::role::cache::2layer::storage_size / 6)
+        $storage_size_up = $::role::cache::2layer::storage_size - 
$storage_size_bigobj
+        $upload_storage_args = join([
+            "-s 
main1=${storage_backend},/srv/${::role::cache::2layer::storage_parts[0]}/varnish.main1,${storage_size_up}G${storage_mma_1}",
+            "-s 
main2=${storage_backend},/srv/${::role::cache::2layer::storage_parts[1]}/varnish.main2,${storage_size_up}G${storage_mma_2}",
+            "-s 
bigobj1=file,/srv/${::role::cache::2layer::storage_parts[0]}/varnish.bigobj1,${storage_size_bigobj}G",
+            "-s 
bigobj2=file,/srv/${::role::cache::2layer::storage_parts[1]}/varnish.bigobj2,${storage_size_bigobj}G",
+        ], ' ')
+    }
 
     # default_ttl=7d
     $common_runtime_params = ['default_ttl=604800']
diff --git a/modules/varnish/templates/upload-backend.inc.vcl.erb 
b/modules/varnish/templates/upload-backend.inc.vcl.erb
index 312eec5..3fc2468 100644
--- a/modules/varnish/templates/upload-backend.inc.vcl.erb
+++ b/modules/varnish/templates/upload-backend.inc.vcl.erb
@@ -1,6 +1,33 @@
 // Varnish VCL include file for upload backends
 include "upload-common.inc.vcl";
 
+<%- if upload_storage_experiment -%>
+sub pick_stevedore {
+       // Select a storage size class/bin
+       if (beresp.http.Content-Length !~ "^[0-9]+$") { // XXX even possible on 
upload?
+               set beresp.storage_hint = "bin1";
+       }
+       elsif (beresp.http.Content-Length ~ "^[0-9]{10}") { // 1GB+
+               set beresp.uncacheable = true;
+               set beresp.storage_hint = "bin4";
+       }
+       elsif (beresp.http.Content-Length < 1024 * 16) { // 16KB
+               set beresp.storage_hint = "bin0";
+       }
+       elsif (beresp.http.Content-Length < 1024 * 16 * 16) { // 256KB
+               set beresp.storage_hint = "bin1";
+       }
+       elsif (beresp.http.Content-Length < 1024 * 16 * 16 * 16) { // 4MB
+               set beresp.storage_hint = "bin2";
+       }
+       elsif (beresp.http.Content-Length < 1024 * 16 * 16 * 16 * 16) { // 64MB
+               set beresp.storage_hint = "bin3";
+       }
+       else { // 64MB-1GB
+               set beresp.storage_hint = "bin4";
+       }
+}
+<%- else -%>
 sub pick_main_stevedore {
        if (std.random(0, 2) < 1) {
                set beresp.<%= @storage_hint_storage %> = "main1";
@@ -16,6 +43,7 @@
                set beresp.<%= @storage_hint_storage %> = "bigobj2";
        }
 }
+<%- end -%>
 
 sub cluster_be_recv_pre_purge { }
 
@@ -69,12 +97,16 @@
        }
        <%- end -%>
 
+       <%- if upload_storage_experiment -%>
+       call pick_stevedore;
+       <%- else -%>
        // Select a random big object storage backend for objects >= 100 MB
        if (beresp.http.Content-Length ~ "^[0-9]{9}") {
                call pick_large_object_stevedore;
        } else {
                call pick_main_stevedore;
        }
+       <%- end -%>
        
        if (beresp.http.Content-Range) {
                // Varnish itself doesn't ask for ranges, so this must have been

-- 
To view, visit https://gerrit.wikimedia.org/r/311600
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I024b54ed3ae1446743f1a3b15dd3842987a0fda9
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: BBlack <bbl...@wikimedia.org>

_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

Reply via email to