[
https://issues.apache.org/jira/browse/TC-32?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15771183#comment-15771183
]
ASF GitHub Bot commented on TC-32:
----------------------------------
Github user mitchell852 commented on a diff in the pull request:
https://github.com/apache/incubator-trafficcontrol/pull/141#discussion_r93701822
--- Diff: traffic_ops/app/lib/API/Configs/ApacheTrafficServer.pm ---
@@ -0,0 +1,1959 @@
+package API::Configs::ApacheTrafficServer;
+
+#
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+#
+use UI::Utils;
+use Mojo::Base 'Mojolicious::Controller';
+use Date::Manip;
+use NetAddr::IP;
+use Data::Dumper;
+use UI::DeliveryService;
+use JSON;
+use API::DeliveryService::KeysUrlSig qw(URL_SIG_KEYS_BUCKET);
+use URI;
+use File::Basename;
+use File::Path;
+
+#Sub to generate ORT json
+sub ort {
+ my $self = shift;
+ my $id = $self->param('id');
+ my $filename = 'ort';
+ my $scope = 'server';
+
+ ##check user access
+ if ( !&is_oper($self) ) {
+ return $self->forbidden();
+ }
+
+ ##verify that a valid server ID has been used
+ my $server_obj = $self->server_data($id);
+ if ( !defined($server_obj) ) {
+ return $self->not_found();
+ }
+
+ my $data_obj;
+ my $host_name = $server_obj->host_name;
+
+ my %condition = ( 'me.host_name' => $host_name );
+ my $rs_profile = $self->db->resultset('Server')->search( \%condition, {
prefetch => [ 'cdn', 'profile' ] } );
+
+ my $row = $rs_profile->next;
+ if ($row) {
+ my $cdn_name = defined( $row->cdn_id ) ? $row->cdn->name : "";
+
+ $data_obj->{'profile'}->{'name'} = $row->profile->name;
+ $data_obj->{'profile'}->{'id'} = $row->profile->id;
+ $data_obj->{'other'}->{'CDN_name'} = $cdn_name;
+
+ %condition = (
+ 'profile_parameters.profile' =>
$data_obj->{'profile'}->{'id'},
+ -or => [ 'name' => 'location',
'name' => 'scope' ]
+ );
+ my $rs_config = $self->db->resultset('Parameter')->search(
\%condition, { join => 'profile_parameters' } );
+ while ( my $row = $rs_config->next ) {
+ if ( $row->name eq 'location' ) {
+ $data_obj->{'config_files'}->{
$row->config_file }->{'location'} = $row->value;
+ }
+ elsif ( $row->name eq 'scope' ) {
+ $data_obj->{'config_files'}->{
$row->config_file }->{'scope'} = $row->value;
+ }
+ }
+ }
+
+ foreach my $file ( keys %$data_obj->{'config_files'} ) {
+ if ( !defined( $data_obj->{'config_files'}->{$file}->{'scope'}
) ) {
+ $data_obj->{'config_files'}->{$file}->{'scope'} =
$self->get_scope($file);
+ }
+ }
+
+ my $file_contents = encode_json($data_obj);
+
+ return $self->render( text => $file_contents, format => 'txt' );
+}
+
+#entry point for server scope api route.
+sub get_server_config {
+ my $self = shift;
+ my $filename = $self->param("filename");
+ my $id = $self->param('id');
+ my $scope = $self->get_scope($filename);
+
+ ##check user access
+ if ( !&is_oper($self) ) {
+ return $self->forbidden();
+ }
+
+ ##check the scope - is this the correct route?
+ if ( $scope ne 'server' ) {
+ return $self->alert( "Error - incorrect file scope for route
used. Please use the " . $scope . " route." );
+ }
+
+ ##verify that a valid server ID has been used
+ my $server_obj = $self->server_data($id);
+ if ( !defined($server_obj) ) {
+ return $self->not_found();
+ }
+
+ #generate the config file using the appropriate function
+ my $file_contents;
+ if ( $filename eq "12M_facts" ) { $file_contents = $self->facts(
$server_obj, $filename, $scope ); }
+ elsif ( $filename =~ /to_ext_.*\.config/ ) { $file_contents =
$self->to_ext_dot_config( $server_obj, $filename, $scope ); }
+ elsif ( $filename eq "ip_allow.config" ) { $file_contents =
$self->ip_allow_dot_config( $server_obj, $filename, $scope ); }
+ elsif ( $filename eq "parent.config" ) { $file_contents =
$self->parent_dot_config( $server_obj, $filename, $scope ); }
+ elsif ( $filename eq "records.config" ) { $file_contents =
$self->generic_server_config( $server_obj, $filename, $scope ); }
+ elsif ( $filename eq "remap.config" ) { $file_contents =
$self->remap_dot_config( $server_obj, $filename, $scope ); }
+ elsif ( $filename eq "hosting.config" ) { $file_contents =
$self->hosting_dot_config( $server_obj, $filename, $scope ); }
+ elsif ( $filename eq "cache.config" ) { $file_contents =
$self->cache_dot_config( $server_obj, $filename, $scope ); }
+ elsif ( $filename eq "packages" ) {
+ $file_contents = $self->get_package_versions( $server_obj,
$filename, $scope );
+ $file_contents = encode_json($file_contents);
+ }
+ elsif ( $filename eq "chkconfig" ) {
+ $file_contents = $self->get_chkconfig( $server_obj, $filename,
$scope );
+ $file_contents = encode_json($file_contents);
+ }
+ else {
+ my $file_param = $self->db->resultset('Parameter')->search( [
config_file => $filename ] )->single;
+ if ( !defined($file_param) ) {
+ return $self->not_found();
+ }
+ $file_contents = $self->take_and_bake_server( $server_obj,
$filename, $scope );
+ }
+
+ #if we get an empty file, just send back an error.
+ if ( !defined($file_contents) ) {
+ return $self->not_found();
+ }
+
+ #return the file contents for fetch and db actions.
+ return $self->render( text => $file_contents, format => 'txt' );
+}
+
+#entry point for cdn scope api route.
+sub get_cdn_config {
+ my $self = shift;
+ my $filename = $self->param("filename");
+ my $id = $self->param('id');
+ my $scope = $self->get_scope($filename);
+
+ ##check user access
+ if ( !&is_oper($self) ) {
+ return $self->forbidden();
+ }
+ print STDERR "well i'm here anyway.";
+ ##check the scope - is this the correct route?
+ if ( $scope ne 'cdn' ) {
+ return $self->alert( "Error - incorrect file scope for route
used. Please use the " . $scope . " route." );
+ }
+
+ ##verify that a valid cdn ID has been used
+ my $cdn_obj = $self->cdn_data($id);
+ if ( !defined($cdn_obj) ) {
+ return $self->not_found();
+ }
+
+ #generate the config file using the appropriate function
+ my $file_contents;
+ if ( $filename eq "bg_fetch.config" ) { $file_contents =
$self->bg_fetch_dot_config( $cdn_obj, $filename, $scope ); }
+ elsif ( $filename =~ /cacheurl.*\.config/ ) { $file_contents =
$self->cacheurl_dot_config( $cdn_obj, $filename, $scope ); }
+ elsif ( $filename =~ /hdr_rw_.*\.config/ ) { $file_contents =
$self->header_rewrite_dot_config( $cdn_obj, $filename, $scope ); }
+ elsif ( $filename =~ /regex_remap_.*\.config/ ) { $file_contents =
$self->regex_remap_dot_config( $cdn_obj, $filename, $scope ); }
+ elsif ( $filename eq "regex_revalidate.config" ) { $file_contents =
$self->regex_revalidate_dot_config( $cdn_obj, $filename, $scope ); }
+ elsif ( $filename =~ /set_dscp_.*\.config/ ) { $file_contents =
$self->set_dscp_dot_config( $cdn_obj, $filename, $scope ); }
+ elsif ( $filename eq "ssl_multicert.config" ) { $file_contents =
$self->ssl_multicert_dot_config( $cdn_obj, $filename, $scope ); }
+ else { return
$self->not_found(); }
+
+ if ( !defined($file_contents) ) {
+ return $self->not_found();
+ }
+
+ return $self->render( text => $file_contents, format => 'txt' );
+}
+
+#entry point for profile scope api route.
+sub get_profile_config {
+ my $self = shift;
+ my $filename = $self->param("filename");
+ my $id = $self->param('id');
+ my $scope = $self->get_scope($filename);
+
+ ##check user access
+ if ( !&is_oper($self) ) {
+ return $self->forbidden();
+ }
+
+ ##check the scope - is this the correct route?
+ if ( $scope ne 'profile' ) {
+ return $self->alert( "Error - incorrect file scope for route
used. Please use the " . $scope . " route." );
+ }
+
+ ##verify that a valid profile ID has been used
+ my $profile_obj = $self->profile_data($id);
+ if ( !defined($profile_obj) ) {
+ return $self->not_found();
+ }
+
+ #generate the config file using the appropriate function
+ my $file_contents;
+ if ( $filename eq "50-ats.rules" ) { $file_contents =
$self->ats_dot_rules( $profile_obj, $filename, $scope ); }
+ elsif ( $filename eq "astats.config" ) { $file_contents =
$self->generic_profile_config( $profile_obj, $filename, $scope ); }
+ elsif ( $filename eq "drop_qstring.config" ) { $file_contents =
$self->drop_qstring_dot_config( $profile_obj, $filename, $scope ); }
+ elsif ( $filename eq "logs_xml.config" ) { $file_contents =
$self->logs_xml_dot_config( $profile_obj, $filename, $scope ); }
+ elsif ( $filename eq "plugin.config" ) { $file_contents =
$self->generic_profile_config( $profile_obj, $filename, $scope ); }
+ elsif ( $filename eq "storage.config" ) { $file_contents =
$self->storage_dot_config( $profile_obj, $filename, $scope ); }
+ elsif ( $filename eq "sysctl.conf" ) { $file_contents =
$self->generic_profile_config( $profile_obj, $filename, $scope ); }
+ elsif ( $filename =~ /url_sig_.*\.config/ ) { $file_contents =
$self->url_sig_dot_config( $profile_obj, $filename, $scope ); }
+ elsif ( $filename eq "volume.config" ) { $file_contents =
$self->volume_dot_config( $profile_obj, $filename, $scope ); }
+ else {
+ my $file_param = $self->db->resultset('Parameter')->search( [
config_file => $filename ] )->single;
+ if ( !defined($file_param) ) {
+ return $self->not_found();
+ }
+ $file_contents = $self->take_and_bake_profile( $profile_obj,
$filename, $scope );
+ }
+
+ if ( !defined($file_contents) ) {
+ return $self->not_found();
+ }
+
+ return $self->render( text => $file_contents, format => 'txt' );
+}
+
+my $separator ||= {
+ "records.config" => " ",
+ "plugin.config" => " ",
+ "sysctl.conf" => " = ",
+ "url_sig_.config" => " = ",
+ "astats.config" => "=",
+};
+
+#identify the correct scope for each filename. if not found, returns
server scope as any
+#undefined parameter based configs are designed with server scope using
the take-and-bake sub.
+sub get_scope {
+ my $self = shift;
+ my $fname = shift;
+ my $scope;
+
+ if ( $fname eq "12M_facts" ) { $scope = 'server' }
+ elsif ( $fname eq "ip_allow.config" ) { $scope = 'server' }
+ elsif ( $fname eq "parent.config" ) { $scope = 'server' }
+ elsif ( $fname eq "records.config" ) { $scope = 'server' }
+ elsif ( $fname eq "remap.config" ) { $scope = 'server' }
+ elsif ( $fname =~ /to_ext_.*\.config/ ) { $scope = 'server' }
+ elsif ( $fname eq "hosting.config" ) { $scope = 'server' }
+ elsif ( $fname eq "cache.config" ) { $scope = 'server' }
+ elsif ( $fname eq "packages" ) { $scope = 'server' }
+ elsif ( $fname eq "chkconfig" ) { $scope = 'server' }
+ elsif ( $fname eq "50-ats.rules" ) { $scope = 'profile' }
+ elsif ( $fname eq "astats.config" ) { $scope = 'profile' }
+ elsif ( $fname eq "drop_qstring.config" ) { $scope = 'profile' }
+ elsif ( $fname eq "logs_xml.config" ) { $scope = 'profile' }
+ elsif ( $fname eq "plugin.config" ) { $scope = 'profile' }
+ elsif ( $fname eq "storage.config" ) { $scope = 'profile' }
+ elsif ( $fname eq "sysctl.conf" ) { $scope = 'profile' }
+ elsif ( $fname =~ /url_sig_.*\.config/ ) { $scope = 'profile' }
+ elsif ( $fname eq "volume.config" ) { $scope = 'profile' }
+ elsif ( $fname eq "bg_fetch.config" ) { $scope = 'cdn' }
+ elsif ( $fname =~ /cacheurl.*\.config/ ) { $scope = 'cdn' }
+ elsif ( $fname =~ /hdr_rw_.*\.config/ ) { $scope = 'cdn' }
+ elsif ( $fname =~ /regex_remap_.*\.config/ ) { $scope = 'cdn' }
+ elsif ( $fname eq "regex_revalidate.config" ) { $scope = 'cdn' }
+ elsif ( $fname =~ /set_dscp_.*\.config/ ) { $scope = 'cdn' }
+ elsif ( $fname eq "ssl_multicert.config" ) { $scope = 'cdn' }
+ else {
+ $scope = $self->db->resultset('Parameter')->search( { -and => [
name => 'scope', config_file => $fname ] } )->get_column('value')->first();
+ if ( !defined($scope) ) {
+ $self->app->log->error("Filename not found. Setting
Server scope.");
+ $scope = 'server';
+ }
+ }
+
+ return $scope;
+}
+
+#takes the server name or ID and turns it into a server object that can
reference either, making either work for the request.
+sub server_data {
+ my $self = shift;
+ my $id = shift;
+
+ my $server_obj;
+
+ #if an ID is passed, look up by ID. Otherwise, look up by hostname.
+ if ( $id =~ /^\d+$/ ) {
+ $server_obj = $self->db->resultset('Server')->search( { id =>
$id } )->single;
+ }
+ else {
+ $server_obj = $self->db->resultset('Server')->search( {
host_name => $id } )->single;
+ }
+
+ return $server_obj;
+}
+
+#takes the profile name or ID and turns it into a server object that can
reference either, making either work for the request.
+sub profile_data {
+ my $self = shift;
+ my $id = shift;
+
+ #if an ID is passed, look up by ID. Otherwise, look up by profile name.
+ my $profile_obj;
+ if ( $id =~ /^\d+$/ ) {
+ $profile_obj = $self->db->resultset('Profile')->search( { id =>
$id } )->single;
+ }
+ else {
+ $profile_obj = $self->db->resultset('Profile')->search( { name
=> $id } )->single;
+ }
+
+ return $profile_obj;
+}
+
+#takes the server name or ID and turns it into a server object that can
reference either, making either work for the request.
+sub cdn_data {
+ my $self = shift;
+ my $id = shift;
+ my $cdn_obj;
+
+ if ( $id =~ /^\d+$/ ) {
+ $cdn_obj = $self->db->resultset('Cdn')->search( { id => $id }
)->single;
+ }
+ else {
+ $cdn_obj = $self->db->resultset('Cdn')->search( { name => $id }
)->single;
+ }
+
+ return $cdn_obj;
+}
+
+#generates the comment at the top of config files.
+sub header_comment {
+ my $self = shift;
+ my $name = shift;
+
+ my $text = "# DO NOT EDIT - Generated for " . $name . " by " .
&name_version_string($self) . " on " . `date`;
+ return $text;
+}
+
+#retrieves parameter data for a specific server by searching by the
server's assigned profile.
+sub param_data {
+ my $self = shift;
+ my $server_obj = shift;
+ my $filename = shift;
+ my $data;
+
+ my $rs = $self->db->resultset('ProfileParameter')->search( { -and => [
profile => $server_obj->profile->id, 'parameter.config_file' => $filename ] },
+ { prefetch => [ { parameter => undef }, { profile => undef } ]
} );
+ while ( my $row = $rs->next ) {
+ if ( $row->parameter->name eq "location" ) {
+ next;
+ }
+ if ( $row->parameter->name eq "scope" ) {
+ next;
+ }
+ my $value = $row->parameter->value;
+
+ # some files have multiple lines with the same key... handle
that with param id.
+ my $key = $row->parameter->name;
+ if ( defined( $data->{$key} ) ) {
+ $key .= "__" . $row->parameter->id;
+ }
+ if ( $value =~ /^STRING __HOSTNAME__$/ ) {
+ $value = "STRING " . $server_obj->host_name . "." .
$server_obj->domain_name;
+ }
+ $data->{$key} = $value;
+ }
+ return $data;
+}
+
+#retrieves parameter data for a specific profile by searching by the
profile id.
+sub profile_param_data {
+ my $self = shift;
+ my $profile = shift;
+ my $filename = shift;
+ my $data;
+
+ my $rs = $self->db->resultset('ProfileParameter')->search( { -and => [
profile => $profile, 'parameter.config_file' => $filename ] },
+ { prefetch => [ { parameter => undef }, { profile => undef } ]
} );
+ while ( my $row = $rs->next ) {
+ if ( $row->parameter->name eq "location" ) {
+ next;
+ }
+ if ( $row->parameter->name eq "scope" ) {
+ next;
+ }
+ my $value = $row->parameter->value;
+
+ # some files have multiple lines with the same key... handle
that with param id.
+ my $key = $row->parameter->name;
+ if ( defined( $data->{$key} ) ) {
+ $key .= "__" . $row->parameter->id;
+ }
+ $data->{$key} = $value;
+ }
+ return $data;
+}
+
+#searches for the a specific parameter by name in a specific profile by
profile ID and returns the data.
+sub profile_param_value {
+ my $self = shift;
+ my $pid = shift;
+ my $file = shift;
+ my $param_name = shift;
+ my $default = shift;
+
+ # assign $ds_domain, $weight and $port, and cache the results
%profile_cache
+ my $param =
+ $self->db->resultset('ProfileParameter')
+ ->search( { -and => [ profile => $pid, 'parameter.config_file'
=> $file, 'parameter.name' => $param_name ] },
+ { prefetch => [ 'parameter', 'profile' ] } )->first();
+
+ return ( defined $param ? $param->parameter->value : $default );
+}
+
+#gets the delivery service data for an entire CDN.
+sub cdn_ds_data {
+ my $self = shift;
+ my $id = shift;
+ my $dsinfo;
+
+ my $rs;
+ $rs = $self->db->resultset('DeliveryServiceInfoForCdnList')->search(
{}, { bind => [$id] } );
+
+ my $j = 0;
+ while ( my $row = $rs->next ) {
+ my $org_server = $row->org_server_fqdn;
+ my $dscp = $row->dscp;
+ my $re_type = $row->re_type;
+ my $ds_type = $row->ds_type;
+ my $signed = $row->signed;
+ my $qstring_ignore = $row->qstring_ignore;
+ my $ds_xml_id = $row->xml_id;
+ my $ds_domain = $row->domain_name;
+ my $edge_header_rewrite = $row->edge_header_rewrite;
+ my $mid_header_rewrite = $row->mid_header_rewrite;
+ my $regex_remap = $row->regex_remap;
+ my $protocol = $row->protocol;
+ my $range_request_handling = $row->range_request_handling;
+ my $origin_shield = $row->origin_shield;
+ my $cacheurl = $row->cacheurl;
+ my $remap_text = $row->remap_text;
+ my $multi_site_origin = $row->multi_site_origin;
+ my $multi_site_origin_algorithm =
$row->multi_site_origin_algorithm;
+
+ if ( $re_type eq 'HOST_REGEXP' && $ds_type ne 'ANY_MAP' ) {
+ my $host_re = $row->pattern;
+ my $map_to = $org_server . "/";
+ if ( $host_re =~ /\.\*$/ ) {
+ my $re = $host_re;
+ $re =~ s/\\//g;
+ $re =~ s/\.\*//g;
+ my $hname = $ds_type =~ /^DNS/ ? "edge" :
"ccr";
+ my $portstr = ":" . "SERVER_TCP_PORT";
+ my $map_from = "http://"; . $hname . $re .
$ds_domain . $portstr . "/";
+ if ( $protocol == 0 ) {
+
$dsinfo->{dslist}->[$j]->{"remap_line"}->{$map_from} = $map_to;
+ }
+ elsif ( $protocol == 1 || $protocol == 3 ) {
+ $map_from = "https://"; . $hname . $re .
$ds_domain . "/";
+
$dsinfo->{dslist}->[$j]->{"remap_line"}->{$map_from} = $map_to;
+ }
+ elsif ( $protocol == 2 ) {
+
+ #add the first one with http
+
$dsinfo->{dslist}->[$j]->{"remap_line"}->{$map_from} = $map_to;
+
+ #add the second one for https
+ my $map_from2 = "https://"; . $hname .
$re . $ds_domain . "/";
+
$dsinfo->{dslist}->[$j]->{"remap_line2"}->{$map_from2} = $map_to;
+ }
+ }
+ else {
+ my $map_from = "http://"; . $host_re . "/";
+ if ( $protocol == 0 ) {
+
$dsinfo->{dslist}->[$j]->{"remap_line"}->{$map_from} = $map_to;
+ }
+ elsif ( $protocol == 1 || $protocol == 3 ) {
+ $map_from = "https://"; . $host_re . "/";
+
$dsinfo->{dslist}->[$j]->{"remap_line"}->{$map_from} = $map_to;
+ }
+ elsif ( $protocol == 2 ) {
+
+ #add the first with http
+
$dsinfo->{dslist}->[$j]->{"remap_line"}->{$map_from} = $map_to;
+
+ #add the second with https
+ my $map_from2 = "https://"; . $host_re .
"/";
+
$dsinfo->{dslist}->[$j]->{"remap_line2"}->{$map_from2} = $map_to;
+ }
+ }
+ }
+
+ $dsinfo->{dslist}->[$j]->{"dscp"} =
$dscp;
+ $dsinfo->{dslist}->[$j]->{"org"} =
$org_server;
+ $dsinfo->{dslist}->[$j]->{"type"} =
$ds_type;
+ $dsinfo->{dslist}->[$j]->{"domain"} =
$ds_domain;
+ $dsinfo->{dslist}->[$j]->{"signed"} =
$signed;
+ $dsinfo->{dslist}->[$j]->{"qstring_ignore"} =
$qstring_ignore;
+ $dsinfo->{dslist}->[$j]->{"ds_xml_id"} =
$ds_xml_id;
+ $dsinfo->{dslist}->[$j]->{"edge_header_rewrite"} =
$edge_header_rewrite;
+ $dsinfo->{dslist}->[$j]->{"mid_header_rewrite"} =
$mid_header_rewrite;
+ $dsinfo->{dslist}->[$j]->{"regex_remap"} =
$regex_remap;
+ $dsinfo->{dslist}->[$j]->{"range_request_handling"} =
$range_request_handling;
+ $dsinfo->{dslist}->[$j]->{"origin_shield"} =
$origin_shield;
+ $dsinfo->{dslist}->[$j]->{"cacheurl"} =
$cacheurl;
+ $dsinfo->{dslist}->[$j]->{"remap_text"} =
$remap_text;
+ $dsinfo->{dslist}->[$j]->{"multi_site_origin"} =
$multi_site_origin;
+ $dsinfo->{dslist}->[$j]->{"multi_site_origin_algorithm"} =
$multi_site_origin_algorithm;
+
+ if ( defined($edge_header_rewrite) ) {
+ my $fname = "hdr_rw_" . $ds_xml_id . ".config";
+ $dsinfo->{dslist}->[$j]->{"hdr_rw_file"} = $fname;
+ }
+ if ( defined($mid_header_rewrite) ) {
+ my $fname = "hdr_rw_mid_" . $ds_xml_id . ".config";
+ $dsinfo->{dslist}->[$j]->{"mid_hdr_rw_file"} = $fname;
+ }
+ if ( defined($cacheurl) ) {
+ my $fname = "cacheurl_" . $ds_xml_id . ".config";
+ $dsinfo->{dslist}->[$j]->{"cacheurl_file"} = $fname;
+ }
+
+ $j++;
+ }
+
+ # $self->app->session->{dsinfo} = $dsinfo;
+ return $dsinfo;
+}
+
+#gets the delivery service data for a specific server.
+sub ds_data {
+ my $self = shift;
+ my $server_obj = shift;
+ my $dsinfo;
+
+ $dsinfo->{host_name} = $server_obj->host_name;
+ $dsinfo->{domain_name} = $server_obj->domain_name;
+
+ my @server_ids = ();
+ my $rs;
+ if ( $server_obj->type->name =~ m/^MID/ ) {
+
+ # the mids will do all deliveryservices in this CDN
+ my $domain = $self->profile_param_value(
$server_obj->profile->id, 'CRConfig.json', 'domain_name', '' );
+ $rs =
$self->db->resultset('DeliveryServiceInfoForDomainList')->search( {}, { bind =>
[$domain] } );
+ }
+ else {
+ $rs =
$self->db->resultset('DeliveryServiceInfoForServerList')->search( {}, { bind =>
[ $server_obj->id ] } );
+ }
+
+ my $j = 0;
+ while ( my $row = $rs->next ) {
+ my $org_server = $row->org_server_fqdn;
+ my $dscp = $row->dscp;
+ my $re_type = $row->re_type;
+ my $ds_type = $row->ds_type;
+ my $signed = $row->signed;
+ my $qstring_ignore = $row->qstring_ignore;
+ my $ds_xml_id = $row->xml_id;
+ my $ds_domain = $row->domain_name;
+ my $edge_header_rewrite = $row->edge_header_rewrite;
+ my $mid_header_rewrite = $row->mid_header_rewrite;
+ my $regex_remap = $row->regex_remap;
+ my $protocol = $row->protocol;
+ my $range_request_handling = $row->range_request_handling;
+ my $origin_shield = $row->origin_shield;
+ my $cacheurl = $row->cacheurl;
+ my $remap_text = $row->remap_text;
+ my $multi_site_origin = $row->multi_site_origin;
+ my $multi_site_origin_algorithm =
$row->multi_site_origin_algorithm;
+
+ if ( $re_type eq 'HOST_REGEXP' && $ds_type ne 'ANY_MAP' ) {
+ my $host_re = $row->pattern;
+ my $map_to = $org_server . "/";
+ if ( $host_re =~ /\.\*$/ ) {
+ my $re = $host_re;
+ $re =~ s/\\//g;
+ $re =~ s/\.\*//g;
+ my $hname = $ds_type =~ /^DNS/ ? "edge" : "ccr";
+ my $portstr = "";
+ if ( $hname eq "ccr" && $server_obj->tcp_port >
0 && $server_obj->tcp_port != 80 ) {
+ $portstr = ":" . $server_obj->tcp_port;
+ }
+ my $map_from = "http://"; . $hname . $re .
$ds_domain . $portstr . "/";
+ if ( $protocol == 0 ) {
+
$dsinfo->{dslist}->[$j]->{"remap_line"}->{$map_from} = $map_to;
+ }
+ elsif ( $protocol == 1 || $protocol == 3 ) {
+ $map_from = "https://"; . $hname . $re .
$ds_domain . "/";
+
$dsinfo->{dslist}->[$j]->{"remap_line"}->{$map_from} = $map_to;
+ }
+ elsif ( $protocol == 2 ) {
+
+ #add the first one with http
+
$dsinfo->{dslist}->[$j]->{"remap_line"}->{$map_from} = $map_to;
+
+ #add the second one for https
+ my $map_from2 = "https://"; . $hname .
$re . $ds_domain . "/";
+
$dsinfo->{dslist}->[$j]->{"remap_line2"}->{$map_from2} = $map_to;
+ }
+ }
+ else {
+ my $map_from = "http://"; . $host_re . "/";
+ if ( $protocol == 0 ) {
+
$dsinfo->{dslist}->[$j]->{"remap_line"}->{$map_from} = $map_to;
+ }
+ elsif ( $protocol == 1 || $protocol == 3 ) {
+ $map_from = "https://"; . $host_re . "/";
+
$dsinfo->{dslist}->[$j]->{"remap_line"}->{$map_from} = $map_to;
+ }
+ elsif ( $protocol == 2 ) {
+
+ #add the first with http
+
$dsinfo->{dslist}->[$j]->{"remap_line"}->{$map_from} = $map_to;
+
+ #add the second with https
+ my $map_from2 = "https://"; . $host_re .
"/";
+
$dsinfo->{dslist}->[$j]->{"remap_line2"}->{$map_from2} = $map_to;
+ }
+ }
+ }
+ $dsinfo->{dslist}->[$j]->{"dscp"} =
$dscp;
+ $dsinfo->{dslist}->[$j]->{"org"} =
$org_server;
+ $dsinfo->{dslist}->[$j]->{"type"} =
$ds_type;
+ $dsinfo->{dslist}->[$j]->{"domain"} =
$ds_domain;
+ $dsinfo->{dslist}->[$j]->{"signed"} =
$signed;
+ $dsinfo->{dslist}->[$j]->{"qstring_ignore"} =
$qstring_ignore;
+ $dsinfo->{dslist}->[$j]->{"ds_xml_id"} =
$ds_xml_id;
+ $dsinfo->{dslist}->[$j]->{"edge_header_rewrite"} =
$edge_header_rewrite;
+ $dsinfo->{dslist}->[$j]->{"mid_header_rewrite"} =
$mid_header_rewrite;
+ $dsinfo->{dslist}->[$j]->{"regex_remap"} =
$regex_remap;
+ $dsinfo->{dslist}->[$j]->{"range_request_handling"} =
$range_request_handling;
+ $dsinfo->{dslist}->[$j]->{"origin_shield"} =
$origin_shield;
+ $dsinfo->{dslist}->[$j]->{"cacheurl"} =
$cacheurl;
+ $dsinfo->{dslist}->[$j]->{"remap_text"} =
$remap_text;
+ $dsinfo->{dslist}->[$j]->{"multi_site_origin"} =
$multi_site_origin;
+ $dsinfo->{dslist}->[$j]->{"multi_site_origin_algorithm"} =
$multi_site_origin_algorithm;
+
+ if ( defined($edge_header_rewrite) ) {
+ my $fname = "hdr_rw_" . $ds_xml_id . ".config";
+ $dsinfo->{dslist}->[$j]->{"hdr_rw_file"} = $fname;
+ }
+ if ( defined($mid_header_rewrite) ) {
+ my $fname = "hdr_rw_mid_" . $ds_xml_id . ".config";
+ $dsinfo->{dslist}->[$j]->{"mid_hdr_rw_file"} = $fname;
+ }
+ if ( defined($cacheurl) ) {
+ my $fname = "cacheurl_" . $ds_xml_id . ".config";
+ $dsinfo->{dslist}->[$j]->{"cacheurl_file"} = $fname;
+ }
+
+ $j++;
+ }
+
+ return $dsinfo;
+}
+
+#generates the 12m_facts file
+sub facts {
+ my $self = shift;
+ my $server_obj = shift;
+ my $filename = shift;
+ my $scope = shift;
+ my $text = $self->header_comment( $server_obj->host_name );
+ $text .= "profile:" . $server_obj->profile->name . "\n";
+
+ return $text;
+}
+
+#generates a generic config file based on a server and parameters which
match the supplied filename.
+sub take_and_bake_server {
+ my $self = shift;
+ my $server_obj = shift;
+ my $filename = shift;
+ my $scope = shift;
+
+ my $data = $self->param_data( $server_obj, $filename );
+ my $text = $self->header_comment( $server_obj->host_name );
+ foreach my $parameter ( sort keys %{$data} ) {
+ $text .= $data->{$parameter} . "\n";
+ }
+ return $text;
+}
+
+#generates a generic config file based on a profile and parameters which
match the supplied filename.
+sub take_and_bake_profile {
+ my $self = shift;
+ my $profile_obj = shift;
+ my $filename = shift;
+ my $scope = shift;
+
+ my $data = $self->profile_param_data( $profile_obj->id, $filename );
+ my $text = $self->header_comment( $profile_obj->name );
+ foreach my $parameter ( sort keys %{$data} ) {
+ $text .= $data->{$parameter} . "\n";
+ }
+ return $text;
+}
+
+#generates a generic config file based on a profile and parameters which
match the supplied filename.
+#differs from take and bake in that it uses predefined separators.
+sub generic_profile_config {
+ my $self = shift;
+ my $profile_obj = shift;
+ my $filename = shift;
+ my $scope = shift;
+
+ my $sep = defined( $separator->{$filename} ) ? $separator->{$filename}
: " = ";
+
+ my $data = $self->profile_param_data( $profile_obj->id, $filename );
+ my $text = $self->header_comment( $profile_obj->name );
+ foreach my $parameter ( sort keys %{$data} ) {
+ my $p_name = $parameter;
+ $p_name =~ s/__\d+$//;
+ $text .= $p_name . $sep . $data->{$parameter} . "\n";
+ }
+
+ return $text;
+}
+
+#generates a generic config file based on a server and parameters which
match the supplied filename.
+#differs from take and bake in that it uses predefined separators.
+sub generic_server_config {
+ my $self = shift;
+ my $server_obj = shift;
+ my $filename = shift;
+ my $scope = shift;
+
+ my $sep = defined( $separator->{$filename} ) ? $separator->{$filename}
: " = ";
+
+ my $data = $self->param_data( $server_obj, $filename );
+ my $text = $self->header_comment( $server_obj->host_name );
+ foreach my $parameter ( sort keys %{$data} ) {
+ my $p_name = $parameter;
+ $p_name =~ s/__\d+$//;
+ $text .= $p_name . $sep . $data->{$parameter} . "\n";
+ }
+
+ return $text;
+}
+
+sub ats_dot_rules {
+ my $self = shift;
+ my $profile_obj = shift;
+ my $filename = shift;
+ my $scope = shift;
+
+ my $text = $self->header_comment( $profile_obj->name );
+ my $data = $self->profile_param_data( $profile_obj->id,
"storage.config" ); # ats.rules is based on the storage.config params
+
+ my $drive_prefix = $data->{Drive_Prefix};
+ my @drive_postfix = split( /,/, $data->{Drive_Letters} );
+ foreach my $l ( sort @drive_postfix ) {
+ $drive_prefix =~ s/\/dev\///;
+ $text .= "KERNEL==\"" . $drive_prefix . $l . "\",
OWNER=\"ats\"\n";
+ }
+ if ( defined( $data->{RAM_Drive_Prefix} ) ) {
+ $drive_prefix = $data->{RAM_Drive_Prefix};
+ @drive_postfix = split( /,/, $data->{RAM_Drive_Letters} );
+ foreach my $l ( sort @drive_postfix ) {
+ $drive_prefix =~ s/\/dev\///;
+ $text .= "KERNEL==\"" . $drive_prefix . $l . "\",
OWNER=\"ats\"\n";
+ }
+ }
+
+ return $text;
+}
+
+sub drop_qstring_dot_config {
+ my $self = shift;
+ my $profile_obj = shift;
+ my $filename = shift;
+ my $scope = shift;
+
+ my $text = $self->header_comment( $profile_obj->name );
+
+ my $drop_qstring = $self->profile_param_value( $profile_obj->id,
'drop_qstring.config', 'content', undef );
+
+ if ($drop_qstring) {
+ $text .= $drop_qstring . "\n";
+ }
+ else {
+ $text .= "/([^?]+) \$s://\$t/\$1\n";
+ }
+
+ return $text;
+}
+
+sub logs_xml_dot_config {
+ my $self = shift;
+ my $profile_obj = shift;
+ my $filename = shift;
+ my $scope = shift;
+
+ my $data = $self->profile_param_data( $profile_obj->id,
"logs_xml.config" );
+ my $text = $self->header_comment( $profile_obj->name );
+
+ my $log_format_name = $data->{"LogFormat.Name"}
|| "";
+ my $log_object_filename = $data->{"LogObject.Filename"}
|| "";
+ my $log_object_format = $data->{"LogObject.Format"}
|| "";
+ my $log_object_rolling_enabled =
$data->{"LogObject.RollingEnabled"} || "";
+ my $log_object_rolling_interval_sec =
$data->{"LogObject.RollingIntervalSec"} || "";
+ my $log_object_rolling_offset_hr =
$data->{"LogObject.RollingOffsetHr"} || "";
+ my $log_object_rolling_size_mb =
$data->{"LogObject.RollingSizeMb"} || "";
+ my $format = $data->{"LogFormat.Format"};
+ $format =~ s/"/\\\"/g;
+ $text .= "<LogFormat>\n";
+ $text .= " <Name = \"" . $log_format_name . "\"/>\n";
+ $text .= " <Format = \"" . $format . "\"/>\n";
+ $text .= "</LogFormat>\n";
+ $text .= "<LogObject>\n";
+ $text .= " <Format = \"" . $log_object_format . "\"/>\n";
+ $text .= " <Filename = \"" . $log_object_filename . "\"/>\n";
+ $text .= " <RollingEnabled = " . $log_object_rolling_enabled . "/>\n"
unless defined();
+ $text .= " <RollingIntervalSec = " . $log_object_rolling_interval_sec
. "/>\n";
+ $text .= " <RollingOffsetHr = " . $log_object_rolling_offset_hr .
"/>\n";
+ $text .= " <RollingSizeMb = " . $log_object_rolling_size_mb . "/>\n";
+ $text .= "</LogObject>\n";
+
+ return $text;
+}
+
+sub storage_dot_config_volume_text {
+ my $prefix = shift;
+ my $letters = shift;
+ my $volume = shift;
+
+ my $text = "";
+ my @postfix = split( /,/, $letters );
+ foreach my $l ( sort @postfix ) {
+ $text .= $prefix . $l;
+ $text .= " volume=" . $volume;
+ $text .= "\n";
+ }
+ return $text;
+}
+
+sub storage_dot_config {
+ my $self = shift;
+ my $profile_obj = shift;
+ my $filename = shift;
+ my $scope = shift;
+
+ my $text = $self->header_comment( $profile_obj->name );
+ my $data = $self->profile_param_data( $profile_obj->id,
"storage.config" );
+
+ my $next_volume = 1;
+ if ( defined( $data->{Drive_Prefix} ) ) {
+ $text .= storage_dot_config_volume_text( $data->{Drive_Prefix},
$data->{Drive_Letters}, $next_volume );
+ $next_volume++;
+ }
+
+ if ( defined( $data->{RAM_Drive_Prefix} ) ) {
+ $text .= storage_dot_config_volume_text(
$data->{RAM_Drive_Prefix}, $data->{RAM_Drive_Letters}, $next_volume );
+ $next_volume++;
+ }
+
+ if ( defined( $data->{SSD_Drive_Prefix} ) ) {
+ $text .= storage_dot_config_volume_text(
$data->{SSD_Drive_Prefix}, $data->{SSD_Drive_Letters}, $next_volume );
+ $next_volume++;
+ }
+
+ return $text;
+}
+
+sub to_ext_dot_config {
+ my $self = shift;
+ my $server_obj = shift;
+ my $filename = shift;
+ my $scope = shift;
+
+ my $text = $self->header_comment( $server_obj->host_name );
+
+ # get the subroutine name for this file from the parameter
+ my $subroutine = $self->profile_param_value( $server_obj->profile->id,
$filename, 'SubRoutine', undef );
+ $self->app->log->error( "ToExtDotConfigFile == " . $subroutine );
+
+ if ( defined $subroutine ) {
+ my $package;
+ ( $package = $subroutine ) =~ s/(.*)(::)(.*)/$1/;
+ eval "use $package;";
+
+ # And call it - the below calls the subroutine in the var
$subroutine.
+ no strict 'refs';
+ $text .= $subroutine->( $self, $server_obj->host_name,
$filename );
+ return $text;
+ }
+ else {
+ return;
+ }
+}
+
+sub get_num_volumes {
+ my $data = shift;
+
+ my $num = 0;
+ my @drive_prefixes = qw( Drive_Prefix SSD_Drive_Prefix
RAM_Drive_Prefix);
+ foreach my $pre (@drive_prefixes) {
+ if ( exists $data->{$pre} ) {
+ $num++;
+ }
+ }
+ return $num;
+}
+
+sub volume_dot_config_volume_text {
+ my $volume = shift;
+ my $num_volumes = shift;
+ my $size = int( 100 / $num_volumes );
+ return "volume=$volume scheme=http size=$size%\n";
+}
+
+sub volume_dot_config {
+ my $self = shift;
+ my $profile_obj = shift;
+ my $filename = shift;
+ my $scope = shift;
+
+ my $data = $self->profile_param_data( $profile_obj->id,
"storage.config" );
+ my $text = $self->header_comment( $profile_obj->name );
+
+ my $num_volumes = get_num_volumes($data);
+
+ my $next_volume = 1;
+ $text .= "# 12M NOTE: This is running with forced volumes - the size is
irrelevant\n";
+ if ( defined( $data->{Drive_Prefix} ) ) {
+ $text .= volume_dot_config_volume_text( $next_volume,
$num_volumes );
+ $next_volume++;
+ }
+ if ( defined( $data->{RAM_Drive_Prefix} ) ) {
+ $text .= volume_dot_config_volume_text( $next_volume,
$num_volumes );
+ $next_volume++;
+ }
+ if ( defined( $data->{SSD_Drive_Prefix} ) ) {
+ $text .= volume_dot_config_volume_text( $next_volume,
$num_volumes );
+ $next_volume++;
+ }
+
+ return $text;
+}
+
+# This is a temporary workaround until we have real partial object caching
support in ATS, so hardcoding for now
+sub bg_fetch_dot_config {
+ my $self = shift;
+ my $cdn_obj = shift;
+ my $filename = shift;
+ my $scope = shift;
+
+ my $text = $self->header_comment( $cdn_obj->name );
+ $text .= "include User-Agent *\n";
+
+ return $text;
+}
+
+sub header_rewrite_dot_config {
+ my $self = shift;
+ my $cdn_obj = shift;
+ my $filename = shift;
+ my $scope = shift;
+
+ my $text = $self->header_comment( $cdn_obj->name );
+ my $ds_xml_id = undef;
+ if ( $filename =~ /^hdr_rw_mid_(.*)\.config$/ ) {
+ $ds_xml_id = $1;
+ my $ds = $self->db->resultset('Deliveryservice')->search( {
xml_id => $ds_xml_id }, { prefetch => [ 'type', 'profile' ] } )->first();
+ my $actions = $ds->mid_header_rewrite;
+ $text .= $actions . "\n";
+ }
+ elsif ( $filename =~ /^hdr_rw_(.*)\.config$/ ) {
+ $ds_xml_id = $1;
+ my $ds = $self->db->resultset('Deliveryservice')->search( {
xml_id => $ds_xml_id }, { prefetch => [ 'type', 'profile' ] } )->first();
+ my $actions = $ds->edge_header_rewrite;
+ $text .= $actions . "\n";
+ }
+
+ $text =~ s/\s*__RETURN__\s*/\n/g;
+
+ return $text;
+}
+
+sub cacheurl_dot_config {
+ my $self = shift;
+ my $cdn_obj = shift;
+ my $filename = shift;
+ my $scope = shift;
+
+ my $text = $self->header_comment( $cdn_obj->name );
+ my $data = $self->cdn_ds_data( $cdn_obj->id );
+
+ if ( $filename eq "cacheurl_qstring.config" ) { # This is the per
remap drop qstring w cacheurl use case, the file is the same for all remaps
+ $text .= "http://([^?]+)(?:\\?|\$) http://\$1\n";;
+ $text .= "https://([^?]+)(?:\\?|\$) https://\$1\n";;
+ }
+ elsif ( $filename =~ /cacheurl_(.*).config/ )
+ { # Yes, it's possibe to have the same plugin invoked multiple times
on the same remap line, this is from the remap entry
+ my $ds_xml_id = $1;
+ my $ds = $self->db->resultset('Deliveryservice')->search( {
xml_id => $ds_xml_id }, { prefetch => [ 'type', 'profile' ] } )->first();
+ if ($ds) {
+ $text .= $ds->cacheurl . "\n";
+ }
+ }
+ elsif ( $filename eq "cacheurl.config" ) { # this is the global drop
qstring w cacheurl use case
+ foreach my $remap ( @{ $data->{dslist} } ) {
+ if ( $remap->{qstring_ignore} == 1 ) {
+ my $org = $remap->{org};
+ $org =~ /(https?:\/\/)(.*)/;
+ $text .= "$1(" . $2 . "/[^?]+)(?:\\?|\$)
$1\$1\n";
+ }
+ }
+
+ }
+
+ $text =~ s/\s*__RETURN__\s*/\n/g;
+
+ return $text;
+}
+
+sub regex_remap_dot_config {
+ my $self = shift;
+ my $cdn_obj = shift;
+ my $filename = shift;
+ my $scope = shift;
+
+ my $text = $self->header_comment( $cdn_obj->name );
+
+ if ( $filename =~ /^regex_remap_(.*)\.config$/ ) {
+ my $ds_xml_id = $1;
+ my $ds = $self->db->resultset('Deliveryservice')->search( {
xml_id => $ds_xml_id }, { prefetch => [ 'type', 'profile' ] } )->first();
+ $text .= $ds->regex_remap . "\n";
+ }
+
+ $text =~ s/\s*__RETURN__\s*/\n/g;
+
+ return $text;
+}
+
+sub regex_revalidate_dot_config {
+ my $self = shift;
+ my $cdn_obj = shift;
+ my $filename = shift;
+ my $scope = shift;
+
+ my $text = "# DO NOT EDIT - Generated for CDN " . $cdn_obj->name . " by
" . &name_version_string($self) . " on " . `date`;
+
+ my $max_days =
+ $self->db->resultset('Parameter')->search( { name =>
"maxRevalDurationDays" }, { config_file => "regex_revalidate.config" }
)->get_column('value')
+ ->single;
+ my $interval = "> now() - interval '$max_days day'";
+
+ my %regex_time;
+ $max_days =
--- End diff --
this query was already run
> Improve efficiency of ATS config generation
> -------------------------------------------
>
> Key: TC-32
> URL: https://issues.apache.org/jira/browse/TC-32
> Project: Traffic Control
> Issue Type: Improvement
> Components: Traffic Ops API
> Reporter: Derek Gelinas
> Assignee: Derek Gelinas
> Fix For: 1.9.0
>
>
> Currently when generating ATS configuration files, each server calls for its
> individual files from traffic ops. This is a very database-intensive process
> that is not scalable enough. I propose the following changes:
> 1) Generate the configuration files only as many times as needed - many files
> are the same (or can be) for the entire CDN or server profile.
> 2) Once generated, the files can be cached locally. Requests for the files
> by ORT will result in pulling down these cached files rather than many
> hundreds of DB queries each time.
> 3) Migrate the routes for these files to the API. Each call will have
> multiple options - a request made with no options would return the cached
> file, another option would return the current DB data, and a third option
> would update the cached file with the current information in the DB.
> 4) When an update is queued, generate and cache the configuration files, then
> activate the update flag for the relevant servers once the cached file
> generation is completed.
> In this way, triggering updates for the caches will result in an initial
> increase in activity as traffic ops generates the files needed, followed by
> the much lower impact of the files themselves being requested by the caches.
> I believe we can cut down the number of server-specific configuration files
> to only 5 intially, and potentially fewer by having ORT fill in certain
> fields on files like records.config with local data during processing.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
