[
https://issues.apache.org/jira/browse/TC-32?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15770273#comment-15770273
]
ASF GitHub Bot commented on TC-32:
----------------------------------
Github user mitchell852 commented on a diff in the pull request:
https://github.com/apache/incubator-trafficcontrol/pull/141#discussion_r93640850
--- Diff: traffic_ops/app/lib/API/Configs/ApacheTrafficServer.pm ---
@@ -0,0 +1,1959 @@
+package API::Configs::ApacheTrafficServer;
+
+#
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+#
+use UI::Utils;
+use Mojo::Base 'Mojolicious::Controller';
+use Date::Manip;
+use NetAddr::IP;
+use Data::Dumper;
+use UI::DeliveryService;
+use JSON;
+use API::DeliveryService::KeysUrlSig qw(URL_SIG_KEYS_BUCKET);
+use URI;
+use File::Basename;
+use File::Path;
+
+#Sub to generate ORT json
+sub ort {
+ my $self = shift;
+ my $id = $self->param('id');
+ my $filename = 'ort';
+ my $scope = 'server';
+
+ ##check user access
+ if ( !&is_oper($self) ) {
+ return $self->forbidden();
+ }
+
+ ##verify that a valid server ID has been used
+ my $server_obj = $self->server_data($id);
+ if ( !defined($server_obj) ) {
+ return $self->not_found();
+ }
+
+ my $data_obj;
+ my $host_name = $server_obj->host_name;
+
+ my %condition = ( 'me.host_name' => $host_name );
+ my $rs_profile = $self->db->resultset('Server')->search( \%condition, {
prefetch => [ 'cdn', 'profile' ] } );
+
+ my $row = $rs_profile->next;
+ if ($row) {
+ my $cdn_name = defined( $row->cdn_id ) ? $row->cdn->name : "";
+
+ $data_obj->{'profile'}->{'name'} = $row->profile->name;
+ $data_obj->{'profile'}->{'id'} = $row->profile->id;
+ $data_obj->{'other'}->{'CDN_name'} = $cdn_name;
+
+ %condition = (
+ 'profile_parameters.profile' =>
$data_obj->{'profile'}->{'id'},
+ -or => [ 'name' => 'location',
'name' => 'scope' ]
+ );
+ my $rs_config = $self->db->resultset('Parameter')->search(
\%condition, { join => 'profile_parameters' } );
+ while ( my $row = $rs_config->next ) {
+ if ( $row->name eq 'location' ) {
+ $data_obj->{'config_files'}->{
$row->config_file }->{'location'} = $row->value;
+ }
+ elsif ( $row->name eq 'scope' ) {
+ $data_obj->{'config_files'}->{
$row->config_file }->{'scope'} = $row->value;
+ }
+ }
+ }
+
+ foreach my $file ( keys %$data_obj->{'config_files'} ) {
+ if ( !defined( $data_obj->{'config_files'}->{$file}->{'scope'}
) ) {
+ $data_obj->{'config_files'}->{$file}->{'scope'} =
$self->get_scope($file);
+ }
+ }
+
+ my $file_contents = encode_json($data_obj);
+
+ return $self->render( text => $file_contents, format => 'txt' );
+}
+
+#entry point for server scope api route.
+sub get_server_config {
+ my $self = shift;
+ my $filename = $self->param("filename");
+ my $id = $self->param('id');
+ my $scope = $self->get_scope($filename);
+
+ ##check user access
+ if ( !&is_oper($self) ) {
+ return $self->forbidden();
+ }
+
+ ##check the scope - is this the correct route?
+ if ( $scope ne 'server' ) {
+ return $self->alert( "Error - incorrect file scope for route
used. Please use the " . $scope . " route." );
+ }
+
+ ##verify that a valid server ID has been used
+ my $server_obj = $self->server_data($id);
+ if ( !defined($server_obj) ) {
+ return $self->not_found();
+ }
+
+ #generate the config file using the appropriate function
+ my $file_contents;
+ if ( $filename eq "12M_facts" ) { $file_contents = $self->facts(
$server_obj, $filename, $scope ); }
+ elsif ( $filename =~ /to_ext_.*\.config/ ) { $file_contents =
$self->to_ext_dot_config( $server_obj, $filename, $scope ); }
+ elsif ( $filename eq "ip_allow.config" ) { $file_contents =
$self->ip_allow_dot_config( $server_obj, $filename, $scope ); }
+ elsif ( $filename eq "parent.config" ) { $file_contents =
$self->parent_dot_config( $server_obj, $filename, $scope ); }
+ elsif ( $filename eq "records.config" ) { $file_contents =
$self->generic_server_config( $server_obj, $filename, $scope ); }
+ elsif ( $filename eq "remap.config" ) { $file_contents =
$self->remap_dot_config( $server_obj, $filename, $scope ); }
+ elsif ( $filename eq "hosting.config" ) { $file_contents =
$self->hosting_dot_config( $server_obj, $filename, $scope ); }
+ elsif ( $filename eq "cache.config" ) { $file_contents =
$self->cache_dot_config( $server_obj, $filename, $scope ); }
+ elsif ( $filename eq "packages" ) {
+ $file_contents = $self->get_package_versions( $server_obj,
$filename, $scope );
+ $file_contents = encode_json($file_contents);
+ }
+ elsif ( $filename eq "chkconfig" ) {
+ $file_contents = $self->get_chkconfig( $server_obj, $filename,
$scope );
+ $file_contents = encode_json($file_contents);
+ }
+ else {
+ my $file_param = $self->db->resultset('Parameter')->search( [
config_file => $filename ] )->single;
+ if ( !defined($file_param) ) {
+ return $self->not_found();
+ }
+ $file_contents = $self->take_and_bake_server( $server_obj,
$filename, $scope );
+ }
+
+ #if we get an empty file, just send back an error.
+ if ( !defined($file_contents) ) {
+ return $self->not_found();
+ }
+
+ #return the file contents for fetch and db actions.
+ return $self->render( text => $file_contents, format => 'txt' );
+}
+
+#entry point for cdn scope api route.
+sub get_cdn_config {
+ my $self = shift;
+ my $filename = $self->param("filename");
+ my $id = $self->param('id');
+ my $scope = $self->get_scope($filename);
+
+ ##check user access
+ if ( !&is_oper($self) ) {
+ return $self->forbidden();
+ }
+ print STDERR "well i'm here anyway.";
+ ##check the scope - is this the correct route?
+ if ( $scope ne 'cdn' ) {
+ return $self->alert( "Error - incorrect file scope for route
used. Please use the " . $scope . " route." );
+ }
+
+ ##verify that a valid cdn ID has been used
+ my $cdn_obj = $self->cdn_data($id);
+ if ( !defined($cdn_obj) ) {
+ return $self->not_found();
+ }
+
+ #generate the config file using the appropriate function
+ my $file_contents;
+ if ( $filename eq "bg_fetch.config" ) { $file_contents =
$self->bg_fetch_dot_config( $cdn_obj, $filename, $scope ); }
+ elsif ( $filename =~ /cacheurl.*\.config/ ) { $file_contents =
$self->cacheurl_dot_config( $cdn_obj, $filename, $scope ); }
+ elsif ( $filename =~ /hdr_rw_.*\.config/ ) { $file_contents =
$self->header_rewrite_dot_config( $cdn_obj, $filename, $scope ); }
+ elsif ( $filename =~ /regex_remap_.*\.config/ ) { $file_contents =
$self->regex_remap_dot_config( $cdn_obj, $filename, $scope ); }
+ elsif ( $filename eq "regex_revalidate.config" ) { $file_contents =
$self->regex_revalidate_dot_config( $cdn_obj, $filename, $scope ); }
+ elsif ( $filename =~ /set_dscp_.*\.config/ ) { $file_contents =
$self->set_dscp_dot_config( $cdn_obj, $filename, $scope ); }
+ elsif ( $filename eq "ssl_multicert.config" ) { $file_contents =
$self->ssl_multicert_dot_config( $cdn_obj, $filename, $scope ); }
+ else { return
$self->not_found(); }
+
+ if ( !defined($file_contents) ) {
+ return $self->not_found();
+ }
+
+ return $self->render( text => $file_contents, format => 'txt' );
+}
+
+#entry point for profile scope api route.
+sub get_profile_config {
+ my $self = shift;
+ my $filename = $self->param("filename");
+ my $id = $self->param('id');
+ my $scope = $self->get_scope($filename);
+
+ ##check user access
+ if ( !&is_oper($self) ) {
+ return $self->forbidden();
+ }
+
+ ##check the scope - is this the correct route?
+ if ( $scope ne 'profile' ) {
+ return $self->alert( "Error - incorrect file scope for route
used. Please use the " . $scope . " route." );
+ }
+
+ ##verify that a valid profile ID has been used
+ my $profile_obj = $self->profile_data($id);
+ if ( !defined($profile_obj) ) {
+ return $self->not_found();
+ }
+
+ #generate the config file using the appropriate function
+ my $file_contents;
+ if ( $filename eq "50-ats.rules" ) { $file_contents =
$self->ats_dot_rules( $profile_obj, $filename, $scope ); }
+ elsif ( $filename eq "astats.config" ) { $file_contents =
$self->generic_profile_config( $profile_obj, $filename, $scope ); }
+ elsif ( $filename eq "drop_qstring.config" ) { $file_contents =
$self->drop_qstring_dot_config( $profile_obj, $filename, $scope ); }
+ elsif ( $filename eq "logs_xml.config" ) { $file_contents =
$self->logs_xml_dot_config( $profile_obj, $filename, $scope ); }
+ elsif ( $filename eq "plugin.config" ) { $file_contents =
$self->generic_profile_config( $profile_obj, $filename, $scope ); }
+ elsif ( $filename eq "storage.config" ) { $file_contents =
$self->storage_dot_config( $profile_obj, $filename, $scope ); }
+ elsif ( $filename eq "sysctl.conf" ) { $file_contents =
$self->generic_profile_config( $profile_obj, $filename, $scope ); }
+ elsif ( $filename =~ /url_sig_.*\.config/ ) { $file_contents =
$self->url_sig_dot_config( $profile_obj, $filename, $scope ); }
+ elsif ( $filename eq "volume.config" ) { $file_contents =
$self->volume_dot_config( $profile_obj, $filename, $scope ); }
+ else {
+ my $file_param = $self->db->resultset('Parameter')->search( [
config_file => $filename ] )->single;
+ if ( !defined($file_param) ) {
+ return $self->not_found();
+ }
+ $file_contents = $self->take_and_bake_profile( $profile_obj,
$filename, $scope );
+ }
+
+ if ( !defined($file_contents) ) {
+ return $self->not_found();
+ }
+
+ return $self->render( text => $file_contents, format => 'txt' );
+}
+
+my $separator ||= {
+ "records.config" => " ",
+ "plugin.config" => " ",
+ "sysctl.conf" => " = ",
+ "url_sig_.config" => " = ",
+ "astats.config" => "=",
+};
+
+#identify the correct scope for each filename. if not found, returns
server scope as any
+#undefined parameter based configs are designed with server scope using
the take-and-bake sub.
+sub get_scope {
+ my $self = shift;
+ my $fname = shift;
+ my $scope;
+
+ if ( $fname eq "12M_facts" ) { $scope = 'server' }
+ elsif ( $fname eq "ip_allow.config" ) { $scope = 'server' }
+ elsif ( $fname eq "parent.config" ) { $scope = 'server' }
+ elsif ( $fname eq "records.config" ) { $scope = 'server' }
+ elsif ( $fname eq "remap.config" ) { $scope = 'server' }
+ elsif ( $fname =~ /to_ext_.*\.config/ ) { $scope = 'server' }
+ elsif ( $fname eq "hosting.config" ) { $scope = 'server' }
+ elsif ( $fname eq "cache.config" ) { $scope = 'server' }
+ elsif ( $fname eq "packages" ) { $scope = 'server' }
+ elsif ( $fname eq "chkconfig" ) { $scope = 'server' }
+ elsif ( $fname eq "50-ats.rules" ) { $scope = 'profile' }
+ elsif ( $fname eq "astats.config" ) { $scope = 'profile' }
+ elsif ( $fname eq "drop_qstring.config" ) { $scope = 'profile' }
+ elsif ( $fname eq "logs_xml.config" ) { $scope = 'profile' }
+ elsif ( $fname eq "plugin.config" ) { $scope = 'profile' }
+ elsif ( $fname eq "storage.config" ) { $scope = 'profile' }
+ elsif ( $fname eq "sysctl.conf" ) { $scope = 'profile' }
+ elsif ( $fname =~ /url_sig_.*\.config/ ) { $scope = 'profile' }
+ elsif ( $fname eq "volume.config" ) { $scope = 'profile' }
+ elsif ( $fname eq "bg_fetch.config" ) { $scope = 'cdn' }
+ elsif ( $fname =~ /cacheurl.*\.config/ ) { $scope = 'cdn' }
+ elsif ( $fname =~ /hdr_rw_.*\.config/ ) { $scope = 'cdn' }
+ elsif ( $fname =~ /regex_remap_.*\.config/ ) { $scope = 'cdn' }
+ elsif ( $fname eq "regex_revalidate.config" ) { $scope = 'cdn' }
+ elsif ( $fname =~ /set_dscp_.*\.config/ ) { $scope = 'cdn' }
+ elsif ( $fname eq "ssl_multicert.config" ) { $scope = 'cdn' }
+ else {
+ $scope = $self->db->resultset('Parameter')->search( { -and => [
name => 'scope', config_file => $fname ] } )->get_column('value')->first();
+ if ( !defined($scope) ) {
+ $self->app->log->error("Filename not found. Setting
Server scope.");
+ $scope = 'server';
+ }
+ }
+
+ return $scope;
+}
+
+#takes the server name or ID and turns it into a server object that can
reference either, making either work for the request.
+sub server_data {
+ my $self = shift;
+ my $id = shift;
+
+ my $server_obj;
+
+ #if an ID is passed, look up by ID. Otherwise, look up by hostname.
+ if ( $id =~ /^\d+$/ ) {
+ $server_obj = $self->db->resultset('Server')->search( { id =>
$id } )->single;
+ }
+ else {
+ $server_obj = $self->db->resultset('Server')->search( {
host_name => $id } )->single;
+ }
+
+ return $server_obj;
+}
+
+#takes the profile name or ID and turns it into a server object that can
reference either, making either work for the request.
+sub profile_data {
+ my $self = shift;
+ my $id = shift;
+
+ #if an ID is passed, look up by ID. Otherwise, look up by profile name.
+ my $profile_obj;
+ if ( $id =~ /^\d+$/ ) {
+ $profile_obj = $self->db->resultset('Profile')->search( { id =>
$id } )->single;
+ }
+ else {
+ $profile_obj = $self->db->resultset('Profile')->search( { name
=> $id } )->single;
+ }
+
+ return $profile_obj;
+}
+
+#takes the server name or ID and turns it into a server object that can
reference either, making either work for the request.
+sub cdn_data {
+ my $self = shift;
+ my $id = shift;
+ my $cdn_obj;
+
+ if ( $id =~ /^\d+$/ ) {
+ $cdn_obj = $self->db->resultset('Cdn')->search( { id => $id }
)->single;
+ }
+ else {
+ $cdn_obj = $self->db->resultset('Cdn')->search( { name => $id }
)->single;
+ }
+
+ return $cdn_obj;
+}
+
+#generates the comment at the top of config files.
+sub header_comment {
+ my $self = shift;
+ my $name = shift;
+
+ my $text = "# DO NOT EDIT - Generated for " . $name . " by " .
&name_version_string($self) . " on " . `date`;
+ return $text;
+}
+
+#retrieves parameter data for a specific server by searching by the
server's assigned profile.
+sub param_data {
--- End diff --
the only difference i see between `sub param_data` and `sub
profile_param_data` is the following piece:
if ( $value =~ /^STRING __HOSTNAME__$/ ) {
$value = "STRING " . $server_obj->host_name . "." .
$server_obj->domain_name;
}
is that bit of code only relevant when pulling a "take and bake" config
file at the server level?
any way you can just collapse the 2 methods?
> Improve efficiency of ATS config generation
> -------------------------------------------
>
> Key: TC-32
> URL: https://issues.apache.org/jira/browse/TC-32
> Project: Traffic Control
> Issue Type: Improvement
> Components: Traffic Ops API
> Reporter: Derek Gelinas
> Assignee: Derek Gelinas
> Fix For: 1.9.0
>
>
> Currently when generating ATS configuration files, each server calls for its
> individual files from traffic ops. This is a very database-intensive process
> that is not scalable enough. I propose the following changes:
> 1) Generate the configuration files only as many times as needed - many files
> are the same (or can be) for the entire CDN or server profile.
> 2) Once generated, the files can be cached locally. Requests for the files
> by ORT will result in pulling down these cached files rather than many
> hundreds of DB queries each time.
> 3) Migrate the routes for these files to the API. Each call will have
> multiple options - a request made with no options would return the cached
> file, another option would return the current DB data, and a third option
> would update the cached file with the current information in the DB.
> 4) When an update is queued, generate and cache the configuration files, then
> activate the update flag for the relevant servers once the cached file
> generation is completed.
> In this way, triggering updates for the caches will result in an initial
> increase in activity as traffic ops generates the files needed, followed by
> the much lower impact of the files themselves being requested by the caches.
> I believe we can cut down the number of server-specific configuration files
> to only 5 intially, and potentially fewer by having ORT fill in certain
> fields on files like records.config with local data during processing.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
