I have attached the python script I call
And this is the new section in weewx.conf
[[S3-Backup]]
# Using S3-Backup to copy the database file to a webserver is
treated as just
# another report, albeit one with an unusual report generator!
skin = S3Backup
enable = true
# You must configure AWS at the command line and create some logon
credentials
# The logon credentials then go into the credentials file.
# This is the name of the profile defined in that file that you use
for the copy
AWS_Profile = mike
# This is the name of the region where the S3 bucket resides
AWS_Region = eu-west-2
# This is the name of the S3 bucket where the files will be copied
S3_BUCKET = s3-archive.revittmk.aws.co.uk
# This determins the frequency of the backup routine
report_timing = 10,30,50 * * * *
# @daily
On Friday, February 21, 2020 at 11:11:59 PM UTC, vince wrote:
>
> On Friday, February 21, 2020 at 2:52:39 PM UTC-8, Mike Revitt wrote:
>>
>> I have created a skin to backup my sqlite database file daily but it runs
>> every archive period. Any advice on what to change to make this work
>>
>>
>>
> Can't say without seeing your skin's python code, but I'd "guess" that
> it's not reading your report_timing values and acting accordingly.
>
> You also have "attempts" spelled incorrectly in your logging :-)
>
> Personally I'd just do this from cron outside weewx as a far simpler
> solution, but it would be interesting to see your code.
>
--
You received this message because you are subscribed to the Google Groups
"weewx-user" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
To view this discussion on the web visit
https://groups.google.com/d/msgid/weewx-user/513007f9-e955-49c5-bf39-384c1dab968e%40googlegroups.com.
#
# Copyright (c) 2009-2020 Mike Revitt <[email protected]>
#
# See the file LICENSE.txt for your full rights.
#
"""For backing up the database to S3"""
import logging
import os
import sys
import time
import mimetypes
import boto3
from weewx.reportengine import ReportGenerator
from weeutil.weeutil import to_bool
from six.moves import cPickle
from datetime import datetime
from botocore.exceptions import ClientError
from botocore.exceptions import ParamValidationError
from botocore.exceptions import ProfileNotFound
log = logging.getLogger(__name__)
# =============================================================================
# Class S3Backup
# =============================================================================
class S3Backup(object):
def __init__(self,
bucket,
profile,
region,
sqlite_root,
database_name,
max_tries = 3,
debug = 0,
secure_data = True):
self.bucket = bucket
self.profile = profile
self.region = region
self.sqlite_root = sqlite_root
self.database_name = database_name
self.max_tries = max_tries
self.debug = debug
self.secure_data = secure_data
def run(self):
# Add some logging information
log.debug("S3Backup started at: " + datetime.now().strftime('%d-%m-%Y %H:%M:%S') + "\n")
try:
session = boto3.Session(profile_name=self.profile, region_name=self.region)
s3 = session.resource('s3')
except ProfileNotFound as e:
log.error("Failed to connect to resource S3: using profile '%s'" %(self.profile))
exit(40)
source_database = os.path.join(self.sqlite_root, self.database_name)
dest_database = source_database.lstrip('/')
# Retry up to max_tries times:
for count in range(self.max_tries):
try:
log.debug("\nresponse=s3.meta.client.upload_file("+source_database+','+self.bucket+','+dest_database+')')
response = s3.meta.client.upload_file(source_database , self.bucket, dest_database,
ExtraArgs={'ContentType' : 'application/x-sqlite3'})
except IOError as e:
log.error("Attempt #%d. Failed uploading %s to %s/%s. \n\tReason: %s" %
(count + 1, source_database, self.bucket, dest_database, e))
except ClientError as e:
log.error("Attempt #%d. Failed uploading %s to %s/%s. \n\tReason: %s" %
(count + 1, source_database, self.bucket, dest_database, e))
break
except ParamValidationError as e:
log.error("Attempt #%d. Failed uploading %s to %s/%s. \n\tReason: %s" %
(count + 1, source_database, self.bucket, dest_database, e))
break
except () as e:
log.error("Attempt #%d. Failed uploading %s to %s/%s. \n\tUnknown error occured $s" %
(count + 1, source_database, self.bucket, dest_database, e ))
break
else:
# Success. Log it, break out of the loop
log.debug("Uploaded file %s" % source_database )
break
return count
# =============================================================================
# Class S3BackupGenerator
# =============================================================================
class S3BackupGenerator(ReportGenerator):
def run(self):
import user.s3backup
# determine how much logging is desired
log_success = to_bool(self.skin_dict.get('log_success', True))
t1 = time.time()
try:
S3_backup = user.s3backup.S3Backup( bucket = self.skin_dict['S3_BUCKET'],
profile = self.skin_dict['AWS_Profile'],
region = self.skin_dict['AWS_Region'],
sqlite_root = self.config_dict['DatabaseTypes']['SQLite']['SQLITE_ROOT'],
database_name = self.config_dict['Databases']['archive_sqlite']['database_name'])
except KeyError as e:
log.error("S3BackupGenerator: S3 Backup not requested. Skipped with error: %" % (e))
return
try:
n = S3_backup.run()
except () as e:
log.error("S3BackupGenerator: Caught exception: %s" % (e))
return
if log_success:
t2 = time.time()
log.info("S3BackupGenerator: S3 copied files to S3 in %d attemps which took in %0.2f seconds" % (n, (t2 - t1)))
# =============================================================================
# Main
# =============================================================================
if __name__ == '__main__':
import configobj
import weewx
import weeutil.logger
weewx.debug = 1
weeutil.logger.setup('S3Backup', {})
if len(sys.argv) < 2:
print("""Usage: S3Backup.py path-to-configuration-file [path-to-be-ftp'd]""")
sys.exit(weewx.CMD_ERROR)
try:
config_dict = configobj.ConfigObj(sys.argv[1], file_error=True, encoding='utf-8')
except IOError:
print("Unable to open configuration file %s" % sys.argv[1])
raise
S3_upload = S3Backup(config_dict['StdReport']['AWS-S3']['S3_BUCKET'],
config_dict['StdReport']['AWS-S3']['AWS_Profile'],
config_dict['StdReport']['AWS-S3']['AWS_Region'],
config_dict['DatabaseTypes']['SQLite']['SQLITE_ROOT'],
config_dict['Databases']['archive_sqlite']['database_name'])
print("\n========================================================================================\n")
print("\tS3Backup started at: " + datetime.now().strftime('%d-%m-%Y %H:%M:%S') + " with the following parameters\n")
print("\tBucket\t\t\tProfile\tRegion\tSQLITE Root\tDatabase Name")
print("\t",
config_dict['StdReport']['AWS-S3']['S3_BUCKET'],
config_dict['StdReport']['AWS-S3']['AWS_Profile'],
config_dict['StdReport']['AWS-S3']['AWS_Region'],
config_dict['DatabaseTypes']['SQLite']['SQLITE_ROOT'],
config_dict['Databases']['archive_sqlite']['database_name'])
print("\n========================================================================================\n")
S3_upload.run()
