This is the s3upload.py file, might be easier than posing extracts
--
You received this message because you are subscribed to the Google Groups
"weewx-user" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
To view this discussion on the web visit
https://groups.google.com/d/msgid/weewx-user/80298daf-c899-40ae-930d-5bf8c362c781%40googlegroups.com.
#
# Copyright (c) 2009-2020 Mike Revitt <[email protected]>
#
# See the file LICENSE.txt for your full rights.
#
"""For uploading files to a remove server via S3"""
import logging
import os
import sys
import time
import boto3
from weewx.reportengine import ReportGenerator
from six.moves import cPickle
log = logging.getLogger(__name__)
# =============================================================================
# Class S3UploadGenerator
# =============================================================================
class S3UploadGenerator(ReportGenerator):
"""Class for managing the "AWS generator".
This will copy everything in the public_html subdirectory to a webserver."""
def run(self):
import user.s3upload
# determine how much logging is desired
log_success = to_bool(search_up(self.skin_dict, 'log_success', True))
t1 = time.time()
try:
S3_upload = user.s3upload.S3Upload(bucket=self.skin_dict['S3_BUCKET'],
profile=self.skin_dict['AWS_Profile'],
html_root=self.config_dict['StdReport']['HTML_ROOT'],
remote_root=self.skin_dict['S3_ROOT'],
name=self.skin_dict['REPORT_NAME'])
except KeyError:
syslog.syslog(syslog.LOG_DEBUG,
"S3UploadGenerator: AWS upload not requested. Skipped.")
return
try:
n = S3_upload.run()
except (socket.timeout, socket.gaierror, ftplib.all_errors, IOError) as e:
(cl, unused_ob, unused_tr) = sys.exc_info()
syslog.syslog(syslog.LOG_ERR, "S3UploadGenerator: "
"Caught exception %s: %s" % (cl, e))
weeutil.weeutil.log_traceback(" **** ")
return
if log_success:
t2 = time.time()
syslog.syslog(syslog.LOG_INFO,
"S3UploadGenerator: AWS-S3 S3'd %d files in %0.2f seconds" %
(n, (t2 - t1)))
# =============================================================================
# Class S3Upload
# =============================================================================
class S3Upload(object):
def __init__(self,
bucket,
profile,
html_root,
remote_root,
name = "S3",
max_tries = 3,
debug = 0,
secure_data = True):
self.bucket = bucket
self.profile = profile
self.html_root = html_root
self.remote_root = remote_root
self.name = name
self.max_tries = max_tries
self.debug = debug
self.secure_data = secure_data
def run(self):
# Get the timestamp and members of the last upload:
(timestamp, fileset) = self.getLastUpload()
n_uploaded = 0
# Try to connect to the S3 server up to max_tries times:
# Walk the local directory structure
for (dirpath, unused_dirnames, filenames) in os.walk(self.html_root):
local_rel_dir_path = dirpath.replace(self.html_root, '.')
if self._skipThisDir(local_rel_dir_path):
continue
# Now iterate over all members of the local directory:
for filename in filenames:
full_local_path = os.path.join(dirpath, filename)
# See if this file can be skipped:
if self._skipThisFile(timestamp, fileset, full_local_path):
continue
# Retry up to max_tries times:
for count in range(self.max_tries):
session = boto3.Session(profile_name=self.profile)
s3 = session.resource('s3')
try:
if local_rel_dir_path != '.':
full_remote_path = os.path.join( self.bucket + self.remote_root + local_rel_dir_path.replace('.', '' ))
else:
full_remote_path = os.path.join( self.bucket + self.remote_root )
print( 'filename is ', full_remote_path, filename)
# response = s3.meta.client.upload_file(full_local_path , full_remote_path, filename)
except ClientError as e:
# Unsuccessful. Log it and go around again.
log.error("Attempt #%d. Failed uploading %s to %s. Reason: %s"
% (count + 1, self.remote_root, self.server, e))
else:
# Success. Log it, break out of the loop
n_uploaded += 1
fileset.add(full_local_path)
log.debug("Uploaded file %s" % self.remote_root)
break
timestamp = time.time()
self.saveLastUpload(timestamp, fileset)
return n_uploaded
def getLastUpload(self):
"""Reads the time and members of the last upload from the local root"""
timeStampFile = os.path.join(self.html_root, "#%s.last" % self.name)
# If the file does not exist, an IOError exception will be raised.
# If the file exists, but is truncated, an EOFError will be raised.
# Either way, be prepared to catch it.
try:
with open(timeStampFile, "rb") as f:
timestamp = cPickle.load(f)
fileset = cPickle.load(f)
except (IOError, EOFError, cPickle.PickleError, AttributeError):
timestamp = 0
fileset = set()
# Either the file does not exist, or it is garbled.
# Either way, it's safe to remove it.
try:
os.remove(timeStampFile)
except OSError:
pass
return (timestamp, fileset)
def saveLastUpload(self, timestamp, fileset):
"""Saves the time and members of the last upload in the local root."""
timeStampFile = os.path.join(self.html_root, "#%s.last" % self.name)
with open(timeStampFile, "wb") as f:
cPickle.dump(timestamp, f)
cPickle.dump(fileset, f)
def _skipThisDir(self, local_dir):
return os.path.basename(local_dir) in ('.svn', 'CVS')
def _skipThisFile(self, timestamp, fileset, full_local_path):
filename = os.path.basename(full_local_path)
if filename[-1] == '~' or filename[0] == '#':
return True
if full_local_path not in fileset:
return False
if os.stat(full_local_path).st_mtime > timestamp:
return False
# Filename is in the set, and is up to date.
return True
# =============================================================================
# Main
# =============================================================================
if __name__ == '__main__':
import configobj
import weewx
import weeutil.logger
weewx.debug = 1
weeutil.logger.setup('S3upload', {})
if len(sys.argv) < 2:
print("""Usage: s3upload.py path-to-configuration-file [path-to-be-ftp'd]""")
sys.exit(weewx.CMD_ERROR)
try:
config_dict = configobj.ConfigObj(sys.argv[1], file_error=True, encoding='utf-8')
except IOError:
print("Unable to open configuration file %s" % sys.argv[1])
raise
S3_upload = S3Upload(config_dict['StdReport']['AWS-S3']['S3_BUCKET'],
config_dict['StdReport']['AWS-S3']['AWS_Profile'],
config_dict['StdReport']['HTML_ROOT'],
config_dict['StdReport']['AWS-S3']['S3_ROOT'],
'S3')
print(config_dict['StdReport']['AWS-S3']['S3_BUCKET'],
config_dict['StdReport']['AWS-S3']['AWS_Profile'],
config_dict['WEEWX_ROOT'],
config_dict['StdReport']['AWS-S3']['S3_ROOT'],
'S3')
S3_upload.run()
