Too bad it doesn't tell us what the directory name is. Try this version of weeutil/ftpload.py. It will log the directory name.
-tk On Sat, Apr 10, 2021 at 4:47 AM Chris Thompstone <[email protected]> wrote: > Hi, thought I'd better start a new topic. > Have just moved to ver 4.5.1 from 3.9.x > Have copied my standard skin over etc, database and so forth. > But I can't use the FTP upload to remote server. > I get the below and it just loops continuously, trying to upload the whole > of the www forever. So I have to turn it off, my broadband can't handle it. > It seems to error on the 1 file, if I delete it all, the error will still > happen, but moves to a different file. > Apr 10 12:33:04 weewx weewx[7372] DEBUG weeutil.ftpupload: 144 > ./NOAA/NOAA-2015-06.txt > 3c7a19998d74ef1457dd61e89a1eb848ff84c4ba5956c71b04227262f10507b2 > Apr 10 12:33:04 weewx weewx[7372] DEBUG weeutil.ftpupload: Uploaded file > /var/www/NOAA/NOAA-2015-06.txt to /NOAA/NOAA-2015-06.txt > Apr 10 12:33:04 weewx weewx[7372] DEBUG weeutil.ftpupload: 145 > ./NOAA/NOAA-2015-05.txt > f261402ab917dc68cccbe73afb6e88213369a894a30613cc82505ea005923ca7 > Apr 10 12:33:04 weewx weewx[7372] DEBUG weeutil.ftpupload: Uploaded file > /var/www/NOAA/NOAA-2015-05.txt to /NOAA/NOAA-2015-05.txt > Apr 10 12:33:04 weewx weewx[7372] DEBUG weeutil.ftpupload: 146 > ./NOAA/NOAA-2015-04.txt > e5dca630c7f48cbd72ad396b4e869a3732497db4580654f6617aa091b443161b > Apr 10 12:33:04 weewx weewx[7372] DEBUG weeutil.ftpupload: Uploaded file > /var/www/NOAA/NOAA-2015-04.txt to /NOAA/NOAA-2015-04.txt > Apr 10 12:33:04 weewx weewx[7372] DEBUG weeutil.ftpupload: 147 > ./NOAA/NOAA-2015-03.txt > 045e6defcc621acc823b67678bab43888cb032d6240d08c8a5db486582c9d19a > Apr 10 12:33:05 weewx weewx[7372] DEBUG weeutil.ftpupload: Uploaded file > /var/www/NOAA/NOAA-2015-03.txt to /NOAA/NOAA-2015-03.txt > Apr 10 12:33:05 weewx weewx[7372] DEBUG weeutil.ftpupload: 148 > ./NOAA/NOAA-2015-02.txt > fe588dc95bf05b3620b7cd40fab59852ad83cf8a2f53c4b232dca8109f6f9c84 > Apr 10 12:33:05 weewx weewx[7372] DEBUG weeutil.ftpupload: Uploaded file > /var/www/NOAA/NOAA-2015-02.txt to /NOAA/NOAA-2015-02.txt > Apr 10 12:33:05 weewx weewx[7372] DEBUG weeutil.ftpupload: 149 > ./NOAA/NOAA-2015-01.txt > 05c6e945b3d3980ac4667a9e2f84b3c1d714de8940bfa4f278b4c77837faa6f9 > Apr 10 12:33:05 weewx weewx[7372] DEBUG weeutil.ftpupload: Uploaded file > /var/www/NOAA/NOAA-2015-01.txt to /NOAA/NOAA-2015-01.txt > Apr 10 12:33:05 weewx weewx[7372] DEBUG weeutil.ftpupload: 150 > ./NOAA/NOAA-2014-12.txt > 1e4d75393d91b17aa8b7d168425685557833940caab67028c4dd2958aecdf5c7 > Apr 10 12:33:06 weewx weewx[7372] DEBUG weeutil.ftpupload: Uploaded file > /var/www/NOAA/NOAA-2014-12.txt to /NOAA/NOAA-2014-12.txt > Apr 10 12:33:06 weewx weewx[7372] ERROR weewx.reportengine: ftpgenerator: > (2): caught exception '<class 'ftplib.error_perm'>': 553 Prohibited > directory name > Apr 10 12:33:06 weewx weewx[7372] ERROR weewx.reportengine: **** > Traceback (most recent call last): > Apr 10 12:33:06 weewx weewx[7372] ERROR weewx.reportengine: **** > File "/home/weewx/bin/weewx/reportengine.py", line 331, in run > Apr 10 12:33:06 weewx weewx[7372] ERROR weewx.reportengine: **** > n = ftp_data.run() > Apr 10 12:33:06 weewx weewx[7372] ERROR weewx.reportengine: **** > File "/home/weewx/bin/weeutil/ftpupload.py", line 154, in run > Apr 10 12:33:06 weewx weewx[7372] ERROR weewx.reportengine: **** > _make_remote_dir(ftp_server, remote_dir_path) > Apr 10 12:33:06 weewx weewx[7372] ERROR weewx.reportengine: **** > File "/home/weewx/bin/weeutil/ftpupload.py", line 269, in _make_remote_dir > Apr 10 12:33:06 weewx weewx[7372] ERROR weewx.reportengine: **** > ftp_server.mkd(remote_dir_path) > Apr 10 12:33:06 weewx weewx[7372] ERROR weewx.reportengine: **** > File "/usr/lib/python3.7/ftplib.py", line 643, in mkd > Apr 10 12:33:06 weewx weewx[7372] ERROR weewx.reportengine: **** > resp = self.voidcmd('MKD ' + dirname) > Apr 10 12:33:06 weewx weewx[7372] ERROR weewx.reportengine: **** > File "/usr/lib/python3.7/ftplib.py", line 278, in voidcmd > Apr 10 12:33:06 weewx weewx[7372] ERROR weewx.reportengine: **** > return self.voidresp() > Apr 10 12:33:06 weewx weewx[7372] ERROR weewx.reportengine: **** > File "/usr/lib/python3.7/ftplib.py", line 251, in voidresp > Apr 10 12:33:06 weewx weewx[7372] ERROR weewx.reportengine: **** > resp = self.getresp() > Apr 10 12:33:06 weewx weewx[7372] ERROR weewx.reportengine: **** > File "/usr/lib/python3.7/ftplib.py", line 246, in getresp > Apr 10 12:33:06 weewx weewx[7372] ERROR weewx.reportengine: **** > raise error_perm(resp) > Apr 10 12:33:06 weewx weewx[7372] ERROR weewx.reportengine: **** > ftplib.error_perm: 553 Prohibited directory name > Apr 10 12:33:06 weewx weewx[7372] ERROR weewx.reportengine: ftpgenerator: > Upload failed > > -- > You received this message because you are subscribed to the Google Groups > "weewx-user" group. > To unsubscribe from this group and stop receiving emails from it, send an > email to [email protected]. > To view this discussion on the web visit > https://groups.google.com/d/msgid/weewx-user/0203f602-132d-4d03-80de-c1b1b6c8e396n%40googlegroups.com > <https://groups.google.com/d/msgid/weewx-user/0203f602-132d-4d03-80de-c1b1b6c8e396n%40googlegroups.com?utm_medium=email&utm_source=footer> > . > -- You received this message because you are subscribed to the Google Groups "weewx-user" group. To unsubscribe from this group and stop receiving emails from it, send an email to [email protected]. To view this discussion on the web visit https://groups.google.com/d/msgid/weewx-user/CAPq0zEDi2ET%2BHjpai1N7uVaaUd57LsV1wDcO1iBMFe8oKpOt1g%40mail.gmail.com.
# # Copyright (c) 2009-2020 Tom Keffer <[email protected]> # # See the file LICENSE.txt for your full rights. # """For uploading files to a remove server via FTP""" from __future__ import absolute_import from __future__ import print_function from __future__ import with_statement import ftplib import logging import os import sys import time from six.moves import cPickle try: import hashlib has_hashlib=True except ImportError: has_hashlib=False log = logging.getLogger(__name__) class FtpUpload(object): """Uploads a directory and all its descendants to a remote server. Keeps track of when a file was last uploaded, so it is uploaded only if its modification time is newer.""" def __init__(self, server, user, password, local_root, remote_root, port=21, name="FTP", passive=True, secure=False, debug=0, secure_data=True, reuse_ssl=False): """Initialize an instance of FtpUpload. After initializing, call method run() to perform the upload. server: The remote server to which the files are to be uploaded. user, password : The user name and password that are to be used. name: A unique name to be given for this FTP session. This allows more than one session to be uploading from the same local directory. [Optional. Default is 'FTP'.] passive: True to use passive mode; False to use active mode. [Optional. Default is True (passive mode)] secure: Set to True to attempt an FTP over TLS (FTPS) session. debug: Set to 1 for extra debug information, 0 otherwise. secure_data: If a secure session is requested (option secure=True), should we attempt a secure data connection as well? This option is useful due to a bug in the Python FTP client library. See Issue #284. [Optional. Default is True] reuse_ssl: Work around a bug in the Python library that closes ssl sockets that should be reused. See https://bit.ly/3dKq4JY [Optional. Default is False] """ self.server = server self.user = user self.password = password self.local_root = os.path.normpath(local_root) self.remote_root = os.path.normpath(remote_root) self.port = port self.name = name self.passive = passive self.secure = secure self.debug = debug self.secure_data = secure_data self.reuse_ssl = reuse_ssl if self.reuse_ssl and sys.version < '3.6': raise ValueError("Reusing an SSL connection requires Python version 3.6 or greater") def run(self): """Perform the actual upload. returns: the number of files uploaded.""" # Get the timestamp and members of the last upload: timestamp, fileset, hashdict = self.get_last_upload() n_uploaded = 0 try: if self.secure: log.debug("Attempting secure connection to %s", self.server) if self.reuse_ssl: # Activate the workaround for the Python ftplib library. from ssl import SSLSocket class ReusedSslSocket(SSLSocket): def unwrap(self): pass class WeeFTPTLS(ftplib.FTP_TLS): """Explicit FTPS, with shared TLS session""" def ntransfercmd(self, cmd, rest=None): conn, size = ftplib.FTP.ntransfercmd(self, cmd, rest) if self._prot_p: conn = self.context.wrap_socket(conn, server_hostname=self.host, session=self.sock.session) conn.__class__ = ReusedSslSocket return conn, size log.debug("Reusing SSL connections.") ftp_server = WeeFTPTLS() else: ftp_server = ftplib.FTP_TLS() else: log.debug("Attempting connection to %s", self.server) ftp_server = ftplib.FTP() if self.debug >= 2: ftp_server.set_debuglevel(self.debug) ftp_server.set_pasv(self.passive) ftp_server.connect(self.server, self.port) ftp_server.login(self.user, self.password) if self.secure and self.secure_data: ftp_server.prot_p() log.debug("Secure data connection to %s", self.server) else: log.debug("Connected to %s", self.server) # Walk the local directory structure for (dirpath, unused_dirnames, filenames) in os.walk(self.local_root): # Strip out the common local root directory. What is left # will be the relative directory both locally and remotely. local_rel_dir_path = dirpath.replace(self.local_root, '.') if _skip_this_dir(local_rel_dir_path): continue # This is the absolute path to the remote directory: remote_dir_path = os.path.normpath(os.path.join(self.remote_root, local_rel_dir_path)) # Make the remote directory if necessary: _make_remote_dir(ftp_server, remote_dir_path) # Now iterate over all members of the local directory: for filename in filenames: full_local_path = os.path.join(dirpath, filename) # calculate hash if has_hashlib: filehash=sha256sum(full_local_path) else: filehash=None # See if this file can be skipped: if _skip_this_file(timestamp, fileset, hashdict, full_local_path, filehash): continue full_remote_path = os.path.join(remote_dir_path, filename) stor_cmd = "STOR %s" % full_remote_path log.debug("%s %s/%s %s" % (n_uploaded,local_rel_dir_path,filename,filehash)) with open(full_local_path, 'rb') as fd: try: ftp_server.storbinary(stor_cmd, fd) except ftplib.all_errors as e: # Unsuccessful. Log it, then reraise the exception log.error("Failed uploading %s to server %s. Reason: '%s'", full_local_path, self.server, e) raise # Success. n_uploaded += 1 fileset.add(full_local_path) hashdict[full_local_path]=filehash log.debug("Uploaded file %s to %s", full_local_path, full_remote_path) finally: try: ftp_server.quit() except Exception: pass timestamp = time.time() self.save_last_upload(timestamp, fileset, hashdict) return n_uploaded def get_last_upload(self): """Reads the time and members of the last upload from the local root""" timestamp_file_path = os.path.join(self.local_root, "#%s.last" % self.name) # If the file does not exist, an IOError exception will be raised. # If the file exists, but is truncated, an EOFError will be raised. # Either way, be prepared to catch it. try: with open(timestamp_file_path, "rb") as f: timestamp = cPickle.load(f) fileset = cPickle.load(f) hashdict = cPickle.load(f) except (IOError, EOFError, cPickle.PickleError, AttributeError): timestamp = 0 fileset = set() hashdict = {} # Either the file does not exist, or it is garbled. # Either way, it's safe to remove it. try: os.remove(timestamp_file_path) except OSError: pass return timestamp, fileset, hashdict def save_last_upload(self, timestamp, fileset, hashdict): """Saves the time and members of the last upload in the local root.""" timestamp_file_path = os.path.join(self.local_root, "#%s.last" % self.name) with open(timestamp_file_path, "wb") as f: cPickle.dump(timestamp, f) cPickle.dump(fileset, f) cPickle.dump(hashdict, f) def _skip_this_file(timestamp, fileset, hashdict, full_local_path, filehash): """Determine whether to skip a specific file.""" filename = os.path.basename(full_local_path) if filename[-1] == '~' or filename[0] == '#': return True if full_local_path not in fileset: return False if has_hashlib and filehash is not None: # use hash if available if full_local_path not in hashdict: return False if hashdict[full_local_path]!=filehash: return False else: # otherwise use file time if os.stat(full_local_path).st_mtime > timestamp: return False # Filename is in the set, and is up to date. return True def _skip_this_dir(local_dir): """Determine whether to skip a directory.""" return os.path.basename(local_dir) in ('.svn', 'CVS') def _make_remote_dir(ftp_server, remote_dir_path): """Make a remote directory if necessary.""" try: ftp_server.mkd(remote_dir_path) except ftplib.all_errors as e: # Got an exception. It might be because the remote directory already exists: if sys.exc_info()[0] is ftplib.error_perm: msg = str(e).strip() # If a directory already exists, some servers respond with a '550' ("Requested # action not taken") code, others with a '521' ("Access denied" or "Pathname # already exists") code. if msg.startswith('550') or msg.startswith('521'): # Directory already exists return # It's a real error. Log it, then re-raise the exception. log.error("Error creating directory %s", remote_dir_path) raise log.debug("Made directory %s", remote_dir_path) # from https://stackoverflow.com/questions/22058048/hashing-a-file-in-python def sha256sum(filename): h = hashlib.sha256() b = bytearray(128*1024) mv = memoryview(b) with open(filename, 'rb', buffering=0) as f: for n in iter(lambda : f.readinto(mv), 0): h.update(mv[:n]) return h.hexdigest()
