Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Settings now read from .ini instead of Python, bug found and noted. #2

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Settings now read from .ini instead of Python, bug found and noted. #2

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e4706ab
Settings now read from .ini instead of Python. Notes added regarding…
Feb 16, 2015
52e17b6
Instruction error in README. Updated to reflect that user needs to e…
Feb 16, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
88 changes: 48 additions & 40 deletions FTP_Upload/src/ftp_upload.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@
# Copyright (C) 2013-2014 Neighborhood Guard, Inc. All rights reserved.
# Original author: Jesper Jurcenoks
# Maintained by the Neighborhood Guard development team
#
# Modifications: Louis T. Getterman IV (@LTGIV) / GotGetLLC.com / opensour.cc
#
# This file is part of FTP_Upload.
#
# FTP_Upload is free software: you can redistribute it and/or modify
Expand All @@ -21,6 +22,8 @@
#
################################################################################



######################################################################################
# #
# Program to upload images as part of the Neighborhood Guard software suite. #
Expand Down Expand Up @@ -48,14 +51,19 @@
import subprocess
import string

# Local library part of ftp_upload
import localsettings
# Parse settings
from configobj import ConfigObj
localsettings = ConfigObj( 'localsettings.ini' ) # pip install configobj
localsettings['incoming_location'] = localsettings['incoming_location'].replace( '%BASE_LOCATION%', localsettings['base_location'] )
localsettings['processed_location'] = localsettings['processed_location'].replace( '%BASE_LOCATION%', localsettings['base_location'] )
localsettings['sleep_upload'] = float( localsettings['sleep_upload'] )
localsettings['retain_days'] = int( localsettings['retain_days'] )

# 3rd Party libraries not part of default Python and needs to be installed
if localsettings.use_sftp==True:
if localsettings['use_sftp']==True:
import pysftp # pip install pysftp

version_string = "1.7.0"
version_string = "1.7.1"

current_priority_threads=0 # global variable shared between threads keeping track of running priority threads.

Expand All @@ -79,7 +87,7 @@ def change_create_server_dir(server_connection, dirname):
# dirname is relative or absolute

if server_connection != None:
if localsettings.use_sftp == True:
if localsettings['use_sftp'] == True:
try:
server_connection.cwd(dirname)
except IOError, e :
Expand Down Expand Up @@ -139,14 +147,14 @@ def get_daydirs(location):
def connect_to_server():
logging.debug("FTP_UPLOAD:Starting connect_to_server()")
server_connection = None
if localsettings.use_sftp==True:
if localsettings['use_sftp']==True:
# SFTP Version
try:
server_connection = pysftp.Connection(localsettings.ftp_server, username=localsettings.ftp_username,password=localsettings.ftp_password)
logging.debug("FTP_UPLOAD: Connected to %s", localsettings.ftp_server)
server_connection = pysftp.Connection( localsettings['ftp_server'], username=localsettings['ftp_username'], password=localsettings['ftp_password'] )
logging.debug("FTP_UPLOAD: Connected to %s", localsettings['ftp_server'])
logging.debug("FTP_UPLOAD:current directory is: %s", server_connection.pwd)
logging.debug("FTP_UPLOAD:changing directory to: %s", localsettings.ftp_destination)
server_connection.cwd(localsettings.ftp_destination)
logging.debug("FTP_UPLOAD:changing directory to: %s", localsettings['ftp_destination'])
server_connection.cwd(localsettings['ftp_destination'])
logging.debug("FTP_UPLOAD:current directory is: %s", server_connection.pwd)
except SSHExection, e:
if e=="Error reading SSH protocol banner" :
Expand All @@ -168,18 +176,18 @@ def connect_to_server():
else:
# FTP Version
try:
server_connection = ftplib.FTP(localsettings.ftp_server,localsettings.ftp_username,localsettings.ftp_password,timeout=30)
server_connection = ftplib.FTP(localsettings['ftp_server'],localsettings['ftp_username'],localsettings['ftp_password'],timeout=30)
logging.debug(server_connection.getwelcome())
logging.debug("FTP_UPLOAD:current directory is: %s", server_connection.pwd())
logging.debug("FTP_UPLOAD:changing directory to: %s", localsettings.ftp_destination)
server_connection.cwd(localsettings.ftp_destination)
logging.debug("FTP_UPLOAD:changing directory to: %s", localsettings['ftp_destination'])
server_connection.cwd(localsettings['ftp_destination'])
logging.debug("FTP_UPLOAD:current directory is: %s", server_connection.pwd())
except ftplib.error_perm, e:
logging.error("FTP_UPLOAD:Failed to open FTP connection, %s", e)
server_connection = None
message = "Sleeping " + str(localsettings.sleep_err_seconds/60) + " minutes before trying again"
message = "Sleeping " + str(localsettings['sleep_err_seconds']/60) + " minutes before trying again"
logging.info(message)
time.sleep(localsettings.sleep_err_seconds)
time.sleep(localsettings['sleep_err_seconds'])
except Exception, e:
logging.error("FTP_UPLOAD:Unexpected exception in connect_to_server():")
logging.exception(e)
Expand All @@ -193,7 +201,7 @@ def connect_to_server():
def quit_server(server_connection):
logging.debug("FTP_UPLOAD:Starting quit_server()")
if server_connection != None :
if localsettings.use_sftp==True:
if localsettings['use_sftp']==True:
# SFTP Version
try:
server_connection.close()
Expand Down Expand Up @@ -221,7 +229,7 @@ def putfile(server_connection, ftp_dir, filepath, filename):
change_create_server_dir(server_connection, ftp_dir)
logging.info("FTP_UPLOAD:Uploading %s", filepath)
try:
if localsettings.use_sftp==True:
if localsettings['use_sftp']==True:
server_connection.put(filepath, remotepath=ftp_dir+"/"+filename, preserve_mtime=True)
else:
filehandle = open(filepath, "rb")
Expand Down Expand Up @@ -276,16 +284,16 @@ def storefile(ftp_dir, filepath, donepath, filename, today):
logging.exception(e)

else:
message = "Sleeping " + str(localsettings.sleep_err_seconds/60) + " minutes before trying again"
message = "Sleeping " + str(localsettings['sleep_err_seconds']/60) + " minutes before trying again"
logging.info(message)
time.sleep(localsettings.sleep_err_seconds)
time.sleep(localsettings['sleep_err_seconds'])

quit_server(server_connection)

else :
message = "Sleeping " + str(localsettings.sleep_err_seconds/60) + " minutes before trying again - general error"
message = "Sleeping " + str(localsettings['sleep_err_seconds']/60) + " minutes before trying again - general error"
logging.info(message)
time.sleep(localsettings.sleep_err_seconds)
time.sleep(localsettings['sleep_err_seconds'])
# end if

if today :
Expand Down Expand Up @@ -318,7 +326,7 @@ def storedir(dirpath, ftp_dir, done_dir, today):
current_threads = threading.active_count()
logging.info("FTP_UPLOAD:current threads: %s", current_threads)

if (current_threads >= localsettings.max_threads) or (not today and current_priority_threads>=localsettings.reserved_priority_threads):
if (current_threads >= localsettings['max_threads']) or (not today and current_priority_threads>=localsettings['reserved_priority_threads']):
# to many threads running already, upload ftp in current thread (don't move forward until upload is done)
storefile(ftp_dir, filepath, donepath, filename, today)
current_threads = threading.active_count()
Expand Down Expand Up @@ -355,7 +363,7 @@ def deltree(deldir):
rmdir(filepath)
else:
logging.info("FTP_UPLOAD:deleting %s", filepath)
if localsettings.delete == False :
if localsettings['delete'] == False :
logging.info("FTP_UPLOAD:would have deleted %s here - to really delete change delete flag to True", filepath)
else :
os.remove(filepath)
Expand All @@ -368,9 +376,9 @@ def purge_old_images(purge_dir):
global files_purged
# Purge directories in Purge_dir, does not delete purge_dir itself
purge_daydirs=get_daydirs(purge_dir)
logging.debug("FTP_UPLOAD:list of directories to be purged: %s", purge_daydirs[0:-localsettings.retain_days])
logging.debug("FTP_UPLOAD:list of directories to be purged: %s", purge_daydirs[ 0:-localsettings['retain_days'] ])
files_purged = False
for purge_daydir in purge_daydirs[0:-localsettings.retain_days]:
for purge_daydir in purge_daydirs[0:-localsettings['retain_days']]:
(dirpath, unused_direc) = purge_daydir
logging.info("FTP_UPLOAD:purging directory %s", dirpath)
deltree(dirpath)
Expand All @@ -391,8 +399,8 @@ def storeday(daydir, today=False):
try:
(dirpath, direc) = daydir
logging.info("FTP_UPLOAD:processing directory %s", direc)
ftp_dir = localsettings.ftp_destination + "/" + direc
done_dir = os.path.join(localsettings.processed_location, direc)
ftp_dir = localsettings['ftp_destination'] + "/" + direc
done_dir = os.path.join(localsettings['processed_location'], direc)
storedir(dirpath, ftp_dir, done_dir, today)
except Exception, e:
logging.exception(e)
Expand Down Expand Up @@ -436,8 +444,8 @@ def set_up_logging():
# set up the rotating log file handler
#
logfile = logging.handlers.TimedRotatingFileHandler('ftp_upload.log',
when='midnight', backupCount=localsettings.logfile_max_days)
logfile.setLevel(localsettings.logfile_log_level)
when='midnight', backupCount=localsettings['logfile_max_days'])
logfile.setLevel( localsettings['logfile_log_level'] )
logfile.setFormatter(logging.Formatter(
'%(asctime)s %(levelname)-8s %(threadName)-10s %(message)s',
'%m-%d %H:%M:%S'))
Expand All @@ -446,7 +454,7 @@ def set_up_logging():
# define a Handler which writes messages equal to or greater than
# console_log_level to the sys.stderr
console = logging.StreamHandler()
console.setLevel(localsettings.console_log_level)
console.setLevel( localsettings['console_log_level'] )
# set a format which is simpler for console use
formatter = logging.Formatter('%(levelname)-8s %(message)s')
# tell the handler to use this format
Expand Down Expand Up @@ -485,7 +493,7 @@ def continous_upload():
signal.signal(signal.SIGINT, sighandler) # dump thread stacks on Ctl-C
logging.info("FTP_UPLOAD:Program Started, version %s", version_string)
try:
mkdir(localsettings.processed_location)
mkdir(localsettings['processed_location'])
# Setup the threads, don't actually run them yet used to test if the threads are alive.
processtoday_thread = threading.Thread(target=storeday, args=())
process_previous_days_thread = threading.Thread(target=storedays, args=())
Expand All @@ -495,7 +503,7 @@ def continous_upload():

while True:

daydirs = get_daydirs(localsettings.incoming_location)
daydirs = get_daydirs(localsettings['incoming_location'])

#reverse sort the days so that today is first
daydirs = sorted(daydirs, reverse=True)
Expand All @@ -522,14 +530,14 @@ def continous_upload():


if not purge_thread.is_alive():
purge_thread = threading.Thread(target=purge_old_images, args=(localsettings.processed_location,))
purge_thread = threading.Thread(target=purge_old_images, args=(localsettings['processed_location'],))
purge_thread.start()


logging.info("FTP_UPLOAD:Sleeping 1 minute for upload")
logging.info("FTP_UPLOAD:Time is %s", time.ctime() )
logging.info("FTP_UPLOAD:Time is %s", time.ctime() )
try:
time.sleep(localsettings.sleep_upload) # sleep
time.sleep( localsettings['sleep_upload'] ) # sleep

# hitting Ctl-C to dump the thread stacks will interrupt
# MainThread's sleep and raise IOError, so catch it here
Expand Down Expand Up @@ -638,10 +646,10 @@ def status():
cameraconnection=ping("camera") #put a line in the hostfile to point camera at the camera's ip address
wificonnection=ping(get_gateway_ip())
internetconnection=ping("www.google.com")
dreamhostconnection=ping(localsettings.ftp_server)
dreamhostconnection=ping(localsettings['ftp_server'])

daycount=len(os.listdir(localsettings.incoming_location))
imagecount=sum([len(files) for r, d, files in os.walk(localsettings.incoming_location)])
daycount=len(os.listdir(localsettings['incoming_location']))
imagecount=sum([len(files) for r, d, files in os.walk(localsettings['incoming_location'])])

freedisk=get_free_disk()

Expand All @@ -662,7 +670,7 @@ def status():
statusfile.write(statusstr)
statusfile.close()

ftp_dir = localsettings.ftp_destination + "/status/" + hostname
ftp_dir = localsettings['ftp_destination'] + "/status/" + hostname

server_connection = connect_to_server()
if server_connection != None:
Expand Down
49 changes: 49 additions & 0 deletions FTP_Upload/src/localsettings.ini
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
# (S)FTP SETTINGS - always change these
ftp_server = "ftp.ng_demo.org"
ftp_username = "ng_demo_user"
ftp_password = "ng_demo_password"

# remember to start with /
# for FTP the root is inside the user dir, for SFTP the root is master root
# e.g. FTP ftp_destination = "/video.yourneighborhood.org"
# e.g. SFTP ftp_destination = "/home/myuser/video.yourneighborhood.org"
ftp_destination = "/video.yourneighborhood.org"

# Specific locations are created below this directory
base_location = "/home/pi"

# The sample values are the defaults for RaspGuard
incoming_location = "%BASE_LOCATION%/images.incoming"

# Make sure this directory is NOT below the incoming_location as you will be creating an endlees upload loop
processed_location = "%BASE_LOCATION%/images.uploaded"

# Time to sleep when error (Default = 600)
sleep_err_seconds = 600

# Time to sleep for new pictures (Default = 60) (Useful to change during testing)
sleep_upload = 60

# Must be True for Purge to work
delete = True

# Use secure ftp? (recommended setting: True)
use_sftp = True

# Number of days to retain local images (on the Raspberry Pi). (Data retention on the destination Cloud server is set somewhere else)
retain_days = 2

# Logging level for console output
console_log_level = "INFO"

# Logging level for output to log file(s)
logfile_log_level = "INFO"

# Max number of previous log files to save, one log file per day
logfile_max_days = 10

# Max number of total threads when needed one thread will be used for purging job, rest of time all threads will be used for upload.
max_threads = 2

# Previous days can only upload multithreaded when running today threads fall below this number.
reserved_priority_threads = 1
77 changes: 0 additions & 77 deletions FTP_Upload/src/localsettings.py
View file
Open in desktop

This file was deleted.

Loading