config.ini.example

@@ -2,6 +2,7 @@
 queue = /some/dir
 storage = /another/dir
 processing = /a/third/dir
+error = /an/error_dir
 notify_url = https://example.com/api/notify
 notify_token = <jwt token>
 

daemon/__init__.py

@@ -11,6 +11,7 @@ from pipeline import Pipeline
 class Daemon:
     def __init__(self, config):
         self.queue_dir = config['daemon']['queue']
+        self.error_dir = config['daemon']['error']
         self.logger = logging.getLogger('play-daemon')
         self.logger.setLevel('DEBUG')
         if 'mail_level' in config['daemon']:
@@ -51,6 +52,10 @@ class Daemon:
             except Exception as e:
                 msg = "Exception during processing of queue item %s: %s"
                 self.logger.exception(msg, pres_id, queue_item, exc_info=e)
+                # write the que info and error to disk for play-api to display the status
+                with open(os.path.join(self.error_dir, pres_id), 'w') as f:
+                    queue_item['error'] = e
+                    json.dump(queue_item, f)
 
 
 class QueueUpdater(FileSystemEventHandler):

daemon/pipeline.py

@@ -14,6 +14,7 @@ class Pipeline:
         self.logger     = logging.getLogger('play-daemon')
         self.queue      = config['daemon']['queue']
         self.storage    = config['daemon']['storage']
+        self.error      = config['daemon']['error']
         self.processing = config['daemon']['processing']
         self.notify_url = config['daemon']['notify_url']
         token           = config['daemon']['notify_token']
@@ -168,6 +169,11 @@ class Pipeline:
 
         # Delete queue file and upload data
         os.remove(os.path.join(self.queue, target_id))
+        try:
+            # if any old error file remove it since it worked
+            os.remove(os.path.join(self.error, target_id))
+        except OSError:
+            pass
         shutil.rmtree(package['base'])
         return