""" Copyright (c) 2008, appengine-utilities project All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - Neither the name of the appengine-utilities project nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """ import os import cgi import re import datetime import pickle from google.appengine.ext import db from google.appengine.api import urlfetch from google.appengine.api import memcache APPLICATION_PORT = '8080' CRON_PORT = '8081' class _AppEngineUtilities_Cron(db.Model): """ Model for the tasks in the datastore. This contains the scheduling and url information, as well as a field that sets the next time the instance should run. """ cron_entry = db.StringProperty() next_run = db.DateTimeProperty() cron_compiled = db.BlobProperty() url = db.LinkProperty() class Cron(object): """ Cron is a scheduling utility built for appengine, modeled after crontab for unix systems. While true scheduled tasks are not possible within the Appengine environment currently, this is an attmempt to provide a request based alternate. You configure the tasks in an included interface, and the import the class on any request you want capable of running tasks. On each request where Cron is imported, the list of tasks that need to be run will be pulled and run. A task is a url within your application. It's important to make sure that these requests fun quickly, or you could risk timing out the actual request. See the documentation for more information on configuring your application to support Cron and setting up tasks. """ def __init__(self): # Check if any tasks need to be run query = _AppEngineUtilities_Cron.all() query.filter('next_run <= ', datetime.datetime.now()) results = query.fetch(1000) if len(results) > 0: one_second = datetime.timedelta(seconds = 1) before = datetime.datetime.now() for r in results: if re.search(':' + APPLICATION_PORT, r.url): r.url = re.sub(':' + APPLICATION_PORT, ':' + CRON_PORT, r.url) #result = urlfetch.fetch(r.url) diff = datetime.datetime.now() - before if int(diff.seconds) < 1: if memcache.add(str(r.key), "running"): result = urlfetch.fetch(r.url) r.next_run = self._get_next_run(pickle.loads(r.cron_compiled)) r.put() memcache.delete(str(r.key)) else: break def add_cron(self, cron_string): cron = cron_string.split(" ") if len(cron) is not 6: raise ValueError, 'Invalid cron string. Format: * * * * * url' cron = { 'min': cron[0], 'hour': cron[1], 'day': cron[2], 'mon': cron[3], 'dow': cron[4], 'url': cron[5], } cron_compiled = self._validate_cron(cron) next_run = self._get_next_run(cron_compiled) cron_entry = _AppEngineUtilities_Cron() cron_entry.cron_entry = cron_string cron_entry.next_run = next_run cron_entry.cron_compiled = pickle.dumps(cron_compiled) cron_entry.url = cron["url"] cron_entry.put() def _validate_cron(self, cron): """ Parse the field to determine whether it is an integer or lists, also converting strings to integers where necessary. If passed bad values, raises a ValueError. """ parsers = { 'dow': self._validate_dow, 'mon': self._validate_mon, 'day': self._validate_day, 'hour': self._validate_hour, 'min': self._validate_min, 'url': self. _validate_url, } for el in cron: parse = parsers[el] cron[el] = parse(cron[el]) return cron def _validate_type(self, v, t): """ Validates that the number (v) passed is in the correct range for the type (t). Raise ValueError, if validation fails. Valid ranges: day of week = 0-7 month = 1-12 day = 1-31 hour = 0-23 minute = 0-59 All can * which will then return the range for that entire type. """ if t == "dow": if v >= 0 and v <= 7: return [v] elif v == "*": return "*" else: raise ValueError, "Invalid day of week." elif t == "mon": if v >= 1 and v <= 12: return [v] elif v == "*": return range(1, 12) else: raise ValueError, "Invalid month." elif t == "day": if v >= 1 and v <= 31: return [v] elif v == "*": return range(1, 31) else: raise ValueError, "Invalid day." elif t == "hour": if v >= 0 and v <= 23: return [v] elif v == "*": return range(0, 23) else: raise ValueError, "Invalid hour." elif t == "min": if v >= 0 and v <= 59: return [v] elif v == "*": return range(0, 59) else: raise ValueError, "Invalid minute." def _validate_list(self, l, t): """ Validates a crontab list. Lists are numerical values seperated by a comma with no spaces. Ex: 0,5,10,15 Arguments: l: comma seperated list of numbers t: type used for validation, valid values are dow, mon, day, hour, min """ elements = l.split(",") return_list = [] # we have a list, validate all of them for e in elements: if "-" in e: return_list.extend(self._validate_range(e, t)) else: try: v = int(e) self._validate_type(v, t) return_list.append(v) except: raise ValueError, "Names are not allowed in lists." # return a list of integers return return_list def _validate_range(self, r, t): """ Validates a crontab range. Ranges are 2 numerical values seperated by a dash with no spaces. Ex: 0-10 Arguments: r: dash seperated list of 2 numbers t: type used for validation, valid values are dow, mon, day, hour, min """ elements = r.split('-') # a range should be 2 elements if len(elements) is not 2: raise ValueError, "Invalid range passed: " + str(r) # validate the minimum and maximum are valid for the type for e in elements: self._validate_type(int(e), t) # return a list of the numbers in the range. # +1 makes sure the end point is included in the return value return range(int(elements[0]), int(elements[1]) + 1) def _validate_step(self, s, t): """ Validates a crontab step. Steps are complicated. They can be based on a range 1-10/2 or just step through all valid */2. When parsing times you should always check for step first and see if it has a range or not, before checking for ranges because this will handle steps of ranges returning the final list. Steps of lists is not supported. Arguments: s: slash seperated string t: type used for validation, valid values are dow, mon, day, hour, min """ elements = s.split('/') # a range should be 2 elements if len(elements) is not 2: raise ValueError, "Invalid step passed: " + str(s) try: step = int(elements[1]) except: raise ValueError, "Invalid step provided " + str(s) r_list = [] # if the first element is *, use all valid numbers if elements[0] is "*" or elements[0] is "": r_list.extend(self._validate_type('*', t)) # check and see if there is a list of ranges elif "," in elements[0]: ranges = elements[0].split(",") for r in ranges: # if it's a range, we need to manage that if "-" in r: r_list.extend(self._validate_range(r, t)) else: try: r_list.extend(int(r)) except: raise ValueError, "Invalid step provided " + str(s) elif "-" in elements[0]: r_list.extend(self._validate_range(elements[0], t)) return range(r_list[0], r_list[-1] + 1, step) def _validate_dow(self, dow): """ """ # if dow is * return it. This is for date parsing where * does not mean # every day for crontab entries. if dow is "*": return dow days = { 'mon': 1, 'tue': 2, 'wed': 3, 'thu': 4, 'fri': 5, 'sat': 6, # per man crontab sunday can be 0 or 7. 'sun': [0, 7], } if dow in days: dow = days[dow] return [dow] # if dow is * return it. This is for date parsing where * does not mean # every day for crontab entries. elif dow is "*": return dow elif "/" in dow: return(self._validate_step(dow, "dow")) elif "," in dow: return(self._validate_list(dow, "dow")) elif "-" in dow: return(self._validate_range(dow, "dow")) else: valid_numbers = range(0, 8) if not int(dow) in valid_numbers: raise ValueError, "Invalid day of week " + str(dow) else: return [int(dow)] def _validate_mon(self, mon): months = { 'jan': 1, 'feb': 2, 'mar': 3, 'apr': 4, 'may': 5, 'jun': 6, 'jul': 7, 'aug': 8, 'sep': 9, 'oct': 10, 'nov': 11, 'dec': 12, } if mon in months: mon = months[mon] return [mon] elif mon is "*": return range(1, 13) elif "/" in mon: return(self._validate_step(mon, "mon")) elif "," in mon: return(self._validate_list(mon, "mon")) elif "-" in mon: return(self._validate_range(mon, "mon")) else: valid_numbers = range(1, 13) if not int(mon) in valid_numbers: raise ValueError, "Invalid month " + str(mon) else: return [int(mon)] def _validate_day(self, day): if day is "*": return range(1, 32) elif "/" in day: return(self._validate_step(day, "day")) elif "," in day: return(self._validate_list(day, "day")) elif "-" in day: return(self._validate_range(day, "day")) else: valid_numbers = range(1, 31) if not int(day) in valid_numbers: raise ValueError, "Invalid day " + str(day) else: return [int(day)] def _validate_hour(self, hour): if hour is "*": return range(0, 24) elif "/" in hour: return(self._validate_step(hour, "hour")) elif "," in hour: return(self._validate_list(hour, "hour")) elif "-" in hour: return(self._validate_range(hour, "hour")) else: valid_numbers = range(0, 23) if not int(hour) in valid_numbers: raise ValueError, "Invalid hour " + str(hour) else: return [int(hour)] def _validate_min(self, min): if min is "*": return range(0, 60) elif "/" in min: return(self._validate_step(min, "min")) elif "," in min: return(self._validate_list(min, "min")) elif "-" in min: return(self._validate_range(min, "min")) else: valid_numbers = range(0, 59) if not int(min) in valid_numbers: raise ValueError, "Invalid min " + str(min) else: return [int(min)] def _validate_url(self, url): # kludge for issue 842, right now we use request headers # to set the host. if url[0] is not "/": url = "/" + url url = 'http://' + str(os.environ['HTTP_HOST']) + url return url # content below is for when that issue gets fixed #regex = re.compile("^(http|https):\/\/([a-z0-9-]\.+)*", re.IGNORECASE) #if regex.match(url) is not None: # return url #else: # raise ValueError, "Invalid url " + url def _calc_month(self, next_run, cron): while True: if cron["mon"][-1] < next_run.month: next_run = next_run.replace(year=next_run.year+1, \ month=cron["mon"][0], \ day=1,hour=0,minute=0) else: if next_run.month in cron["mon"]: return next_run else: one_month = datetime.timedelta(months=1) next_run = next_run + one_month def _calc_day(self, next_run, cron): # start with dow as per cron if dow and day are set # then dow is used if it comes before day. If dow # is *, then ignore it. if str(cron["dow"]) != str("*"): # convert any integers to lists in order to easily compare values m = next_run.month while True: if next_run.month is not m: next_run = next_run.replace(hour=0, minute=0) next_run = self._calc_month(next_run, cron) if next_run.weekday() in cron["dow"] or next_run.day in cron["day"]: return next_run else: one_day = datetime.timedelta(days=1) next_run = next_run + one_day else: m = next_run.month while True: if next_run.month is not m: next_run = next_run.replace(hour=0, minute=0) next_run = self._calc_month(next_run, cron) # if cron["dow"] is next_run.weekday() or cron["day"] is next_run.day: if next_run.day in cron["day"]: return next_run else: one_day = datetime.timedelta(days=1) next_run = next_run + one_day def _calc_hour(self, next_run, cron): m = next_run.month d = next_run.day while True: if next_run.month is not m: next_run = next_run.replace(hour=0, minute=0) next_run = self._calc_month(next_run, cron) if next_run.day is not d: next_run = next_run.replace(hour=0) next_run = self._calc_day(next_run, cron) if next_run.hour in cron["hour"]: return next_run else: m = next_run.month d = next_run.day one_hour = datetime.timedelta(hours=1) next_run = next_run + one_hour def _calc_minute(self, next_run, cron): one_minute = datetime.timedelta(minutes=1) m = next_run.month d = next_run.day h = next_run.hour while True: if next_run.month is not m: next_run = next_run.replace(minute=0) next_run = self._calc_month(next_run, cron) if next_run.day is not d: next_run = next_run.replace(minute=0) next_run = self._calc_day(next_run, cron) if next_run.hour is not h: next_run = next_run.replace(minute=0) next_run = self._calc_day(next_run, cron) if next_run.minute in cron["min"]: return next_run else: m = next_run.month d = next_run.day h = next_run.hour next_run = next_run + one_minute def _get_next_run(self, cron): one_minute = datetime.timedelta(minutes=1) # go up 1 minute because it shouldn't happen right when added now = datetime.datetime.now() + one_minute next_run = now.replace(second=0, microsecond=0) # start with month, which will also help calculate year next_run = self._calc_month(next_run, cron) next_run = self._calc_day(next_run, cron) next_run = self._calc_hour(next_run, cron) next_run = self._calc_minute(next_run, cron) return next_run