# Programmer: Chris Bunch
# mapreduce app: A sample app to test out the following in AppScale:
# - MapReduce functionality ala Hadoop
# - Memcache Support
import wsgiref.handlers
from google.appengine.api import memcache
from google.appengine.api import users
from google.appengine.api.appscale.mapreduce import mapreduce
from google.appengine.ext import webapp
from google.appengine.ext.webapp import template
class MapReduceHandler(webapp.RequestHandler):
def get(self):
if users.is_current_user_capable("mr_api"):
currentVal = memcache.get("currentVal")
currentTask = memcache.get("currentTask")
if currentVal is None:
currentVal = ""
currentTask = ""
power = memcache.get("power-" + currentVal)
result = None
if currentTask == "runMRJob":
result = memcache.get("result-" + currentVal)
elif currentTask == "timing":
result = memcache.get("timing-" + currentVal)
if power is None:
power = "(Not run yet)"
if result is None:
result = "No result data yet."
else:
power = ""
result = """You are not authorized to run MapReduce jobs. Please get
authorization on your user account for MapReduce jobs and try again.
"""
self.response.out.write(template.render('mapreduce.html',
{'power': power,
'result': result}))
def post(self):
buttonClicked = self.request.get('button')
power = self.request.get('power')
try:
power = int(power)
except:
power = 3
powerStr = str(power)
memcache.set("currentVal", powerStr)
if buttonClicked == "Run MapReduce Job":
memcache.set("currentTask", "runMRJob")
self.runMRJob(powerStr)
elif buttonClicked == "Get Timing Info":
memcache.set("currentTask", "timing")
self.getLogs(powerStr)
self.redirect('/')
def getLogs(self, powerStr):
resultStr = memcache.get("timing-" + powerStr)
if resultStr is None:
outputFile = "output-" + powerStr
resultStr = mapreduce.getMRLogs(outputFile)
if resultStr == "this user cannot call the mapreduce api":
resultStr = """You are not authorized to view the log files.
Please log in as a user that it authorized to do so and try again.
"""
else:
resultStr = resultStr.replace("\n", "
")
resultStr = resultStr.replace("", " = ")
memcache.set("power-" + powerStr, powerStr)
memcache.set("timing-" + powerStr, resultStr)
def runMRJob(self,powerStr):
mapFile = "map.rb"
redFile = "reduce.rb"
power = int(powerStr)
outputFile = "output-" + powerStr
resultStr = memcache.get("result-" + powerStr)
if resultStr is None:
inputFile = "input-" + powerStr
authorized = self.genInput(power,inputFile)
if authorized == "success":
jobStarted = mapreduce.runMRJob(mapFile, redFile, inputFile, outputFile)
if jobStarted == "this user cannot call the mapreduce api":
return
resultStr = mapreduce.getMROutput(outputFile)
if resultStr == "this user cannot call the mapreduce api":
return
resultStr = resultStr.replace("\n", "
")
resultStr = resultStr.replace("sum", "Σ")
memcache.set("currentVal", powerStr)
memcache.set("currentTask", "runMRJob")
memcache.set("power-" + powerStr, powerStr)
memcache.set("result-" + powerStr, resultStr)
def genInput(self, power, inputLoc):
n = 2 ** power
numOfNodes = mapreduce.getNumOfNodes()
if numOfNodes == "this user cannot call the mapreduce api":
return numOfNodes
bucket_size = n / numOfNodes
vals = range(1, n / bucket_size + 1)
vals = [i * bucket_size for i in vals]
buckets = ""
index = 0
for i in vals:
if index == 0:
start = 0
else:
start = vals[index - 1]
this_range = str(start+1) + "\t" + str(vals[index]) + "\n"
buckets += this_range
index += 1
result = mapreduce.putMRInput(buckets, inputLoc)
if result == "this user cannot call the mapreduce api":
return result
return "success"
def main():
wsgiref.handlers.CGIHandler().run(webapp.WSGIApplication([
('/', MapReduceHandler),
]))
if __name__ == '__main__':
main()