Merge branch 'master' of github.com:seekshreyas/obidroid

* 'master' of github.com:seekshreyas/obidroid:
  concatenate json files
  moved concat to scripts
  mergedJson
  Create concat.py
  Adding web service capabilities and configuration
  Updates to sqlalchemy files that communicate with the DB.
  Updated files to include app developer information

db/getAppsReviews.py

@@ -0,0 +1,39 @@
+from sqlalchemy import *
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.orm import sessionmaker, relationship
+from itertools import islice, count
+from models.review import Review
+from models.app import App
+import pdb
+
+engine = create_engine('postgresql://postgres:[email protected]:28432/obidroid')
+engine.echo = True
+Base = declarative_base()
+Session = sessionmaker(bind=engine)
+session = Session()
+
+
+
+def get_all_apps():
+    return session.query(App).all()
+
+def get_app_by_appid(appid):
+    return session.query(App).filter(App.appid==appid).first()
+
+def get_apps_by_category(category):
+    return session.query(App).filter(App.category==category).all()
+
+def get_reviews_by_category(category):
+    qry = session.query(App, Review).filter(App.category==category)
+    reviews = qry.filter(App.appid==Review.playappid).all()
+    return reviews
+
+def get_reviews_by_appid(appid):
+    return session.query(Review).filter(Review.playappid==appid).all()
+
+if __name__ == '__main__':
+    #main
+    pass
+    #app_instance = get_reviews_by_appid('test.luis.app')
+    #revs = get_reviews_by_category('Test')
+    #print revs

db/models/app.py

@@ -1,5 +1,5 @@
 from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy import Column, Integer, String
+from sqlalchemy import Column, Float, Integer, String
 
 Base = declarative_base()
 
@@ -11,26 +11,35 @@ class App(Base):
     company = Column(String)
     content_rating = Column(String)
     description = Column(String)
+    dev_mail = Column(String)
+    dev_privacy_url = Column(String)
+    dev_url = Column(String)
     install = Column(String)
     name = Column(String)
+    rating = Column(Float)
     screen_count = Column(Integer)
     size = Column(String)
     total_reviewers = Column(Integer)
     version = Column(String)
 
-    def __init__(self, appid, category, company, content_rating, description, install, name, screen_count, size, total_reviewers, version):
+    def __init__(self, appid, category, company, content_rating, description, dev_mail, dev_privacy_url, dev_url, 
+    install, name, rating, screen_count, size, total_reviewers, version):
         self.appid = appid
         self.category = category
         self.company = company
         self.content_rating = content_rating
         self.description = description
+        self.dev_mail = dev_mail
+        self.dev_privacy_url = dev_privacy_url
+        self.dev_url = dev_url
         self.install = install
         self.name = name
+        self.rating = rating
         self.screen_count = screen_count
         self.size = size
         self.total_reviewers = total_reviewers
         self.version = version
 
     def __repr__(self):
-        return "<App(appid='%d', name='%s', description='%s')>" % (
+        return "<App(appid='%s', name='%s', description='%s')>" % (
         self.appid, self.name, self.description)

db/loadAppsReviews.py → db/putAppsReviews.py

@@ -17,10 +17,17 @@ def parse_review_json(fname='../exports/rawdata_reviews.json'):
     json_data = open(fname).read()
     return json.loads(json_data)
 
-def parse_all_json(fname='../exports/rawdata_all.json'):
+def parse_all_json(fname='../exports/game_brain_all.json'):
     json_data = open(fname).read()
     return json.loads(json_data)
 
+def calculateSingularRating(ratings):
+    total = 0; count = 0
+    for rating in ratings:
+        total = total + (int(rating[0].strip()) * int(rating[1]))
+        count = count + int(rating[1])
+    return total/float(count)
+
 def parse_reviews():
     # list container 
     review_list = list()
@@ -49,8 +56,8 @@ def parse_apps():
     # loop through the json file, first by app and then by reviews
     for idx, app in enumerate(islice(parse_all_json(), 1)):
         a = App(app['id'], app['category'], app['company'], app['contentRating'], 
-        app['description'], app['install'], app['name'], app['screenCount'], app['size'], 
-        app['totalReviewers'], app['version'])
+        app['description'], app['install'], app['name'], calculateSingularRating(app['rating']),
+        app['screenCount'], app['size'], app['totalReviewers'], app['version'])
         app_list.append(a)
 
     # only commit all the review for every 10 apps
@@ -71,9 +78,11 @@ def parse_all():
 
     # loop through the json file, first by app and then by reviews
     for idx, app in enumerate(islice(parse_all_json(), None)):
+
         a = App(app['id'], app['category'], app['company'], app['contentRating'], 
-        app['description'], app['install'], app['name'], app['screenCount'], app['size'], 
-        app['totalReviewers'], app['version'])
+        app['description'], app.get('devmail'), app.get('devprivacyurl'), app.get('devurl'),
+        app['install'], app['name'], calculateSingularRating(app['rating']), 
+        app['screenCount'], app['size'], app['totalReviewers'], app['version'])
         app_list.append(a)
         for review in app['reviews']:
             r = Review(app['id'], review[0], review[1])

nginx_ws/conf/nginx.conf

@@ -0,0 +1,46 @@
+worker_processes 4;
+
+events {}
+
+http {
+  upstream database {
+    postgres_server 127.0.0.1 dbname=obidroid user=postgres password=i219kkls;
+  }
+
+  server {
+    listen       8080;
+    server_name  localhost;
+
+    location /appsByCategory {
+      postgres_pass database;
+      rds_json on;
+      postgres_escape $category $arg_category;
+      postgres_query "SELECT * FROM app WHERE category=$category";
+      postgres_rewrite  HEAD GET no_rows 410;
+    }
+
+    location /reviewByAppId {
+      postgres_pass database;
+      rds_json on;
+      postgres_escape $appid $arg_appid;
+      postgres_query    HEAD GET  "SELECT * FROM review WHERE playappid=$appid";
+      postgres_rewrite  HEAD GET  no_rows 410;
+    }
+
+    location /appByAppId {
+      postgres_pass database;
+      rds_json  on;
+      postgres_escape $appid $arg_appid;
+      postgres_query    HEAD GET  "SELECT * FROM app WHERE appid=$appid";
+      postgres_rewrite  HEAD GET  no_rows 410;
+    }
+
+    location /reviewsByCategory {
+      postgres_pass database;
+      rds_json  on;
+      postgres_escape $category $arg_category;
+      postgres_query    HEAD GET  "SELECT app.appid, review.title, review.review FROM app, review WHERE app.category=$category AND app.appid=review.playappid";
+      postgres_rewrite  HEAD GET  no_rows 410;
+    }
+  }
+}

scripts/concat.py

@@ -0,0 +1,38 @@
+import json,os
+
+#ie mergeAllFiles('/Users/Artemis/Desktop/gpstore')
+
+def mergeAllFiles(directory):
+	"""Searches through current directory and merges 
+	the files that contain app info with the corresponding
+	reviews from the matching "reviews.json" file.
+	"""
+	allDirFiles = os.listdir(directory)
+	for jsonFile in allDirFiles:
+		if "_all.json" in jsonFile:
+			reviewFile = jsonFile[:-8] + "reviews.json"
+			mergeJson(jsonFile,reviewFile,directory)
+	return 
+
+def mergeJson(allJson,reviewJson,directory):
+	"""Modifies the json file containing the 
+	main information about the apps to also include 
+	the information contained in the reviews"""
+	results = []
+
+	allFile = json.load(open(directory + "/" + allJson))
+	reviewFile = json.load(open(directory + "/" + reviewJson))
+
+	for app in zip(allFile,reviewFile):
+		for field in app[1].keys():
+			app[0][field] = app[1][field]
+		results +=[app[0]]
+
+	with open(directory + "/" + allJson[:-8]+"merged.json",'w') as f:
+		json.dump(results,f,sort_keys=True)
+
+	print "Results written to",directory+"/"+allJson[:-8]+"merged.json"
+	return
+
+
+