Feature/abhishekangale/issue 65/codechef routes

codedigger/codechef/cron.py

@@ -4,7 +4,7 @@
 
 def update_AllContests():
     # Creates new contests and problems in Database
-    all_contests = ContestData('past')
+    all_contests = ContestData('all', 'past')
     for contest in all_contests:
         create_or_update_codechefContest(contest)
         contest_problems_info = ProblemData(contest['ContestCode'])

codedigger/codechef/model_utils.py

@@ -6,16 +6,19 @@
 def create_or_update_codechefProblem(problemdata):
     for problem in problemdata:
         Prob, created = Problem.objects.get_or_create(
-            name=problem['Name'],
             prob_id=problem['ProblemCode'],
-            url=problem['ProblemURL'],
-            contest_id=problem['ContestId'],
-            platform=problem['Platform'])
+            platform=problem['Platform'],
+            defaults={
+                'name': problem['Name'],
+                'url': problem['ProblemURL'],
+                'contest_id': problem['ContestId'],
+            },
+        )
+
         cont = CodechefContest.objects.get(contestId=problem['ContestId'])
-        prob = Problem.objects.get(prob_id=problem['ProblemCode'],
-                                   contest_id=problem['ContestId'])
+
         ccprob, created = CodechefContestProblems.objects.get_or_create(
-            contest=cont, problem=prob)
+            contest=cont, problem=Prob)
 
 
 def create_or_update_codechefContest(contest):

codedigger/codechef/scraper.py

@@ -21,7 +21,8 @@ def divisionScraper(contest_id):
 
 def contestScraper(offset, contest_type):
 
-    query_contest_url = f"https://www.codechef.com/api/list/contests/{contest_type}?sort_by=START&sorting_order=desc&offset={offset}&mode=premium"
+    query_contest_url = f"https://www.codechef.com/api/list/contests/" + contest_type + "?sort_by=START&sorting_order=desc&offset=" + str(
+        offset) + "&mode=premium"
     # Query URL might change in future.
     contest_data = requests.get(query_contest_url)
 
@@ -46,104 +47,19 @@ def problemScraper(contest_code):
     return problem_data
 
 
-def UserSubmissionDetail(problemcode, contest, user):
-    URL = f"https://www.codechef.com/{contest}/status/{problemcode},{user}"
+def UserSubmissionScraper(URL):
+
     r = requests.get(URL)
     soup = BeautifulSoup(r.content, 'html5lib')
-    problemTable = soup.findAll('table', class_="dataTable")
-    problemRow = problemTable[0].findAll('tr')
-    problemRow.pop(0)
-    submissionlist = []
-    if len(problemRow) == 0 or problemRow[0].text == 'No Recent Activity':
-        return submissionlist
-
-    for problem in problemRow:
-        baseurl = "https://www.codechef.com"
-        problemDetails = problem.findAll('td')
-        subid = problemDetails[0].text
-        subtime = problemDetails[1].text
-        verdict = problemDetails[3].find('span').get('title')
-        if len(verdict) == 0:
-            verdict = (problemDetails[3].find('span').text)
-            verdict = verdict[:verdict.index('[')]
-            if int(verdict) == 100:
-                verdict = "accepted [100/100]"
-            else:
-                verdict = "partially accepted [" + verdict + "/100]"
-        lang = problemDetails[6].text
-        link = baseurl + problemDetails[7].find('a').get('href')
-
-        subformat = {
-            'subid': subid,
-            'subtime': subtime,
-            'verdict': verdict,
-            'lang': lang,
-            'link': link,
-        }
-
-        submissionlist.append(subformat)
-
-    return submissionlist
-
-
-def recentSubmissions(userid):
-    URL = f"https://www.codechef.com/recent/user?user_handle={userid}"
+    return soup
+
+
+def recentSubmissionScraper(user_handle):
+
+    URL = f"https://www.codechef.com/recent/user?user_handle={user_handle}"
     r = requests.get(URL)
-    r = BeautifulSoup(r.content, 'html5lib')
-    recentSubs = r.findAll('tbody')
-
-    recentlist = []
-    for sub in recentSubs:
-        subd = sub.findAll('tr')
-        subd.pop(-1)
-        try:
-            query = subd[0].text[:18]
-        except:
-            query = "Profile found successfully"
-        if query == 'No Recent Activity':
-            break
-
-        for prob in subd:
-            baseurl = "https://www.codechef.com"
-            det = prob.findAll('td')
-
-            probid = det[1].find('a').text
-            probid = probid[:probid.index('<')]
-
-            link = det[1].find('a').get('href')
-            link = link.replace("\\", "")
-            link = baseurl + link
-
-            subtime = prob.find('span', class_="tooltiptext")
-            try:
-                subtime = subtime.text
-                subtime = subtime[:subtime.index('<')]
-            except:
-                break
-
-            verdict = det[2].find('span').get('title')
-            if len(verdict) == 0:
-                verdict = (det[2].find('span').text)
-                verdict = verdict[:verdict.index('[')]
-                if int(verdict) == 100:
-                    verdict = "accepted [100/100]"
-                else:
-                    verdict = "partially accepted [" + verdict + "/100]"
-
-            lang = det[3].text
-            lang = lang[:lang.index('<')]
-
-            subformat = {
-                'probid': probid,
-                'subtime': subtime,
-                'verdict': verdict,
-                'lang': lang,
-                'link': link,
-            }
-
-            recentlist.append(subformat)
-
-    return recentlist
+    soup = BeautifulSoup(r.content, 'html5lib')
+    return soup
 
 
 def profilePageScraper(user_handle):

codedigger/codechef/scraper_utils.py

@@ -1,4 +1,4 @@
-from codechef.scraper import contestScraper, problemScraper, profilePageScraper
+from codechef.scraper import contestScraper, problemScraper, profilePageScraper, recentSubmissionScraper, UserSubmissionScraper
 
 
 def OffsetLoader(contest_type):
@@ -30,9 +30,9 @@ def getContestDivision(contest_id):
     return subcontests
 
 
-def ContestData(type):
+def ContestData(cont_type, cont_time):
 
-    contests_data = OffsetLoader(type)
+    contests_data = OffsetLoader(cont_time)
     all_contests = []
     dateDict = {
         "Jan": "January",
@@ -48,7 +48,35 @@ def ContestData(type):
         "Nov": "November",
         "Dec": "December"
     }
+
+    longContestCode = [
+        'JAN', 'FEB', 'MARCH', 'APRIL', 'MAY', 'JUNE', 'JULY', 'AUG', 'SEPT',
+        'OCT', 'NOV', 'DEC'
+    ]
+
     for contest in contests_data:
+
+        if cont_type == 'starters':
+            if contest['contest_code'][:5] != 'START':
+                continue
+
+        if cont_type == 'lunchtime':
+            if contest['contest_code'][:5] != 'LTIME':
+                continue
+
+        if cont_type == 'cookoff':
+            if contest['contest_code'][:4] != 'COOK':
+                continue
+
+        if cont_type == 'long':
+            found = 0
+            for month in longContestCode:
+                if contest['contest_code'][:len(month)] == month:
+                    found = 1
+                    break
+            if found == 0:
+                continue
+
         childContests = getContestDivision(contest['contest_code'])
 
         for contest_id in childContests:
@@ -174,3 +202,139 @@ def userScraper(user_handle):
     # user.country_rank = country_rank
     # user.global_rank = global_rank
     # user.save()
+
+
+def RecentSubmission(user_handle):
+    soup = recentSubmissionScraper(user_handle)
+    recentSubs = soup.findAll('tbody')
+
+    recentlist = []
+    for sub in recentSubs:
+        subd = sub.findAll('tr')
+        subd.pop(-1)
+        try:
+            query = subd[0].text[:18]
+        except:
+            query = "Profile found successfully"
+        if query == 'No Recent Activity':
+            break
+
+        for prob in subd:
+            baseurl = "https://www.codechef.com"
+            det = prob.findAll('td')
+
+            probid = det[1].find('a').text
+            probid = probid[:probid.index('<')]
+
+            link = det[1].find('a').get('href')
+            link = link.replace("\\", "")
+            link = baseurl + link
+
+            subtime = prob.find('span', class_="tooltiptext")
+            try:
+                subtime = subtime.text
+                subtime = subtime[:subtime.index('<')]
+            except:
+                break
+
+            subtime = subtime.replace("\\", "")
+
+            verdict = det[2].find('span').get('title')
+            if len(verdict) == 0:
+                verdict = (det[2].find('span').text)
+                verdict = verdict[:verdict.index('[')]
+                if int(verdict) == 100:
+                    verdict = "accepted [100/100]"
+                else:
+                    verdict = "partially accepted [" + verdict + "/100]"
+
+            lang = det[3].text
+            lang = lang[:lang.index('<')]
+
+            subformat = {
+                'probid': probid,
+                'subtime': subtime,
+                'verdict': verdict,
+                'lang': lang,
+                'link': link,
+            }
+
+            recentlist.append(subformat)
+
+    return recentlist
+
+
+def allProblemsSolved(user_handle):
+
+    soup = profilePageScraper(user_handle)
+    print(user_handle)
+    problems_solved = []
+
+    all_contests = soup.find('article')
+    contests_list = all_contests.find_all('p')
+    cont = (contests_list[0].find('strong').contents)[0][:-1]
+
+    # if cont == "Practice":
+    #     probs = contests_list[0].find_all('a')
+    #     for prob in probs:
+    #         upsolved_problems.append(prob.contents[0])
+
+    probs = all_contests.find_all('a')
+    for prob in probs:
+        link = prob['href']
+        name = prob.contents[0]
+
+        problems_solved.append((name, link))
+
+    return problems_solved
+
+
+def UserSubmissionDetails(problemcode, user_handle):
+
+    url = ""
+    solvedByUser = allProblemsSolved(user_handle)
+
+    for solved in solvedByUser:
+        if solved[0] == problemcode:
+            url = solved[1]
+            break
+
+    if len(url) == 0:
+        return []
+
+    baseurl = f'https://www.codechef.com/'
+    soup = UserSubmissionScraper(baseurl + url)
+    problemTable = soup.findAll('table', class_="dataTable")
+    problemRow = problemTable[0].findAll('tr')
+    problemRow.pop(0)
+    submissionlist = []
+    if len(problemRow) == 0 or problemRow[0].text == 'No Recent Activity':
+        return submissionlist
+
+    for problem in problemRow:
+        baseurl = "https://www.codechef.com"
+        problemDetails = problem.findAll('td')
+        subid = problemDetails[0].text
+        subtime = problemDetails[1].text
+        verdict = problemDetails[3].find('span').get('title')
+        if len(verdict) == 0:
+            verdict = (problemDetails[3].find('span').text)
+            verdict = verdict[:verdict.index('[')]
+            if int(verdict) == 100:
+                verdict = "accepted [100/100]"
+            else:
+                verdict = "partially accepted [" + verdict + "/100]"
+        lang = problemDetails[6].text
+        link = baseurl + problemDetails[7].find('a').get('href')
+
+        subformat = {
+            'subid': subid,
+            'subtime': subtime,
+            'verdict': verdict,
+            'lang': lang,
+            'link': link,
+        }
+
+        submissionlist.append(subformat)
+
+    return submissionlist

codedigger/codechef/urls.py

@@ -5,5 +5,13 @@
     path('upsolve',
          views.CodechefUpsolveAPIView.as_view(),
          name="codechef-upsolve"),
+    path('recentsub/<str:username>',
+         views.CodechefRecentSubmissionAPIView.as_view()),
+    path('problemsub/<str:username>/<str:problem>',
+         views.CodechefUserSubmissionAPIView.as_view()),
+    path('contestproblems/<str:contest>',
+         views.CodechefContestProblemsAPIView.as_view()),
+    path('contests/<str:time>/<str:typec>',
+         views.CodechefContestsAPIView.as_view()),
     path('testing', views.testing)
 ]