HtmlParser

import re
import json
import requests
import numpy
class HtmlParser(object):
    def parser_url(self,page_url,response):
        pattern=re.compile(r'(http://movie.mtime.com/(\d+)/)')
        urls=pattern.findall(response)
        if urls!=None:
            return list(set(urls))
        else:
            return None
    def parser_json(self,page_url,response):
        pattern=re.compile(r'=(.*?);')
        result=pattern.findall(response)[0]
        if result!=None:
            value=json.loads(result)
            print(value)
            try:
                isRelease=value['value']['isRelease']
            except Exception as e:
                print('***this is parser_json!***',e)
                return None
            if isRelease:
                if value.get('value').get('hotValue')==None:
                    return self._parser_release(page_url,value)
                else:
                    return self._parser_no_release(page_url,value,isRelease=2)
            else:
                return self._parser_no_release(page_url,value)
    def _parser_release(self,page_url,value):
        try:
            isRelease=1
            movieRating=value.get('value').get('movieRating')
            boxOffice=value.get('value').get('boxOffice')
            movieTitle=value.get('value').get('movieTitle')
            RPictureFinal=movieRating.get('RPictureFinal')
            RStoryFinal=movieRating.get('RStoryFinal')
            RDirectorFinal=movieRating.get('RDirectorFinal')
            ROtherFinal=movieRating.get('ROtherFinal')
            RatingFinal=movieRating.get('RatingFinal')
            MovieId=movieRating.get('MovieId')
            Usercount=movieRating.get('Usercount')
            AttitudeCount=movieRating.get('AttitudeCount')
            TotalBoxOffice=boxOffice.get('TotalBoxOffice')
            TotalBoxOfficeUnit=boxOffice.get('TotalBoxOfficeUnit')
            TodayBoxOffice=boxOffice.get('TodayBoxOffice')
            TodayBoxOfficeUnit=boxOffice.get('TodayBoxOfficeUnit')
            ShowDays=boxOffice.get('ShowDays')
            try:
                Rank=boxOffice.get('Rank')
            except Exception as e:
                Rank=0
            return (MovieId,movieTitle,RatingFinal,
                    ROtherFinal,RPictureFinal,RDirectorFinal,
                    RStoryFinal,Usercount,AttitudeCount,
                    TotalBoxOffice+TotalBoxOfficeUnit,
                    TodayBoxOffice+TodayBoxOfficeUnit,
                    Rank,ShowDays,isRelease)
        except Exception as e:
            print(e,page_url,value)
            return None

    def _parser_no_release(self,page_url,value,isRelease=0):
        try:
            movieRating=value.get('value').get('movieRating')
            movieTitle=value.get('value').get('movieTitle')
            RPictureFinal=movieRating.get('RPictureFinal')
            RStoryFinal=movieRating.get('RStoryFinal')
            RDirectorFinal=movieRating.get('RDirectorFinal')
            ROtherFinal=movieRating.get('ROtherFinal')
            RatingFinal=movieRating.get('RatingFinal')
            MovieId=movieRating.get('MovieId')
            Usercount=movieRating.get('Usercount')
            AttitudeCount=movieRating.get('AttitudeCount')
            try:
                Rank=value.get('value').get('hotvalue').get('Ranking')
            except Exception as e:
                Rank=0
            return (MovieId,movieTitle,RatingFinal,
                    ROtherFinal,RPictureFinal,RDirectorFinal,
                    RStoryFinal,Usercount,AttitudeCount,
                    'N/a','N/a',Rank,0,isRelease)
        except Exception as e:
            print(e,page_url,value)
            return None