Initial Commit

environment.py

@@ -0,0 +1,36 @@
+APPEND_SYS_PATH='/home/pi/.kodi/addons/plugin.video.fox.news/resources/lib'
+PATH_VIDEO_DATABASE='/home/pi/.kodi/addons/plugin.video.fox.news/resources/lib/archive'
+USE_ICON_URL=True
+PATH_LOG_FILE="/home/pi/.kodi/temp/MyLog.log"
+FOX_NEWS_URL="https://www.foxnews.com/video"
+FOX_NEWS_US_URL="hhttps://www.foxnews.com/video/topics/us"
+FOX_NEWS_EXCLUSIVE_URL="https://foxnews.com"
+FOX_NEWS_AMERICAS_NEWSROOM_URL="https://www.foxnews.com/video/shows/americas-newsroom"
+FOX_NEWS_OUTNUMBERED_URL="https://www.foxnews.com/video/shows/outnumbered"
+SELECTED_ARCHIVE_URL="https://www.foxnews.com/video/dummy"
+#FOX_NEWS_ICON_OF_LAST_RESORT="https://static.foxnews.com/static/orion/styles/img/fox-news/favicons/apple-touch-icon-180x180.png"
+#ENABLE_USE_ICON_OF_LAST_RESORT=False
+
+
+VIDEODB_AMERICAS_NEWSROOM_FILENAME="videodb_americasnewsroom.txt"
+VIDEODB_OUTNUMBERED_FILENAME="videodb_outnumbered.txt"
+VIDEODB_FILENAME="videodb.txt"
+VIDEODB_EXCLUSIVE_FILENAME="videodb_exc.txt"
+VIDEODB_US_FILENAME="videodb_us.txt"
+ARCHIVEDB_FILENAME="archivedb.txt"
+
+CACHE_EXPIRY_MINS=10
+
+class PathHelper:
+    pathChar="/"
+
+    def __init__(self):
+      pass
+
+    @staticmethod
+    def makePathFileName(file,path):
+      while file.endswith(PathHelper.pathChar):
+        file=file[:-1]
+      while path.endswith(PathHelper.pathChar):
+        path=path[:-1]
+      return path+PathHelper.pathChar+file

newsfeed.py

@@ -0,0 +1,358 @@
+import tkinter as tk
+import math
+from functools import partial
+import json
+import webbrowser
+import os
+import requests
+import traceback
+import time
+import glob
+import re
+import shutil
+from datetime import timedelta
+from datetime import datetime
+from datetime import timezone
+from environment import *
+from utility import *
+
+class Video:
+    def __init__(self):
+        self.description=None
+        self.url=None
+        self.icon=None
+        self.timestamp=DateTime()
+        self.feedTimeOffset='just now'
+
+    def __init__(self, description, url):
+        self.description=description
+        self.url=url
+        self.icon=None
+        self.timestamp=DateTime()
+        self.feedTimeOffset='just now'
+
+    def __init__(self, description, url, icon, timestamp=None):
+        self.description=description
+        self.url=url
+        self.icon=icon
+        if None==timestamp:
+            self.timestamp=DateTime()
+        else:
+            self.timestamp=timestamp
+        self.feedTimeOffset='just now'
+
+    def getDescription(self):
+        return self.description
+
+    def getUrl(self):
+        return self.url
+
+    def getIcon(self):
+        return self.icon
+
+    def getTimestamp(self):
+        return self.timestamp
+
+    def setFeedTime(self,feedtime):
+        self.feedtime=feedtime
+
+    def getFeedTime(self):
+        return self.feedtime
+
+    def getFeedTimeOffset(self):
+        return self.feedTimeOffset
+
+    def setFeedTimeOffset(self, feedtimeoffset):
+        self.feedTimeOffset=feedtimeoffset
+
+    def toString(self):
+        return(self.description+"|"+self.url+"|"+self.icon+"|"+self.timestamp.toString())
+
+    @staticmethod
+    def fromString(line):
+        splits=line.split("|")
+        description=splits[0].strip()
+        url=splits[1].strip()
+        icon=splits[2].strip()
+        timestamp=DateTime(splits[3].strip())
+        return(Video(description,url,icon,timestamp))
+
+class NewsFeed:
+    def __init__(self, logger=None):
+        self.logger=logger
+
+    @staticmethod
+    def isResourceAvailable(url):
+        try:
+          response=requests.head(url, timeout=2.5)
+          if not response.ok:
+              return False
+          return True
+        except:
+          return False
+
+    def getItemsInFeed(self,url):
+      now=datetime.now()
+      sections=Sections()
+      videos = {}
+      httpNetRequest=HttpNetRequest()
+      response=httpNetRequest=httpNetRequest.getHttpNetRequest(url)
+      status=response.status_code
+      searchIndex=0
+      response.close()
+      if status!=200:
+          return None
+      while -1!= searchIndex:
+        video, searchIndex= sections.getItemsInSection(response.text,"article",searchIndex)
+        if video is not None and not (video.description in videos):
+            videos[video.description]=video
+            video.setFeedTime(DateTimeHelper.applyRelativeTime(now,video.feedTimeOffset))
+      videoList=NewsFeed.filterFeedMaxDays(list(videos.values()),7)
+#      videoList=list(videos.values())
+      videoList=sorted(videoList, key=lambda x:x.getFeedTime(),reverse=True)
+      return (videoList)
+
+    @staticmethod
+    def filterFeedMaxDays(videos,days):
+          now=datetime.now()
+          filteredList=[]
+          for video in videos:
+              delta=now-video.getFeedTime()
+              if delta.days <= days:
+                  filteredList.insert(0,video) 
+          return filteredList
+
+class Sections:
+    def __init__(self):
+      self.dummy=None
+
+    def getItemsInSection(self, strInput, sectionName, searchIndex):
+        video=None
+        startSection='<'+sectionName
+        endSection='</'+sectionName
+
+        startIndex=strInput.find(startSection,searchIndex)
+        if -1 == startIndex:
+            searchIndex=-1
+            return video, searchIndex
+
+        endIndex=strInput.find(endSection,startIndex)
+        if -1 == endIndex:
+            searchIndex=-1
+            return video, searchIndex
+
+        searchIndex=endIndex+len(endSection)
+        strContainingString=strInput[startIndex:endIndex+1+len(endSection)]
+
+        if not strContainingString or strContainingString=="":
+            return video, searchIndex
+
+        indexPreview=strContainingString.find("preview=\"")
+        if -1 == indexPreview:
+            return video, searchIndex
+        previewUrl=strContainingString[indexPreview:]
+        previewUrl=self.betweenString(previewUrl,'"','"')
+        if "tokenvod" in previewUrl:
+            return video, searchIndex
+
+        indexDescription=strContainingString.index("alt=\"")
+        description=strContainingString[indexDescription:]
+        description=self.betweenString(description,'"','"')
+        description=self.removeHtml(description)
+        description=description.replace("- Fox News","")
+        if "vod.foxbusiness" in description:
+            return video, searchIndex
+        indexDuration=strContainingString.index("<div class=\"duration\">")
+        if -1 != indexDuration:
+            strDuration=strContainingString[indexDuration:]
+            strDuration=self.betweenString(strDuration,">","<")
+            description=description+" - "+strDuration
+        indexPublication=strContainingString.index("<div class=\"pub-date\">")
+        if -1 != indexPublication:
+            strPublication=strContainingString[indexPublication:]
+            strPublication=self.betweenString(strPublication,"<time>","</time>")
+            description=description+" ("+strPublication+")"
+        icon=None
+        indexIcon=strContainingString.index("srcset=")
+        if -1 != indexIcon:
+            icon=strContainingString[indexIcon:]
+            icon=self.betweenString(icon,"\"","\"")
+            splits=icon.split(',')
+            icon=self.betweenString(splits[len(splits)-1],None,'?')
+            icon=icon.strip()
+        video=Video(description,previewUrl,icon)
+        video.feedTimeOffset=strPublication
+        return video, searchIndex
+
+    def getVideoIdInSection(self, strInput, sectionName, searchIndex):
+        video=None
+        startSection='<'+sectionName
+        endSection='</'+sectionName
+
+        startIndex=strInput.find(startSection,searchIndex)
+        if -1 == startIndex:
+            searchIndex=-1
+            return video, searchIndex
+
+        endIndex=strInput.find(endSection,startIndex)
+        if -1 == endIndex:
+            searchIndex=-1
+            return video, searchIndex
+
+        searchIndex=endIndex+len(endSection)
+        strContainingString=strInput[startIndex:endIndex+1+len(endSection)]
+        if not strContainingString or strContainingString=="":
+            return video, searchIndex
+        indexVideoId=strContainingString.find("data-video-id")
+        if -1 ==indexVideoId:
+            return video, searchIndex
+        videoId=strContainingString[indexVideoId:]
+        videoId=self.betweenString(videoId,"\"","\"")
+        return videoId, searchIndex
+
+    def getVideoContentInSection(self, strInput):
+        video=None
+        searchItem="\"contentUrl\":"
+        indexContentUrl=strInput.find(searchItem)
+        if -1 == indexContentUrl:
+            return None
+        strContentUrl=strInput[indexContentUrl+len(searchItem):]
+        strContentUrl=self.betweenString(strContentUrl,"\"","\"")
+        strContentUrl=strContentUrl.strip()
+
+        searchItem="\"description\":"
+        indexDescription=strInput.find(searchItem)
+        if -1 == indexDescription:
+            return None
+        strDescription=strInput[indexDescription+len(searchItem):]
+        strDescription=self.betweenString(strDescription,"\"","\"")
+        strDescription=strDescription.strip()
+
+        searchItem="\"thumbnailUrl\":"
+        indexIcon=strInput.find(searchItem)
+        if -1 == indexIcon:
+            return None
+        strIcon=strInput[indexIcon+len(searchItem):]
+        strIcon=self.betweenString(strIcon,"\"","\"")
+        strIcon=strIcon.strip()
+
+        searchItem="\"duration\""
+        indexDuration=strInput.find(searchItem)
+        if -1 != indexDuration:
+            strDuration=strInput[indexDuration+len(searchItem):]
+            strDuration=self.betweenString(strDuration,"\"","\"")
+            strDuration=strDuration.strip()
+            minutes, seconds = parseDuration(strDuration)
+            if None!=minutes and None!=seconds:
+                strDescription=strDescription+" - "+minutes+":"+seconds
+        video=Video(strDescription,strContentUrl,strIcon)
+        return video
+
+    def betweenString(self, strItem, strBegin, strEnd ):
+        return StringHelper.betweenString(strItem, strBegin, strEnd)
+
+    def removeHtml(self,strItem):
+        if strItem is None:
+            return None
+        codes={"&#x27;","&#187;"}
+        for code in codes:
+            strItem=strItem.replace(code,"'")
+        strItem=strItem.replace("&amp;","&")
+        strItem=strItem.replace("&#x2018;","'")
+        strItem=strItem.replace("&#x2019;","'")
+        strItem=strItem.replace("&#x2014;","-")
+        strItem=strItem.replace("???","'")
+        return strItem
+
+def pad(str,filler,length):
+    stringLength=len(str)
+    sb=""
+    if stringLength>=length:
+        return str
+    while stringLength < length:
+        sb=sb+filler
+        stringLength=stringLength+1
+    return sb+str
+
+def parseDuration(strDuration):
+    expression=re.compile(r"\d+")
+    result=expression.findall(strDuration)
+    if 2!=len(result):
+        return None, None
+    return pad(result[0],'0',2), pad(result[1],'0',2)
+
+
+def selectedItem(event):
+  widget=event.widget
+  selectedIndex=int(widget.curselection()[0])
+  itemsInList=len(videoCollection.getVideoList())
+  if selectedIndex >=0 and selectedIndex < itemsInList:
+    videoUrl=videoCollection.getVideoList()[selectedIndex].getUrl()
+    webbrowser.open(videoUrl)
+
+def refresh():
+    listbox.delete(0,'end')
+    loadVideos()
+
+def loadVideos():
+   newsfeed=NewsFeed()
+   videos=newsfeed.getItemsInFeed(FOX_NEWS_URL)
+   itemIndex=1
+   for video in videos:
+     listbox.insert(itemIndex,video.getDescription())
+     itemIndex=itemIndex+1
+   videoCollection.setVideoList(videos)
+
+class VideoCollection:
+    def __init__(self):
+        self.videoList=[]
+
+    def getVideoList(self):
+        return self.videoList
+
+    def setVideoList(self,videoList):
+          self.videoList=videoList
+
+videoCollection=VideoCollection()
+
+master = tk.Tk()   
+master.geometry("450x300")  
+#master.geometry("640x480")  
+
+#Frame
+frame=tk.Frame(master,height=100)
+frame.pack(fill='both',expand='true')
+
+bottomframe=tk.Frame()
+bottomframe.pack(side='bottom')
+
+#Entry
+label_heading = tk.Label(frame, text = "Fox News Video Feed",font='arial 13 bold',background='blue',foreground='white')  
+label_heading.pack(side='top', fill='both')
+
+#yscrollbar for list
+yscrollbar=tk.Scrollbar(frame)
+yscrollbar.pack(side=tk.RIGHT, fill=tk.Y)
+
+#xscrollbar for list
+xscrollbar=tk.Scrollbar(frame, orient=tk.HORIZONTAL)
+xscrollbar.pack(side=tk.BOTTOM,fill=tk.X)
+
+# List
+listbox=tk.Listbox(frame,selectmode='single',yscrollcommand=yscrollbar.set,xscrollcommand=xscrollbar.set)
+listbox.pack(side='left', fill='both')
+listbox.bind('<Double-Button>',selectedItem)
+listbox.configure(background='skyblue4',foreground='white',font='arial 8 bold',width=0)
+
+#configure the scrollbar
+yscrollbar.config(command=listbox.yview)
+xscrollbar.config(command=listbox.xview)
+
+#Button
+btn_refresh = tk.Button(bottomframe,text = 'Refresh',command = refresh)
+btn_refresh.pack(side='bottom')
+
+loadVideos()                                                                                                
+listbox.focus_set()
+                                                                                          
+master.mainloop()

utility.py

@@ -0,0 +1,231 @@
+import time
+import webbrowser
+import requests
+from datetime import timedelta
+from datetime import datetime
+from datetime import timezone
+from environment import *
+
+class myLog():
+  def __init__(self):
+    self._file = open(PATH_LOG_FILE,"a",encoding='utf-8')
+
+  def write(self,item):
+    currentDateTime = DateTimeHelper.getCurrentDateTime()
+    strCurrentDateTime = DateTimeHelper.getDateTimeAsString(currentDateTime) 
+    strOutput = '[' + strCurrentDateTime +'] '+item
+    self._file.write(strOutput)
+    self._file.write("\n")
+    self._file.flush()
+
+class StringHelper:
+    def __init__(self):
+        pass
+
+    @staticmethod
+    def betweenString(strItem, strBegin, strEnd):
+        if strItem is None:
+            return None
+        index=-1
+        if strBegin is None:
+            index=0
+        else:
+            index = strItem.index(strBegin)
+        if -1==index:
+            return None
+        str=None
+        if strBegin is not None:
+            str=strItem[index+len(strBegin):]
+        else:
+            str=strItem
+        if strEnd is None:
+            return str
+        index=str.index(strEnd)
+        if -1==index :
+            return None
+        sb=""
+        for strIndex in range(0, len(str)-1):
+            if index==strIndex:
+                break
+            sb=sb+str[strIndex]
+        return (sb)
+
+class HttpNetRequest:
+    def __init__(self):
+        self.Message=""
+
+    def getHttpNetRequest(self,url):
+        retrycount=0
+        maxretries=5
+        while retrycount<maxretries:
+            try:
+              response=requests.get(url, timeout=10)
+              return response
+            except:
+                retrycount=retrycount+1
+                if retrycount > maxretries:
+                    raise
+
+
+class DateTimeHelper:
+    def __init__(self):
+        pass
+
+    def __init__(self,capturedatetime,timeChange):
+        self.datetimestamp=capturedatetime
+        self.timeChange=timeChange
+        self.offsetTime=DateTimeHelper.applyRelativeTime(self.datetimestamp, self.timeChange)
+
+    def getDateTimeStamp(self):
+        return self.datetimestamp
+
+    def getTimeChange(self):
+        return self.timeChange
+
+    def getOffsetTime(self):
+        return self.offsetTime
+
+    def getOffsetTimeAsString(self):
+        return DateTimeHelper.getDateTimeAsString(self.offsetTime)
+
+    def toString(self):
+        pass
+
+    @staticmethod
+    def getDateTimeAsString(someDateTime):
+        if(not isinstance(someDateTime,datetime)):
+            raise Exception('Invalid type for parameter')
+        return someDateTime.strftime("%m-%d-%Y %H:%M:%S")
+
+    @staticmethod 
+    def getDateTimeFromString(someDateTimeString):
+        if(not isinstance(someDateTimeString,str)):
+            raise Exception('Invalid type for parameter')
+        return DateTimeHelper.strptime(someDateTimeString,"%m-%d-%Y %H:%M:%S")
+
+    @staticmethod
+    def getCurrentDateTime():
+        return datetime.now()
+
+    @staticmethod
+    def strptime(theTime,theFormat):
+        try:
+            return datetime.strptime(theTime,theFormat)
+        except:
+            return datetime(*(time.strptime(theTime,theFormat)[0:6]))
+
+    @staticmethod 
+    def canstrptime(theTime,theFormat):
+        try:
+            datetime.strptime(theTime,theFormat)
+            return True
+        except:
+            return False
+
+
+
+# returns a datetime
+    @staticmethod
+    def applyRelativeTime(sometime,relativetime):
+        if(not isinstance(sometime,datetime)):
+            raise Exception('Invalid type for parameter')
+        if(not isinstance(relativetime,str)):
+            raise Exception('Invalid type for parameter')
+        if DateTimeHelper.canstrptime(relativetime,'%B %d, %Y'):
+            sometime = DateTimeHelper.strptime(relativetime,'%B %d, %Y')
+            return sometime
+        if relativetime=='just now':
+            return sometime
+        if relativetime=='just in':
+            return sometime
+        relativetimesplit=relativetime.split()
+        if len(relativetimesplit)==2:
+            year=datetime.now().year
+            relativetimex=relativetime+', '+str(year)
+            relativeDate = DateTimeHelper.strptime(relativetimex, '%B %d, %Y')
+            if(relativeDate>datetime.now()):
+              year=datetime.now().year-1
+              relativetimex=relativetime+', '+str(year)
+              relativeDate=DateTimeHelper.strptime(relativetimex,'%B %d, %Y')
+            days=sometime-relativeDate
+            sometime=sometime-days
+        elif relativetimesplit[1]=='hour' or relativetimesplit[1]=='hours':
+            hours=int(relativetimesplit[0])
+            sometime=sometime-timedelta(hours=hours)
+        elif relativetimesplit[1]=='day' or relativetimesplit[1]=='days':
+            days=int(relativetimesplit[0])
+            sometime=sometime-timedelta(days=days)
+        elif relativetimesplit[1]=='minute' or relativetimesplit[1]=='minutes':
+            minutes=int(relativetimesplit[0])
+            sometime=sometime-timedelta(minutes=minutes)
+        elif len(relativetimesplit)==3:  # '16 mins ago'  '2 hours ago'
+          if relativetimesplit[1]=='mins':
+              minutes=int(relativetimesplit[0])
+              sometime=sometime-timedelta(minutes=minutes)
+          elif relativetimesplit[1]=='hours':
+            hours=int(relativetimesplit[0])
+            sometime=sometime-timedelta(hours=hours)
+          elif relativetimesplit[1]=='day' or relativetimesplit[1]=='days':
+            days=int(relativetimesplit[0])
+            sometime=sometime-timedelta(days=days)              
+        return sometime
+    
+class DateTime:
+    def __init__(self):
+        self.date=DateTime.getCurrentTime()
+
+    def __init__(self,strDate=None):
+        if None!=strDate:
+            self.date=DateTime.dateFromString(strDate)
+        else:
+            self.date=DateTime.getCurrentTime()
+
+    def toString(self):
+        return DateTime.dateToString(self.date)
+
+    def deltaTime(self,someDate):
+      return DateTime.deltaTime(self.date,someDate)
+
+    @staticmethod
+    def dateToString(someDate):
+        return str(someDate)
+
+    @staticmethod
+    def dateFromString(strDate):
+        return datetime.fromisoformat(strDate)
+
+    @staticmethod
+    def now():
+        return DateTime.getCurrentTime()
+
+    @staticmethod
+    def getCurrentTime():
+        return datetime.now(timezone.utc)
+
+    @staticmethod
+    def sortList(dateList):
+        dateList.sort(key=lambda x:x.toString())
+
+    @staticmethod
+    def deltaTime(startTime,endTime):
+        if startTime > endTime:
+            timedelta=startTime-endTime
+        else:
+            timedelta=endTime-startTime
+        days, seconds=timedelta.days, timedelta.seconds
+        hours=timedelta.total_seconds()//3600
+        minutes=(seconds %3600)//60
+        seconds=seconds%60
+        return days, hours, minutes, seconds
+
+
+
+#currentDate=DateTimeHelper.getCurrentDateTime()
+#strDateTime=DateTimeHelper.getDateTimeAsString(currentDate)
+#print(strDateTime)
+#relativeTime=DateTimeHelper.applyRelativeTime(currentDate,'October 9')
+#strDateTime=DateTimeHelper.getDateTimeAsString(relativeTime)
+#print(relativeTime)
+#if(relativeTime>currentDate):
+#  print('It is greater')
+