Initial Commit

.gitignore

@@ -0,0 +1,4 @@
+__pycache__/environment.cpython-39.pyc
+__pycache__/utility.cpython-39.pyc
+__pycache__/video.cpython-39.pyc
+.vscode/settings.json

Test.py

@@ -0,0 +1,124 @@
+from datetime import datetime
+from datetime import timedelta
+
+class DateTimeHelper:
+    def __init__(self):
+        pass
+
+    def __init__(self,capturedatetime,timeChange):
+        self.datetimestamp=capturedatetime
+        self.timeChange=timeChange
+        self.offsetTime=DateTimeHelper.applyRelativeTime(self.datetimestamp, self.timeChange)
+
+    def getDateTimeStamp(self):
+        return self.datetimestamp
+    
+    def getTimeChange(self):
+        return self.timeChange
+    
+    def getOffsetTime(self):
+        return self.offsetTime
+    
+    def getOffsetTimeAsString(self):
+        return DateTimeHelper.getDateTimeAsString(self.offsetTime)
+    
+    def toString(self):
+        pass
+
+    @staticmethod
+    def getDateTimeAsString(someDateTime):
+        if(not isinstance(someDateTime,datetime)):
+            raise Exception('Invalid type for parameter')
+        return someDateTime.strftime("%m-%d-%Y %H:%M:%S")
+
+    @staticmethod 
+    def getDateTimeFromString(someDateTimeString):
+        if(not isinstance(someDateTimeString,str)):
+            raise Exception('Invalid type for parameter')
+        return datetime.strptime(someDateTimeString,"%m-%d-%Y %H:%M:%S")
+
+    @staticmethod
+    def getCurrentDateTime():
+        return datetime.now()
+
+    @staticmethod
+    def applyRelativeTime(sometime,relativetime):
+        if(not isinstance(sometime,datetime)):
+            raise Exception('Invalid type for parameter')
+        if(not isinstance(relativetime,str)):
+            raise Exception('Invalid type for parameter')
+        if relativetime=='just now':
+            return sometime
+        relativetimesplit=relativetime.split()
+        if len(relativetimesplit)==2:
+            year=datetime.now().year
+            relativetime=relativetime+', '+str(year)
+            relativeDate = datetime.strptime(relativetime, '%B %d, %Y')
+            days=sometime-relativeDate
+            # sometime=sometime-timedelta(days=days)
+            sometime=sometime-days
+        elif relativetimesplit[1]=='hour' or relativetimesplit[1]=='hours':
+            hours=int(relativetimesplit[0])
+            sometime=sometime-timedelta(hours=hours)
+        elif relativetimesplit[1]=='day' or relativetimesplit[1]=='days':
+            days=int(relativetimesplit[0])
+            sometime=sometime-timedelta(days=days)
+        elif relativetimesplit[1]=='minute' or relativetimesplit[1]=='minutes':
+            minutes=int(relativetimesplit[0])
+            sometime=sometime-timedelta(minutes=minutes)
+        return sometime
+
+list=[]
+
+
+# date_time_str='03-03-2023 12:00:00'
+# capturedatetime = datetime.strptime(date_time_str, '%m-%d-%Y %H:%M:%S')
+# Item.applyRelativeTime(capturedatetime, '2 days ago')
+
+deltaTime='March 7'
+now = datetime.now()
+relativeTime=DateTimeHelper.applyRelativeTime(now, deltaTime)
+relativeTimeStr=DateTimeHelper.getDateTimeAsString(relativeTime)
+
+print('time:{p1}'.format(p1=DateTimeHelper.getDateTimeAsString(now)))
+print('delta:{p1}'.format(p1=deltaTime))
+print('result:{p1}'.format(p1=relativeTimeStr))
+
+
+#now = Item.getCurrentDateTime()
+#capturedatetime = Item.getDateTimeAsString(now)
+
+#Item.applyRelativeTime(capturedatetime, 'February 24')
+
+
+list.append(Item(capturedatetime,'February 24'))
+list.append(Item(capturedatetime,'2 days ago'))
+list.append(Item(capturedatetime,'21 days ago'))
+list.append(Item(capturedatetime,'1 day ago'))
+list.append(Item(capturedatetime,'1 minute ago'))
+list.append(Item(capturedatetime,'5 minutes ago'))
+
+for item in list:
+    print (item)
+
+
+
+# date_time_str='01-01-2023 12:00:00'
+# b = datetime.strptime(date_time_str, '%m-%d-%Y %H:%M:%S')
+
+# date_time_str='01-01-2023 18:00:00'
+# c = datetime.strptime(date_time_str, '%m-%d-%Y %H:%M:%S')
+
+# list.append(a)
+# list.append(b)
+# list.append(c)
+
+# print(list)
+
+# another=sorted(list, key=lambda x:x)
+
+# print(another)
+# for item in another:
+#     print('{:02d}-{:02d}-{:d} {:02d}:{:02d}:{:02d}'.format(item.month,item.day,item.year,item.hour,item.minute,item.second))
+
+

archive.py

@@ -0,0 +1,159 @@
+import os
+import glob
+import functools
+from environment import *
+from utility import *
+from video import *
+
+# This file is executed in a cron job.
+# To view the cron schedule type sudo crontab -r in a shell.  Use Ctrl-S to save after editing
+# This cron job should run evrry 30 minutes.  Shorter intervals burden the system
+# The ouptut from the print statements is generated in the syslog /var/log/syslog    sudo nano /var/log/syslog
+# Overall system perfromance can be monitored using htop
+
+def comparator(item1, item2):
+    try:
+      list1=item1.split('.')
+      list2=item2.split('.')
+      index1=int(list1[len(list1)-1])
+      index2=int(list2[len(list2)-1])
+      if index1<index2:
+        return -1
+      elif index1>index2:
+        return 1
+      return 0
+    except:
+      return 0
+
+def createArchive(pathOutputFile,tokens,files):
+   lines=0
+   unique={}
+   videos={}
+
+   for token in tokens:
+     print('Filtering for "{token}"'.format(token=token))
+
+   videos = Video.load(pathOutputFile) 
+
+   for video in list(videos.values()):
+     description = description=createDescription(video.description,video.getTimestamp())
+     if not description in unique:
+       unique[description]=createDescription(video.description, video.getTimestamp())
+
+   try:
+     print('found {count} archive files.'.format(count=len(files)))
+     print('processing {pathOutputFile}'.format(pathOutputFile=pathOutputFile))
+     for file in files:
+        try:
+          with open(file, "r", encoding='utf-8') as inputStream:
+              for line in inputStream:
+                lowerLine=line.lower()
+                for token in tokens:
+                  token=token.lower()
+                  result = lowerLine.find(token)
+                  if -1 != result:
+                    video = Video.fromString(line)
+                    heading = video.getDescription()
+                    if not heading in unique:
+                      unique[heading]=heading
+                      video = Video.fromString(line)
+                      video.description=createDescription(video.description,video.getTimestamp())
+                      videos[video.description]=video
+                      lines = lines + 1
+          inputStream.close()
+        except Exception as exception:
+          print('Exception reading {file} {exception}'.format(file=file,exception=exception))
+          continue
+     print('writing {pathOutputFile}'.format(pathOutputFile=pathOutputFile))
+     Video.write(pathOutputFile, videos)   
+   except Exception as exception:
+     print('Exception creating output file {file} {exception}'.format(file=pathOutputFile,exception=exception))
+   return
+
+# clean the archive files by removing files older than 'expiryDays'
+def cleanArchive(files, expiryDays):
+  expiredList = []
+  for pathFileName in files:
+    modification_date = os.path.getmtime(pathFileName)
+    modification_date = datetime.fromtimestamp(modification_date, timezone.utc)
+    now = DateTime.now()
+    days, hours, minutes, seconds = DateTime.deltaTime(modification_date, now)
+    if(days > expiryDays):
+      expiredList.append(pathFileName)
+  print('Expiring {count} files.'.format(count=len(expiredList)))
+  for file in expiredList:
+    os.remove(file)
+  return
+
+def createDescription(strDescription, timeStamp):
+  textElement=StringHelper.betweenString(strDescription,None,'-')
+  timeElement=StringHelper.betweenString(strDescription,'-',None)
+  durationElement=StringHelper.betweenString(timeElement,' ',' ')
+  newDescription=textElement+'-'+' '+ durationElement+' ('+timeStamp.toStringMonthDay()+')'
+  return newDescription
+
+def getFiles(archiveFileLike):
+  files = glob.glob(archiveFileLike)
+  files=files+glob.glob(archiveFileLike+'.*')
+  return files
+
+# This program runs through all of the videodb*.txt files looking for keywords with which to
+# build each of the individually named mini-archives.
+# 1) Search for all videodb.txt.* files 
+# 2) Expire files older than specified number of days
+# 3) Load the archive (for each fo the types enumerated below)
+# 4) Run through file collection for the given archive archive and append to the archive as tags are found
+# 5) Sort the archive
+# 6) Truncate existing archive if it exists
+# 7) Write the new archive
+
+path=PATH_VIDEO_DATABASE
+archiveFile=path+'/videodb'
+archiveFileLike=archiveFile+'.txt'
+
+#For debugging
+# path='/home/pi/Projects/Python/NewsFeed/Archive'
+# archiveFile=path+'/videodb'
+# archiveFileLike=archiveFile+'.txt'
+
+files = getFiles(archiveFileLike)
+print('There are {count} archive files to process before cleaning'.format(count=len(files)))
+cleanArchive(files, 30)
+files = getFiles(archiveFileLike)
+print('There are {count} archive files to process after cleaning'.format(count=len(files)))
+
+print('archive.py running...')
+
+archiveFileName=ARCHIVEDB_FILENAME
+pathOutputFile=PathHelper.makePathFileName(archiveFileName,path)
+print('pathOutputFile={pathOutputFile}'.format(pathOutputFile=pathOutputFile))
+tokens=["Keane","Jesse","Israel","Hamas"," War ","Iran","Hezzbollah","Gaza","Ukraine"]
+createArchive(pathOutputFile,tokens,files)
+
+hannityFileName=HANNITYARCHIVEDB_FILENAME
+pathOutputFile=PathHelper.makePathFileName(hannityFileName,path)
+print('pathOutputFile={pathOutputFile}'.format(pathOutputFile=pathOutputFile))
+tokens=["Hannity"]
+createArchive(pathOutputFile,tokens,files)
+
+levinFileName=LEVINARCHIVEDB_FILENAME
+pathOutputFile=PathHelper.makePathFileName(levinFileName,path)
+print('pathOutputFile={pathOutputFile}'.format(pathOutputFile=pathOutputFile))
+tokens=["Levin"]
+createArchive(pathOutputFile,tokens,files)
+
+hawleyFileName=HAWLEYARCHIVEDB_FILENAME
+pathOutputFile=PathHelper.makePathFileName(hawleyFileName,path)
+print('pathOutputFile={pathOutputFile}'.format(pathOutputFile=pathOutputFile))
+tokens=["Hawley"]
+createArchive(pathOutputFile,tokens,files)
+
+militaryFileName=MILITARYARCHIVEDB_FILENAME
+pathOutputFile=PathHelper.makePathFileName(militaryFileName,path)
+print('pathOutputFile={pathOutputFile}'.format(pathOutputFile=pathOutputFile))
+tokens=["Keane","Kellogg","Russia","Ukraine","Israel","Korea","Iran","Venezuela","Cuba","China"]
+createArchive(pathOutputFile,tokens,files)
+
+print('archive.py done.')
+
+

environment.py

@@ -0,0 +1,49 @@
+APPEND_SYS_PATH='/home/pi/.kodi/addons/plugin.video.fox.news/resources/lib'
+PATH_VIDEO_DATABASE='/home/pi/.kodi/addons/plugin.video.fox.news/resources/lib/archive'
+USE_ICON_URL=True
+PATH_LOG_FILE="/home/pi/.kodi/temp/MyLog.log"
+FOX_NEWS_URL="https://www.foxnews.com/video"
+FOX_NEWS_US_URL="hhttps://www.foxnews.com/video/topics/us"
+FOX_NEWS_EXCLUSIVE_URL="https://foxnews.com"
+FOX_NEWS_AMERICAS_NEWSROOM_URL="https://www.foxnews.com/video/shows/americas-newsroom"
+FOX_NEWS_OUTNUMBERED_URL="https://www.foxnews.com/video/shows/outnumbered"
+SELECTED_ARCHIVE_URL="https://www.foxnews.com/video/dummy"
+LEVIN_ARCHIVE_URL="https://www.foxnews.com/video/levindummyurl"
+HANNITY_ARCHIVE_URL="https://www.foxnews.com/video/hannitydummyurl"
+HAWLEY_ARCHIVE_URL="https://www.foxnews.com/video/hawleydummyurl"
+MILITARY_ARCHIVE_URL="https://www.foxnews.com/video/militarydummyurl"
+
+#FOX_NEWS_ICON_OF_LAST_RESORT="https://static.foxnews.com/static/orion/styles/img/fox-news/favicons/apple-touch-icon-180x180.png"
+#ENABLE_USE_ICON_OF_LAST_RESORT=False
+
+
+VIDEODB_AMERICAS_NEWSROOM_FILENAME="videodb_americasnewsroom.txt"
+VIDEODB_OUTNUMBERED_FILENAME="videodb_outnumbered.txt"
+VIDEODB_FILENAME="videodb.txt"
+VIDEODB_EXCLUSIVE_FILENAME="videodb_exc.txt"
+VIDEODB_US_FILENAME="videodb_us.txt"
+ARCHIVEDB_FILENAME="archivedb.txt"
+LEVINARCHIVEDB_FILENAME="levindb.txt"
+HANNITYARCHIVEDB_FILENAME="hannitydb.txt"
+HAWLEYARCHIVEDB_FILENAME="hawleydb.txt"
+MILITARYARCHIVEDB_FILENAME="militarydb.txt"
+
+CACHE_EXPIRY_MINS=10
+
+LOG_HTTP_RESPONSES = False
+
+FEED_REJECT_IF_OLDER_THAN_DAYS = 7
+
+class PathHelper:
+    pathChar="/"
+
+    def __init__(self):
+      pass
+
+    @staticmethod
+    def makePathFileName(file,path):
+      while file.endswith(PathHelper.pathChar):
+        file=file[:-1]
+      while path.endswith(PathHelper.pathChar):
+        path=path[:-1]
+      return path+PathHelper.pathChar+file

newsfeed.py

@@ -0,0 +1,544 @@
+import json
+import os
+import webbrowser
+import requests
+import traceback
+import time
+import re
+import glob
+import shutil
+from datetime import timedelta
+from datetime import datetime
+from datetime import timezone
+from environment import *
+from utility import *
+from video import *
+
+class NewsFeed:
+    def __init__(self, pathDb, logger=None):
+        self.pathDb=pathDb
+        self.logger=logger
+
+    @staticmethod
+    def isResourceAvailable(url):
+        try:
+          response=requests.head(url, timeout=2.5)
+          if not response.ok:
+              return False
+          return True
+        except:
+          return False
+
+    def getItemsInAmericasNewsRoomFeed(self,url):
+      now=datetime.now()
+      cachePathFileName=PathHelper.makePathFileName(VIDEODB_AMERICAS_NEWSROOM_FILENAME,self.pathDb)
+      if self.isFeedCacheAvailable(cachePathFileName,CACHE_EXPIRY_MINS):
+        videos=self.readFeedCache(cachePathFileName)
+        if videos is not None:
+            return(videos)
+      sections=Sections()
+      videos = {}
+      httpNetRequest=HttpNetRequest()
+      response=httpNetRequest=httpNetRequest.getHttpNetRequest(url)
+      status=response.status_code
+      searchIndex=0
+      response.close()
+      if status!=200:
+          return None
+      if LOG_HTTP_RESPONSES:
+         self.writeLog(url)
+         self.writeLog(response.text)
+      while -1!= searchIndex:
+        video, searchIndex = sections.getItemsInSection(response.text,"article",searchIndex)
+        if video is not None and not (video.description in videos):
+            videos[video.description]=video
+            video.setFeedTime(DateTimeHelper.applyRelativeTime(now,video.feedTimeOffset))
+      videoList=list(videos.values())
+      videoList=sorted(videoList, key=lambda x:x.getFeedTime(),reverse=False)
+      self.writeFeedCache(cachePathFileName,videoList)
+      return (videoList)
+
+    def getItemsInOutnumberedFeed(self,url):
+      now=datetime.now()
+      cachePathFileName=PathHelper.makePathFileName(VIDEODB_OUTNUMBERED_FILENAME,self.pathDb)
+      if self.isFeedCacheAvailable(cachePathFileName,CACHE_EXPIRY_MINS):
+        videos=self.readFeedCache(cachePathFileName)
+        if videos is not None:
+            return(videos)
+      sections=Sections()
+      videos = {}
+      httpNetRequest=HttpNetRequest()
+      response=httpNetRequest=httpNetRequest.getHttpNetRequest(url)
+      status=response.status_code
+      searchIndex=0
+      response.close()
+      if status!=200:
+          return None
+      if LOG_HTTP_RESPONSES:
+         self.writeLog(url)
+         self.writeLog(response.text)
+      while -1!= searchIndex:
+        video, searchIndex = sections.getItemsInSection(response.text,"article",searchIndex)
+        if video is not None and not (video.description in videos):
+            videos[video.description]=video
+            video.setFeedTime(DateTimeHelper.applyRelativeTime(now,video.feedTimeOffset))
+      videoList=list(videos.values())
+      videoList=sorted(videoList, key=lambda x:x.getFeedTime(),reverse=True)
+      self.writeFeedCache(cachePathFileName,videoList)
+      return (videoList)
+
+    def getItemsInFeed(self,url):
+      now=datetime.now()
+      cachePathFileName=PathHelper.makePathFileName(VIDEODB_FILENAME,self.pathDb)
+      if self.isFeedCacheAvailable(cachePathFileName,CACHE_EXPIRY_MINS):
+        self.writeLog(f"Loading videos from cache {cachePathFileName}")
+        videos=self.readFeedCache(cachePathFileName)
+        if videos is not None:
+            return(videos)
+      sections=Sections()
+      videos = {}
+      httpNetRequest=HttpNetRequest()
+      self.writeLog(f"Loading videos from {url}")
+      response=httpNetRequest=httpNetRequest.getHttpNetRequest(url)
+      status=response.status_code
+      searchIndex=0
+      response.close()
+      if status!=200:
+          return None
+      if LOG_HTTP_RESPONSES:
+         self.writeLog(url)
+         self.writeLog(response.text)
+      while -1!= searchIndex:
+        video, searchIndex= sections.getItemsInSection(response.text,"article",searchIndex)
+        if video is not None and not (video.description in videos):
+            videos[video.description]=video
+            video.setFeedTime(DateTimeHelper.applyRelativeTime(now,video.feedTimeOffset))
+#      videoList=list(videos.values())
+      videoList=self.filterFeedMaxDays(list(videos.values()),FEED_REJECT_IF_OLDER_THAN_DAYS)
+      videoList=sorted(videoList, key=lambda x:x.getFeedTime(),reverse=True)
+      self.writeFeedCache(cachePathFileName,videoList)
+      return (videoList)
+
+    def filterFeedMaxDays(self, videos, days):
+       now = datetime.now()
+       filteredList=[]
+       for video in videos:
+          delta = now - video.getFeedTime()
+          if delta.days <= days:
+             message = f"INCL. days={delta.days},feed time={video.getFeedTime()} feed time offset (strPublication)=:'{video.feedTimeOffset}',  description={video.description}"
+             self.writeLog(message)
+             filteredList.insert(0,video)
+          else:
+             message = f"EXCL. days={delta.days},feed time={video.getFeedTime()} feed time offset (strPublication)=:'{video.feedTimeOffset}',  description={video.description}"
+             self.writeLog(message)
+       return filteredList
+
+    def getUSItemsInFeed(self,url):
+      now=datetime.now()
+      cachePathFileName=PathHelper.makePathFileName(VIDEODB_US_FILENAME,self.pathDb)
+      if self.isFeedCacheAvailable(cachePathFileName,CACHE_EXPIRY_MINS):
+        videos=self.readFeedCache(cachePathFileName)
+        if videos is not None:
+            return(videos)
+      sections=Sections()
+      videos = {}
+      httpNetRequest=HttpNetRequest()
+      response=httpNetRequest.getHttpNetRequest(url)
+      status=response.status_code
+      searchIndex=0
+      response.close()
+      if status!=200:
+          return None
+      if LOG_HTTP_RESPONSES:
+         self.writeLog(url)
+         self.writeLog(response.text)
+      while -1!= searchIndex:
+        videoId, searchIndex = sections.getVideoIdInSection(response.text,"article",searchIndex)
+        if videoId is None:
+            continue
+        url='https://video.foxnews.com/v/'+videoId
+        httpNetRequest=HttpNetRequest()
+        innerResponse=httpNetRequest.getHttpNetRequest(url)
+        status=innerResponse.status_code
+        innerResponse.close()
+        if status!=200:
+            continue
+        video=sections.getVideoContentInSection(innerResponse.text)
+        if video is not None and not (video.description in videos):
+            videos[video.description]=video
+            video.setFeedTime(DateTimeHelper.applyRelativeTime(now,video.feedTimeOffset))
+      videoList=list(videos.values())
+      videoList=sorted(videoList, key=lambda x:x.getFeedTime(),reverse=True)
+      self.writeFeedCache(cachePathFileName,videoList)
+      return (videoList)
+
+    def getExclusiveItemsInFeed(self,url):
+      now=datetime.now()
+      cachePathFileName=PathHelper.makePathFileName(VIDEODB_EXCLUSIVE_FILENAME,self.pathDb)
+      if self.isFeedCacheAvailable(cachePathFileName,CACHE_EXPIRY_MINS):
+        videos=self.readFeedCache(cachePathFileName)
+        if videos is not None:
+            return(videos)
+      sections=Sections()
+      videos = {}
+      httpNetRequest=HttpNetRequest()
+      response=httpNetRequest.getHttpNetRequest(url)
+      status=response.status_code
+      searchIndex=0
+      response.close()
+      if status!=200:
+          return None
+      if LOG_HTTP_RESPONSES:
+         self.writeLog(url)
+         self.writeLog(response.Text)
+      while -1!= searchIndex:
+        videoId, searchIndex = sections.getVideoIdInSection(response.text,"article",searchIndex)
+        if videoId is None:
+            continue
+        url='https://video.foxnews.com/v/'+videoId
+        httpNetRequest=HttpNetRequest()
+        innerResponse=httpNetRequest.getHttpNetRequest(url)
+        status=innerResponse.status_code
+        innerResponse.close()
+        if status!=200:
+            continue
+        video=sections.getVideoContentInSection(innerResponse.text)
+        if video is not None and not (video.description in videos):
+            videos[video.description]=video
+            video.setFeedTime(DateTimeHelper.applyRelativeTime(now,video.feedTimeOffset))
+      videoList=list(videos.values())
+      videoList=sorted(videoList, key=lambda x:x.getFeedTime(),reverse=True)
+      self.writeFeedCache(cachePathFileName,videoList)
+      return (videoList)
+
+    def getItemsInArchiveFeed(self,url,archiveDbFileName):
+      cachePathFileName=PathHelper.makePathFileName(archiveDbFileName,self.pathDb)
+      videos=self.readFeedCache(cachePathFileName)
+      if videos is not None:
+        return(videos)
+      return(None)
+
+    def readFeedCache(self,pathFileName):
+        try:
+          videos=[]  
+          with open(pathFileName,"r",encoding='utf-8') as inputStream:
+            for line in inputStream:
+                video=Video.fromString(line)
+                videos.append(video)
+          inputStream.close()
+          return(videos)
+        except:
+          self.writeLog(traceback.format_exc())
+          return(None)
+
+    def writeFeedCache(self,pathFileName,videos):
+        try:
+          with open(pathFileName,"w",encoding='utf-8') as outputStream:
+            for video in videos:
+                outputStream.write(video.toString()+"\n")
+          outputStream.close()
+          return(videos)
+        except:
+          self.writeLog(traceback.format_exc())
+          return(videos)
+
+    def isFeedCacheAvailable(self,pathFileName,expireMinutes):
+        try:
+          self.writeLog('Inspecting cache file {pathFileName}'.format(pathFileName=pathFileName))
+          if not os.path.isfile(pathFileName):
+            return(False)
+          modifiedTime=os.path.getmtime(pathFileName)
+          convertTime=time.localtime(modifiedTime)
+          formatTime=time.strftime('%d%m%Y %H:%M:%S',convertTime)
+          fileDateTime=DateTimeHelper.strptime(formatTime,'%d%m%Y %H:%M:%S')
+          currentTime=datetime.now()
+          timedelta=currentTime-fileDateTime
+          hours, hremainder = divmod(timedelta.seconds,3600)
+          minutes, mremainder = divmod(timedelta.seconds,60)
+          self.writeLog('file is  = "{age}" hours old'.format(age=hours))
+          self.writeLog('file is  = "{age}" minutes old'.format(age=minutes))
+          if hours > 1 or minutes > expireMinutes:
+            self.archiveFile(pathFileName)
+            return(False)
+          return (True)
+        except:
+          self.writeLog(traceback.format_exc());
+          return(False)
+    
+    def archiveFile(self, pathFileName):
+        if not os.path.isfile(pathFileName):
+          return(False)
+        archiveFile=StringHelper.betweenString(pathFileName, None, '.txt')
+        archiveFileLike=archiveFile+'.txt.*'
+        files = glob.glob(archiveFileLike)
+        index=len(files)+1
+        archiveFileName=archiveFile+'.txt.'+str(index)
+        print('archiveFile:  Copying "{pathFileName}" to "{archiveFileName}".'.format(pathFileName=pathFileName,archiveFileName=archiveFileName))
+        shutil.copy(pathFileName,archiveFileName)
+        os.remove(pathFileName)
+        return(True)
+        
+    def writeLog(self,message):
+        if self.logger is not None:
+            self.logger.write(message)
+        else:
+            print(message)
+
+class Sections:
+    def __init__(self):
+      self.dummy=None
+
+    def getItemsInSection(self, strInput, sectionName, searchIndex):
+        video=None
+        startSection='<'+sectionName
+        endSection='</'+sectionName
+
+        startIndex=strInput.find(startSection,searchIndex)
+        if -1 == startIndex:
+            searchIndex=-1
+            return video, searchIndex
+
+        endIndex=strInput.find(endSection,startIndex)
+        if -1 == endIndex:
+            searchIndex=-1
+            return video, searchIndex
+
+        searchIndex=endIndex+len(endSection)
+        strContainingString=strInput[startIndex:endIndex+1+len(endSection)]
+
+        if not strContainingString or strContainingString=="":
+            return video, searchIndex
+
+        indexPreview=strContainingString.find("preview=\"")
+        if -1 == indexPreview:
+            return video, searchIndex
+        previewUrl=strContainingString[indexPreview:]
+        previewUrl=self.betweenString(previewUrl,'"','"')
+        if "tokenvod" in previewUrl:
+            return video, searchIndex
+
+        indexDescription=strContainingString.index("alt=\"")
+        description=strContainingString[indexDescription:]
+        description=self.betweenString(description,'"','"')
+        description=self.removeHtml(description)
+        description=description.replace("- Fox News","")
+        if "vod.foxbusiness" in description:
+            return video, searchIndex
+        indexDuration=strContainingString.index("<div class=\"duration\">")
+        if -1 != indexDuration:
+            strDuration=strContainingString[indexDuration:]
+            strDuration=self.betweenString(strDuration,">","<")
+            description=description+" - "+strDuration
+        indexPublication=strContainingString.index("<div class=\"pub-date\">")
+        if -1 != indexPublication:
+            strPublication=strContainingString[indexPublication:]
+            strPublication=self.betweenString(strPublication,"<time>","</time>")
+            description=description+" ("+strPublication+")"
+        icon=None
+        indexIcon=strContainingString.index("srcset=")
+        if -1 != indexIcon:
+            icon=strContainingString[indexIcon:]
+            icon=self.betweenString(icon,"\"","\"")
+            splits=icon.split(',')
+            icon=self.betweenString(splits[len(splits)-1],None,'?')
+            icon=icon.strip()
+        description = description.strip()
+        video=Video(description,previewUrl,icon)
+        video.feedTimeOffset=strPublication
+        return video, searchIndex
+
+    def getVideoIdInSection(self, strInput, sectionName, searchIndex):
+        video=None
+        startSection='<'+sectionName
+        endSection='</'+sectionName
+
+        startIndex=strInput.find(startSection,searchIndex)
+        if -1 == startIndex:
+            searchIndex=-1
+            return video, searchIndex
+
+        endIndex=strInput.find(endSection,startIndex)
+        if -1 == endIndex:
+            searchIndex=-1
+            return video, searchIndex
+
+        searchIndex=endIndex+len(endSection)
+        strContainingString=strInput[startIndex:endIndex+1+len(endSection)]
+        if not strContainingString or strContainingString=="":
+            return video, searchIndex
+        indexVideoId=strContainingString.find("data-video-id")
+        if -1 ==indexVideoId:
+            return video, searchIndex
+        videoId=strContainingString[indexVideoId:]
+        videoId=self.betweenString(videoId,"\"","\"")
+        return videoId, searchIndex
+
+    def getVideoContentInSection(self, strInput):
+        video=None
+        searchItem="\"contentUrl\":"
+        indexContentUrl=strInput.find(searchItem)
+        if -1 == indexContentUrl:
+            return None
+        strContentUrl=strInput[indexContentUrl+len(searchItem):]
+        strContentUrl=self.betweenString(strContentUrl,"\"","\"")
+        strContentUrl=strContentUrl.strip()
+
+        searchItem="\"description\":"
+        indexDescription=strInput.find(searchItem)
+        if -1 == indexDescription:
+            return None
+        strDescription=strInput[indexDescription+len(searchItem):]
+        strDescription=self.betweenString(strDescription,"\"","\"")
+        strDescription=strDescription.strip()
+
+        searchItem="\"thumbnailUrl\":"
+        indexIcon=strInput.find(searchItem)
+        if -1 == indexIcon:
+            return None
+        strIcon=strInput[indexIcon+len(searchItem):]
+        strIcon=self.betweenString(strIcon,"\"","\"")
+        strIcon=strIcon.strip()
+
+        searchItem="\"duration\""
+        indexDuration=strInput.find(searchItem)
+        if -1 != indexDuration:
+            strDuration=strInput[indexDuration+len(searchItem):]
+            strDuration=self.betweenString(strDuration,"\"","\"")
+            strDuration=strDuration.strip()
+            minutes, seconds = parseDuration(strDuration)
+            if None!=minutes and None!=seconds:
+                strDescription=strDescription+" - "+minutes+":"+seconds
+        strDescription = strDescription.strip()
+        video=Video(strDescription,strContentUrl,strIcon)
+        return video
+
+    def betweenString(self, strItem, strBegin, strEnd ):
+        return StringHelper.betweenString(strItem, strBegin, strEnd)
+
+    def removeHtml(self,strItem):
+        if strItem is None:
+            return None
+        codes={"&#x27;","&#187;"}
+        for code in codes:
+            strItem=strItem.replace(code,"'")
+        strItem=strItem.replace("&amp;","&")
+        strItem=strItem.replace("&#x2018;","'")
+        strItem=strItem.replace("&#x2019;","'")
+        strItem=strItem.replace("&#x2014;","-")
+        strItem=strItem.replace("???","'")
+        return strItem
+
+def pad(str,filler,length):
+    stringLength=len(str)
+    sb=""
+    if stringLength>=length:
+        return str
+    while stringLength < length:
+        sb=sb+filler
+        stringLength=stringLength+1
+    return sb+str
+
+def parseDuration(strDuration):
+    expression=re.compile(r"\d+")
+    result=expression.findall(strDuration)
+    if 2!=len(result):
+        return None, None
+    return pad(result[0],'0',2), pad(result[1],'0',2)
+
+
+# DON'T LEAVE ANYTHING OPEN BELOW THIS LINE BECAUSE THIS FILE IS IMPORTED BY OTHER MODULES AND ANY CODE NOT IN A CLASS WILL BE RUN
+
+#print(FOX_NEWS_URL)
+# pathFileName='/home/pi/.kodi/addons/plugin.video.fox.news/resources/lib/videodb.txt'
+# newsFeed=NewsFeed('/home/pi/.kodi/addons/plugin.video.fox.news/resources/lib/')
+# newsFeed.ArchiveFile(pathFileName)
+
+
+# pathFileName='/home/pi/.kodi/addons/plugin.video.fox.news/resources/lib/videodb.txt'
+# modifiedTime=os.path.getmtime(pathFileName)
+# convertTime=time.localtime(modifiedTime)
+# formatTime=time.strftime('%d%m%Y %H:%M:%S',convertTime)
+# fileDateTime=DateTimeHelper.strptime(formatTime)
+
+#fileDateTime=datetime.strptime(formatTime,'%d%m%Y %H:%M:%S')
+#fileDateTime2=datetime(*(time.strptime(formatTime,'%d%m%Y %H:%M:%S')[0:6]))
+#currentTime=datetime.now()
+
+#Test the main feed
+# newsFeed=NewsFeed('/home/pi/Projects/Python/NewsFeed/')
+# newsFeed=NewsFeed('/home/pi/.kodi/addons/plugin.video.fox.news/resources/lib/videodb.txt')
+# newsFeed=NewsFeed(PATH_VIDEO_DATABASE, myLog())
+# newsFeed=NewsFeed('/home/pi/Projects/Python/NewsFeed/', myLog())
+# videos=newsFeed.getItemsInFeed(FOX_NEWS_URL)
+# for video in videos:
+#  if(video.description.startswith("Martha")):
+#     print(f"Description={video.description}")
+#     print(f"Url={video.url}")
+#     print(f"getTimestamp={video.getTimestamp().toStringMonthDay()}")
+#     print(f"getFeedTimeOffset={video.getFeedTimeOffset()}")
+#     print(f"getFeedTime={video.getFeedTime()}")
+#     print(f"daysOld={(datetime.now()-video.getFeedTime()).days}")
+#     print(' ')
+
+# pull the time out of the description and subtract it from the time we scanned the feed.
+# the result will be the time of the article..use this to sort on.
+# (i.e.) FeedTime:02/03/2023 12:00:00  Article Time:2 hours ago  Real time:10:00:00
+
+
+#Test the exclusive items feed
+#newsFeed=NewsFeed('/home/pi/.kodi/addons/plugin.video.fox.news/resources/lib/')
+#videos=newsFeed.getExclusiveItemsInFeed("https://www.foxnews.com")
+# for video in videos:
+#     print(video.description)
+
+
+# Test the U.S. Feed
+# newsFeed=NewsFeed('/home/pi/.kodi/addons/plugin.video.fox.news/resources/lib/')
+# videos=newsFeed.getUSItemsInFeed("https://www.foxnews.com/video/topics/us")
+# for video in videos:
+#     print(video.description)
+
+# Test the America's NewsRoom Feed
+# newsFeed=NewsFeed('/home/pi/.kodi/addons/plugin.video.fox.news/resources/lib/')
+# videos=newsFeed.getItemsInAmericasNewsRoomFeed("https://www.foxnews.com/video/shows/americas-newsroom")
+# print('got {count} videos for America''s Newsroom'.format(count=len(videos)))
+# for video in videos:
+#     print(video.description)
+#     print(video.url)
+
+# Test the Outnumbered Feed
+# newsFeed=NewsFeed('/home/pi/.kodi/addons/plugin.video.fox.news/resources/lib/')
+# videos=newsFeed.getItemsInOutnumbereFeed("https://www.foxnews.com/video/shows/outnumbered")
+# print('got {count} videos for Outnumbered'.format(count=len(videos)))
+# for video in videos:
+#     print(video.description)
+#     print(video.url)
+
+#minutes, seconds = parseDuration('PT24M5S')
+#print('Duration is {minutes}:{seconds}'.format(minutes=minutes,seconds=seconds))
+
+# isoDate="2022-10-27T10:24:11Z".replace("Z","+00:00")
+# articleTime=datetime.datetime.fromisoformat(isoDate)
+# print('time:{time}'.format(time=articleTime))
+# currentTime=Date.getCurrentTime()
+# print('time:{time}'.format(time=currentTime))
+# days, hours, minutes, seconds=Date.deltaTime(articleTime,currentTime)
+# print('elapsed time {days} days, {hours} hours, {minutes} minutes, {seconds} seconds'.format(days=days,hours=hours,minutes=minutes,seconds=seconds))
+
+# currentTime2=Date.getCurrentTime()
+# strCurrentTime2=str(currentTime2)
+# currentTime2=datetime.datetime.fromisoformat(strCurrentTime2)
+# days, hours, minutes, seconds=Date.deltaTime(currentTime2,currentTime)
+# print('elapsed time {days} days, {hours} hours, {minutes} minutes, {seconds} seconds'.format(days=days,hours=hours,minutes=minutes,seconds=seconds))
+
+# dateList=[]
+
+# currentDate=Date()
+# dateList.append(currentDate)
+# currentDate2=Date()
+# dateList.append(currentDate2)
+
+# dateList.sort(key=lambda x:x.toString())
+# for date in dateList:
+#     print(date.toString())
+# #print(dateList)

scraper.py

@@ -0,0 +1,183 @@
+# -*- coding: utf-8 -*-
+# Fox News Kodi Video Addon
+#
+
+APPEND_SYS_PATH='/home/pi/.kodi/addons/plugin.video.fox.news/resources/lib'
+
+import os
+import sys
+import traceback
+sys.path.append(APPEND_SYS_PATH)
+from t1mlib import t1mAddon
+import json
+import re
+import xbmc
+import xbmcplugin
+import xbmcgui
+import xbmcvfs
+import xbmc
+import xbmcaddon
+import html.parser
+import sys
+import datetime
+import time
+import random
+import requests
+import sqlite3
+from newsfeed import NewsFeed
+from functools import reduce
+from simplecache import SimpleCache
+from environment import *
+from utility import *
+
+#class myLog():
+#    def __init__(self):
+#        self._file = open(PATH_LOG_FILE,"w",encoding='utf-8')
+#
+#    def write(self,item):
+#        self._file.write(item)
+#        self._file.write("\n")
+#        self._file.flush()
+
+class myAddon(t1mAddon):
+
+    def __init__(self, aname):
+        t1mAddon.__init__(self, aname)
+        self._logfile = myLog()
+        self._cache = SimpleCache()
+        self._pDialog = xbmcgui.DialogProgressBG()
+
+    def getAddonMenu(self,url,ilist):
+        try:
+            self._logfile.write('getAddonMenu')
+            ilist = self.addMenuItem("Latest Fox News Featured Clips",'GE', ilist, FOX_NEWS_URL, self.addonIcon, self.addonFanart, {}, isFolder=True)
+            ilist = self.addMenuItem("Fox News Outnumbered Clips",'GE', ilist, FOX_NEWS_OUTNUMBERED_URL, self.addonIcon, self.addonFanart, {}, isFolder=True)
+            ilist = self.addMenuItem("Selected Archive Clips",'GE', ilist, SELECTED_ARCHIVE_URL, self.addonIcon, self.addonFanart, {}, isFolder=True)
+            ilist = self.addMenuItem("Mark Levin Archive Clips",'GE', ilist, LEVIN_ARCHIVE_URL, self.addonIcon, self.addonFanart, {}, isFolder=True)
+            ilist = self.addMenuItem("Sean Hannity Archive Clips",'GE', ilist, HANNITY_ARCHIVE_URL, self.addonIcon, self.addonFanart, {}, isFolder=True)
+            ilist = self.addMenuItem("Josh Hawley Archive Clips",'GE', ilist, HAWLEY_ARCHIVE_URL, self.addonIcon, self.addonFanart, {}, isFolder=True)
+            ilist = self.addMenuItem("Military Archive Clips",'GE', ilist, MILITARY_ARCHIVE_URL, self.addonIcon, self.addonFanart, {}, isFolder=True)
+            return(ilist)
+        except:
+            self._logfile.write(traceback.format_exc())
+            raise
+        finally:
+            return(ilist)
+
+    def getAddonEpisodes(self,url,ilist):
+        try:
+            if url == FOX_NEWS_URL:
+              self._logfile.write('getAddonEpisodes url={url}'.format(url=url))
+              self._pDialog.create('Retrieving News Articles...')
+              newsFeed=NewsFeed(PATH_VIDEO_DATABASE, self._logfile)
+              videos=newsFeed.getItemsInFeed(url)
+              for video in videos:
+                if USE_ICON_URL:
+                  ilist = self.addMenuItem(video.description,'GV', ilist, video.url, video.icon, self.addonFanart, {}, isFolder=False)
+                else:
+                  ilist = self.addMenuItem(video.description,'GV', ilist, video.url, self.addonIcon, self.addonFanart, {}, isFolder=False)
+              self._logfile.write('Processed {articles} articles.'.format(articles=len(videos)))
+            elif url == SELECTED_ARCHIVE_URL:
+              self._logfile.write('getAddonEpisodes url={url}'.format(url=url))
+              self._pDialog.create('Retrieving News Articles...')
+              newsFeed=NewsFeed(PATH_VIDEO_DATABASE, self._logfile)
+              videos=newsFeed.getItemsInArchiveFeed(url,ARCHIVEDB_FILENAME)
+              for video in videos:
+                if USE_ICON_URL:
+                  ilist = self.addMenuItem(video.description,'GV', ilist, video.url, video.icon, self.addonFanart, {}, isFolder=False)
+                else:
+                  ilist = self.addMenuItem(video.description,'GV', ilist, video.url, self.addonIcon, self.addonFanart, {}, isFolder=False)
+              self._logfile.write('Processed {articles} articles.'.format(articles=len(videos)))              
+            elif url == LEVIN_ARCHIVE_URL:
+              self._logfile.write('getAddonEpisodes url={url}'.format(url=url))
+              self._pDialog.create('Retrieving News Articles...')
+              newsFeed=NewsFeed(PATH_VIDEO_DATABASE, self._logfile)
+              videos=newsFeed.getItemsInArchiveFeed(url,LEVINARCHIVEDB_FILENAME)
+              for video in videos:
+                if USE_ICON_URL:
+                  ilist = self.addMenuItem(video.description,'GV', ilist, video.url, video.icon, self.addonFanart, {}, isFolder=False)
+                else:
+                  ilist = self.addMenuItem(video.description,'GV', ilist, video.url, self.addonIcon, self.addonFanart, {}, isFolder=False)
+              self._logfile.write('Processed {articles} articles.'.format(articles=len(videos)))              
+            elif url == HANNITY_ARCHIVE_URL:
+              self._logfile.write('getAddonEpisodes url={url}'.format(url=url))
+              self._pDialog.create('Retrieving News Articles...')
+              newsFeed=NewsFeed(PATH_VIDEO_DATABASE, self._logfile)
+              videos=newsFeed.getItemsInArchiveFeed(url,HANNITYARCHIVEDB_FILENAME)
+              for video in videos:
+                if USE_ICON_URL:
+                  ilist = self.addMenuItem(video.description,'GV', ilist, video.url, video.icon, self.addonFanart, {}, isFolder=False)
+                else:
+                  ilist = self.addMenuItem(video.description,'GV', ilist, video.url, self.addonIcon, self.addonFanart, {}, isFolder=False)
+              self._logfile.write('Processed {articles} articles.'.format(articles=len(videos)))              
+            elif url == HAWLEY_ARCHIVE_URL:
+              self._logfile.write('getAddonEpisodes url={url}'.format(url=url))
+              self._pDialog.create('Retrieving News Articles...')
+              newsFeed=NewsFeed(PATH_VIDEO_DATABASE, self._logfile)
+              videos=newsFeed.getItemsInArchiveFeed(url,HAWLEYARCHIVEDB_FILENAME)
+              for video in videos:
+                if USE_ICON_URL:
+                  ilist = self.addMenuItem(video.description,'GV', ilist, video.url, video.icon, self.addonFanart, {}, isFolder=False)
+                else:
+                  ilist = self.addMenuItem(video.description,'GV', ilist, video.url, self.addonIcon, self.addonFanart, {}, isFolder=False)
+              self._logfile.write('Processed {articles} articles.'.format(articles=len(videos)))              
+            elif url == MILITARY_ARCHIVE_URL:
+              self._logfile.write('getAddonEpisodes url={url}'.format(url=url))
+              self._pDialog.create('Retrieving News Articles...')
+              newsFeed=NewsFeed(PATH_VIDEO_DATABASE, self._logfile)
+              videos=newsFeed.getItemsInArchiveFeed(url,MILITARYARCHIVEDB_FILENAME)
+              for video in videos:
+                if USE_ICON_URL:
+                  ilist = self.addMenuItem(video.description,'GV', ilist, video.url, video.icon, self.addonFanart, {}, isFolder=False)
+                else:
+                  ilist = self.addMenuItem(video.description,'GV', ilist, video.url, self.addonIcon, self.addonFanart, {}, isFolder=False)
+              self._logfile.write('Processed {articles} articles.'.format(articles=len(videos)))              
+            elif url== FOX_NEWS_OUTNUMBERED_URL:
+              self._logfile.write('getAddonEpisodes url={url}'.format(url=url))
+              self._pDialog.create('Retrieving Outnumbered Articles...')
+              newsFeed=NewsFeed(PATH_VIDEO_DATABASE, self._logfile)
+              videos=newsFeed.getItemsInOutnumberedFeed(url)
+              for video in videos:
+                if USE_ICON_URL:
+                  ilist = self.addMenuItem(video.description,'GV', ilist, video.url, video.icon, self.addonFanart, {}, isFolder=False)
+                else:
+                  ilist = self.addMenuItem(video.description,'GV', ilist, video.url, self.addonIcon, self.addonFanart, {}, isFolder=False)
+              self._logfile.write('Processed {articles} articles.'.format(articles=len(videos)))
+        except Exception as exception:
+            self._logfile.write('Exception:{exception}'.format(exception=exception))
+            self._logfile.write(traceback.format_exc())
+            raise
+        finally:
+            self._pDialog.close()
+            return(ilist)
+
+    def getAddonMovies(self,url,ilist):
+        try:
+            self._logfile.write('getAddonMovies url={url}'.format(url=url))
+            self._pDialog.create('getAddonMovies...')
+            self._pDialog.update(0,message='getAddonMovies...')
+            self._pDialog.close()
+            return(ilist)
+        except Exception as exception:
+            self._logfile.write('Exception:{exception}'.format(exception=exception))
+            self._logfile.write(traceback.format_exc())
+            raise
+        finally:
+            self._pDialog.close()
+            return(ilist)
+
+    def getAddonShows(self,url,ilist):
+        try:
+            self._logfile.write('getAddonShows')
+            self._pDialog.create('getAddonShows...')
+            self._pDialog.update(0,message='getAddonShows...')
+            self._pDialog.close()
+            return ilist
+        except Exception as exception:
+            self._logfile.write('Exception:{exception}'.format(exception=exception))
+            self._logfile.write(traceback.format_exc())
+            raise
+        finally:
+            self._pDialog.close()
+            return(ilist)
+

simplecache.py

@@ -0,0 +1,302 @@
+# -*- coding: utf-8 -*-
+# Fox News Kodi Video Addon
+#
+import os
+import sys
+import traceback
+from t1mlib import t1mAddon
+import json
+import re
+import xbmc
+import xbmcplugin
+import xbmcgui
+import xbmcvfs
+import xbmc
+import xbmcaddon
+import html.parser
+import sys
+import datetime
+import time
+import random
+import requests
+import sqlite3
+
+ADDON_ID = "plugin.video.fox.news"
+
+class SimpleCache(object):
+    '''simple stateless caching system for Kodi'''
+    enable_mem_cache = True
+    global_checksum = None
+    _exit = False
+    _auto_clean_interval = datetime.timedelta(hours=4)
+    _win = None
+    _busy_tasks = []
+    _database = None
+
+    def __init__(self):
+        '''Initialize our caching class'''
+        self._win = xbmcgui.Window(10000)
+        self._monitor = xbmc.Monitor()
+        self.check_cleanup()
+        self._log_msg("Initialized")
+
+    def close(self):
+        '''tell any tasks to stop immediately (as we can be called multithreaded) and cleanup objects'''
+        self._exit = True
+        # wait for all tasks to complete
+        while self._busy_tasks and not self._monitor.abortRequested():
+            xbmc.sleep(25)
+        del self._win
+        del self._monitor
+        self._log_msg("Closed")
+
+    def __del__(self):
+        '''make sure close is called'''
+        if not self._exit:
+            self.close()
+
+    def get(self, endpoint, checksum=""):
+        '''
+            get object from cache and return the results
+            endpoint: the (unique) name of the cache object as reference
+            checkum: optional argument to check if the checksum in the cacheobject matches the checkum provided
+        '''
+        checksum = self._get_checksum(checksum)
+        cur_time = self._get_timestamp(datetime.datetime.now())
+        result = None
+        # 1: try memory cache first
+        if self.enable_mem_cache:
+            result = self._get_mem_cache(endpoint, checksum, cur_time)
+
+        # 2: fallback to _database cache
+        if result is None:
+            result = self._get_db_cache(endpoint, checksum, cur_time)
+
+        return result
+
+    def set(self, endpoint, data, checksum="", expiration=datetime.timedelta(days=30)):
+        '''
+            set data in cache
+        '''
+        task_name = "set.%s" % endpoint
+        self._busy_tasks.append(task_name)
+        checksum = self._get_checksum(checksum)
+        expires = self._get_timestamp(datetime.datetime.now() + expiration)
+
+        # memory cache: write to window property
+        if self.enable_mem_cache and not self._exit:
+            self._set_mem_cache(endpoint, checksum, expires, data)
+
+        # db cache
+        if not self._exit:
+            self._set_db_cache(endpoint, checksum, expires, data)
+
+        # remove this task from list
+        self._busy_tasks.remove(task_name)
+
+    def check_cleanup(self):
+        '''check if cleanup is needed - public method, may be called by calling addon'''
+        cur_time = datetime.datetime.now()
+        lastexecuted = self._win.getProperty("simplecache.clean.lastexecuted")
+        if not lastexecuted:
+            self._win.setProperty("simplecache.clean.lastexecuted", repr(cur_time))
+        elif (eval(lastexecuted) + self._auto_clean_interval) < cur_time:
+            # cleanup needed...
+            self._do_cleanup()
+
+    def _get_mem_cache(self, endpoint, checksum, cur_time):
+        '''
+            get cache data from memory cache
+            we use window properties because we need to be stateless
+        '''
+        result = None
+        cachedata = self._win.getProperty(endpoint)
+
+        if cachedata:
+            cachedata = eval(cachedata)
+            if cachedata[0] > cur_time:
+                if not checksum or checksum == cachedata[2]:
+                    result = cachedata[1]
+        return result
+
+    def _set_mem_cache(self, endpoint, checksum, expires, data):
+        '''
+            window property cache as alternative for memory cache
+            usefull for (stateless) plugins
+        '''
+        cachedata = (expires, data, checksum)
+        cachedata_str = repr(cachedata)
+        self._win.setProperty(endpoint, cachedata_str)
+
+
+    def _get_db_cache(self, endpoint, checksum, cur_time):
+        '''get cache data from sqllite _database'''
+        result = None
+        query = "SELECT expires, data, checksum FROM simplecache WHERE id = ?"
+        cache_data = self._execute_sql(query, (endpoint,))
+        if cache_data:
+            cache_data = cache_data.fetchone()
+            if cache_data and cache_data[0] > cur_time:
+                if not checksum or cache_data[2] == checksum:
+                    result = eval(cache_data[1])
+                    # also set result in memory cache for further access
+                    if self.enable_mem_cache:
+                        self._set_mem_cache(endpoint, checksum, cache_data[0], result)
+        return result
+
+    def _set_db_cache(self, endpoint, checksum, expires, data):
+        ''' store cache data in _database '''
+        query = "INSERT OR REPLACE INTO simplecache( id, expires, data, checksum) VALUES (?, ?, ?, ?)"
+        data = repr(data)
+        self._execute_sql(query, (endpoint, expires, data, checksum))
+
+    def _do_cleanup(self):
+        '''perform cleanup task'''
+        if self._exit or self._monitor.abortRequested():
+            return
+        self._busy_tasks.append(__name__)
+        cur_time = datetime.datetime.now()
+        cur_timestamp = self._get_timestamp(cur_time)
+        self._log_msg("Running cleanup...")
+        if self._win.getProperty("simplecachecleanbusy"):
+            return
+        self._win.setProperty("simplecachecleanbusy", "busy")
+
+        query = "SELECT id, expires FROM simplecache"
+        for cache_data in self._execute_sql(query).fetchall():
+            cache_id = cache_data[0]
+            cache_expires = cache_data[1]
+
+            if self._exit or self._monitor.abortRequested():
+                return
+
+            # always cleanup all memory objects on each interval
+            self._win.clearProperty(cache_id)
+
+            # clean up db cache object only if expired
+            if cache_expires < cur_timestamp:
+                query = 'DELETE FROM simplecache WHERE id = ?'
+                self._execute_sql(query, (cache_id,))
+                self._log_msg("delete from db %s" % cache_id)
+
+        # compact db
+        self._execute_sql("VACUUM")
+
+        # remove task from list
+        self._busy_tasks.remove(__name__)
+        self._win.setProperty("simplecache.clean.lastexecuted", repr(cur_time))
+        self._win.clearProperty("simplecachecleanbusy")
+        self._log_msg("Auto cleanup done")
+
+    def _get_database(self):
+        '''get reference to our sqllite _database - performs basic integrity check'''
+        addon = xbmcaddon.Addon(ADDON_ID)
+        dbpath = addon.getAddonInfo('profile')
+        dbfile = xbmcvfs.translatePath("%s/simplecache.db" % dbpath)
+
+        if not xbmcvfs.exists(dbpath):
+            xbmcvfs.mkdirs(dbpath)
+        del addon
+        try:
+            connection = sqlite3.connect(dbfile, timeout=30, isolation_level=None)
+            connection.execute('SELECT * FROM simplecache LIMIT 1')
+            return connection
+        except Exception as error:
+            # our _database is corrupt or doesn't exist yet, we simply try to recreate it
+            if xbmcvfs.exists(dbfile):
+                xbmcvfs.delete(dbfile)
+            try:
+                connection = sqlite3.connect(dbfile, timeout=30, isolation_level=None)
+                connection.execute(
+                    """CREATE TABLE IF NOT EXISTS simplecache(
+                    id TEXT UNIQUE, expires INTEGER, data TEXT, checksum INTEGER)""")
+                return connection
+            except Exception as error:
+                self._log_msg("Exception while initializing _database: %s" % str(error), xbmc.LOGWARNING)
+                self.close()
+                return None
+
+    def _execute_sql(self, query, data=None):
+        '''little wrapper around execute and executemany to just retry a db command if db is locked'''
+        retries = 0
+        result = None
+        error = None
+        # always use new db object because we need to be sure that data is available for other simplecache instances
+        with self._get_database() as _database:
+            while not retries == 10 and not self._monitor.abortRequested():
+                if self._exit:
+                    return None
+                try:
+                    if isinstance(data, list):
+                        result = _database.executemany(query, data)
+                    elif data:
+                        result = _database.execute(query, data)
+                    else:
+                        result = _database.execute(query)
+                    return result
+                except sqlite3.OperationalError as error:
+                    if "_database is locked" in error:
+                        self._log_msg("retrying DB commit...")
+                        retries += 1
+                        self._monitor.waitForAbort(0.5)
+                    else:
+                        break
+                except Exception as error:
+                    break
+            self._log_msg("_database ERROR ! -- %s" % str(error), xbmc.LOGWARNING)
+        return None
+
+    @staticmethod
+    def _log_msg(msg, loglevel=xbmc.LOGDEBUG):
+        '''helper to send a message to the kodi log'''
+        xbmc.log("Skin Helper Simplecache --> %s" % msg, level=loglevel)
+
+    @staticmethod
+    def _get_timestamp(date_time):
+        '''Converts a datetime object to unix timestamp'''
+        return int(time.mktime(date_time.timetuple()))
+
+    def _get_checksum(self, stringinput):
+        '''get int checksum from string'''
+        if not stringinput and not self.global_checksum:
+            return 0
+        if self.global_checksum:
+            stringinput = "%s-%s" %(self.global_checksum, stringinput)
+        else:
+            stringinput = str(stringinput)
+        return reduce(lambda x, y: x + y, map(ord, stringinput))
+
+
+def use_cache(cache_days=14):
+    '''
+        wrapper around our simple cache to use as decorator
+        Usage: define an instance of SimpleCache with name "cache" (self.cache) in your class
+        Any method that needs caching just add @use_cache as decorator
+        NOTE: use unnamed arguments for calling the method and named arguments for optional settings
+    '''
+    def decorator(func):
+        '''our decorator'''
+        def decorated(*args, **kwargs):
+            '''process the original method and apply caching of the results'''
+            method_class = args[0]
+            method_class_name = method_class.__class__.__name__
+            cache_str = "%s.%s" % (method_class_name, func.__name__)
+            # cache identifier is based on positional args only
+            # named args are considered optional and ignored
+            for item in args[1:]:
+                cache_str += u".%s" % item
+            cache_str = cache_str.lower()
+            cachedata = method_class.cache.get(cache_str)
+            global_cache_ignore = False
+            try:
+                global_cache_ignore = method_class.ignore_cache
+            except Exception:
+                pass
+            if cachedata is not None and not kwargs.get("ignore_cache", False) and not global_cache_ignore:
+                return cachedata
+            else:
+                result = func(*args, **kwargs)
+                method_class.cache.set(cache_str, result, expiration=datetime.timedelta(days=cache_days))
+                return result
+        return decorated
+    return decorator

updatefeed.py

@@ -0,0 +1,10 @@
+from newsfeed import *
+from environment import *
+
+logger = myLog()
+logger.write("updatefeed.py running")
+newsFeed=NewsFeed(PATH_VIDEO_DATABASE, logger)
+logger.write("getItemsInFeed")
+videos=newsFeed.getItemsInFeed(FOX_NEWS_URL)
+logger.write(f"updatefeed got {len(videos)}")
+logger.write("updatefeed.py Done.")

utility.py

@@ -0,0 +1,283 @@
+import time
+import webbrowser
+import requests
+from datetime import timedelta
+from datetime import datetime
+from datetime import timezone
+from environment import *
+
+class myLog():
+  def __init__(self):
+    self._file = open(PATH_LOG_FILE,"a",encoding='utf-8')
+
+  def write(self,item):
+    currentDateTime = DateTimeHelper.getCurrentDateTime()
+    strCurrentDateTime = DateTimeHelper.getDateTimeAsString(currentDateTime) 
+    strOutput = '[' + strCurrentDateTime +'] '+item
+    self._file.write(strOutput)
+    self._file.write("\n")
+    self._file.flush()
+
+class Utility:
+    def __init__(self):
+        pass
+
+    @staticmethod
+    def pad(strItem, strPad, length):
+        while len(strItem)<length:
+            strItem=strPad + strItem
+        return strItem
+
+class StringHelper:
+    def __init__(self):
+        pass
+
+    @staticmethod
+    def betweenString(strItem, strBegin, strEnd):
+        if strItem is None:
+            return None
+        index=-1
+        if strBegin is None:
+            index=0
+        else:
+            index = strItem.index(strBegin)
+        if -1==index:
+            return None
+        str=None
+        if strBegin is not None:
+            str=strItem[index+len(strBegin):]
+        else:
+            str=strItem
+        if strEnd is None:
+            return str
+        index=str.index(strEnd)
+        if -1==index :
+            return None
+        sb=""
+        for strIndex in range(0, len(str)-1):
+            if index==strIndex:
+                break
+            sb=sb+str[strIndex]
+        return (sb)
+
+class HttpNetRequest:
+    def __init__(self):
+        self.Message=""
+
+    def getHttpNetRequest(self,url):
+        retrycount=0
+        maxretries=5
+        while retrycount<maxretries:
+            try:
+              response=requests.get(url, timeout=10)
+              return response
+            except:
+                retrycount=retrycount+1
+                if retrycount > maxretries:
+                    raise
+
+
+class DateTimeHelper:
+    def __init__(self):
+        pass
+
+    def __init__(self,capturedatetime,timeChange):
+        self.datetimestamp=capturedatetime
+        self.timeChange=timeChange
+        self.offsetTime=DateTimeHelper.applyRelativeTime(self.datetimestamp, self.timeChange)
+
+    def getDateTimeStamp(self):
+        return self.datetimestamp
+
+    def getTimeChange(self):
+        return self.timeChange
+
+    def getOffsetTime(self):
+        return self.offsetTime
+
+    def getOffsetTimeAsString(self):
+        return DateTimeHelper.getDateTimeAsString(self.offsetTime)
+
+    def toString(self):
+        pass
+
+    @staticmethod
+    def getDateTimeAsString(someDateTime):
+        if(not isinstance(someDateTime,datetime)):
+            raise Exception('Invalid type for parameter')
+        return someDateTime.strftime("%m-%d-%Y %H:%M:%S")
+
+    @staticmethod 
+    def getDateTimeFromString(someDateTimeString):
+        if(not isinstance(someDateTimeString,str)):
+            raise Exception('Invalid type for parameter')
+        return DateTimeHelper.strptime(someDateTimeString,"%m-%d-%Y %H:%M:%S")
+
+    @staticmethod
+    def getCurrentDateTime():
+        return datetime.now()
+
+    @staticmethod
+    def strptime(theTime,theFormat):
+        try:
+            return datetime.strptime(theTime,theFormat)
+        except:
+            return datetime(*(time.strptime(theTime,theFormat)[0:6]))
+        
+    @staticmethod 
+    def canstrptime(theTime,theFormat):
+        try:
+            datetime.strptime(theTime,theFormat)
+            return True
+        except:
+            return False
+        
+# returns a datetime
+    @staticmethod
+    def applyRelativeTime(sometime,relativetime):
+        if(not isinstance(sometime,datetime)):
+            raise Exception('Invalid type for parameter')
+        if(not isinstance(relativetime,str)):
+            raise Exception('Invalid type for parameter')
+        if DateTimeHelper.canstrptime(relativetime,'%B %d, %Y'):
+            sometime = DateTimeHelper.strptime(relativetime,'%B %d, %Y')
+            return sometime
+        if relativetime=='just now':
+            return sometime
+        if relativetime=='just in':
+            return sometime
+        relativetimesplit=relativetime.split()
+        if len(relativetimesplit)==2:
+            year=datetime.now().year
+            relativetimex=relativetime+', '+str(year)
+            relativeDate = DateTimeHelper.strptime(relativetimex, '%B %d, %Y')
+            if(relativeDate>datetime.now()):
+              year=datetime.now().year-1
+              relativetimex=relativetime+', '+str(year)
+              relativeDate=DateTimeHelper.strptime(relativetimex,'%B %d, %Y')
+            days=sometime-relativeDate
+            sometime=sometime-days
+        elif relativetimesplit[1]=='hour' or relativetimesplit[1]=='hours':
+            hours=int(relativetimesplit[0])
+            sometime=sometime-timedelta(hours=hours)
+        elif relativetimesplit[1]=='day' or relativetimesplit[1]=='days':
+            days=int(relativetimesplit[0])
+            sometime=sometime-timedelta(days=days)
+        elif relativetimesplit[1]=='minute' or relativetimesplit[1]=='minutes':
+            minutes=int(relativetimesplit[0])
+            sometime=sometime-timedelta(minutes=minutes)
+        elif len(relativetimesplit)==3:  # '16 mins ago'  '2 hours ago'
+          if relativetimesplit[1]=='mins':
+              minutes=int(relativetimesplit[0])
+              sometime=sometime-timedelta(minutes=minutes)
+          elif relativetimesplit[1]=='hours':
+            hours=int(relativetimesplit[0])
+            sometime=sometime-timedelta(hours=hours)
+          elif relativetimesplit[1]=='day' or relativetimesplit[1]=='days':
+            days=int(relativetimesplit[0])
+            sometime=sometime-timedelta(days=days)              
+        return sometime
+   
+class DateTime:
+    def __init__(self):
+        self.date=DateTime.getCurrentTime()
+
+    def __init__(self,strDate=None):
+        if None!=strDate:
+            self.date=DateTime.dateFromString(strDate)
+        else:
+            self.date=DateTime.getCurrentTime()
+
+    def toString(self):
+        return DateTime.dateToString(self.date)
+    
+    def toStringMonthDay(self):
+        return self.getMonthAsString() + ' ' + Utility.pad(str(self.date.day),'0',2) + ', '+ str(self.date.year)
+    
+    def getMonthAsString(self):
+        strMonth=None
+        if(self.date.month==1):
+            strMonth='January'
+        elif(self.date.month==2):
+            strMonth='February'
+        elif(self.date.month==3):
+            strMonth='March'
+        elif(self.date.month==4):
+            strMonth='April'
+        elif(self.date.month==5):
+            strMonth='May'
+        elif(self.date.month==6):
+            strMonth='June'
+        elif(self.date.month==7):
+            strMonth='July'
+        elif(self.date.month==8):
+            strMonth='August'
+        elif(self.date.month==9):
+            strMonth='September'
+        elif(self.date.month==10):
+            strMonth='October'
+        elif(self.date.month==11):
+            strMonth='November'
+        elif(self.date.month==12):
+            strMonth='December'
+        else:
+            strMonth='???'
+        return strMonth
+
+    def deltaTime(self,someDate):
+      return DateTime.deltaTime(self.date,someDate)
+    
+    def getDate(self):
+        return self.date
+
+    @staticmethod
+    def fromdatetime(somedatetime):
+        if(not isinstance(somedatetime,datetime)):
+            raise Exception('Invalid type for parameter')
+        theDate=DateTime()
+        theDate.date=somedatetime
+        return theDate
+
+    @staticmethod
+    def dateToString(someDate):
+        return str(someDate)
+
+    @staticmethod
+    def dateFromString(strDate):
+        return datetime.fromisoformat(strDate)
+
+    @staticmethod
+    def now():
+        return DateTime.getCurrentTime()
+
+    @staticmethod
+    def getCurrentTime():
+        return datetime.now(timezone.utc)
+
+    @staticmethod
+    def sortList(dateList):
+        dateList.sort(key=lambda x:x.toString())
+
+    @staticmethod
+    def deltaTime(startTime,endTime):
+        if startTime > endTime:
+            timedelta=startTime-endTime
+        else:
+            timedelta=endTime-startTime
+        days, seconds=timedelta.days, timedelta.seconds
+        hours=timedelta.total_seconds()//3600
+        minutes=(seconds %3600)//60
+        seconds=seconds%60
+        return days, hours, minutes, seconds
+
+
+
+#currentDate=DateTimeHelper.getCurrentDateTime()
+#strDateTime=DateTimeHelper.getDateTimeAsString(currentDate)
+#print(strDateTime)
+#relativeTime=DateTimeHelper.applyRelativeTime(currentDate,'October 9')
+#strDateTime=DateTimeHelper.getDateTimeAsString(relativeTime)
+#print(relativeTime)
+#if(relativeTime>currentDate):
+#  print('It is greater')
+

video.py

@@ -0,0 +1,132 @@
+from datetime import timedelta
+from datetime import datetime
+from datetime import timezone
+from environment import *
+from utility import *
+import os
+
+class Video:
+    def __init__(self):
+        self.description=None
+        self.url=None
+        self.icon=None
+        self.timestamp=DateTime()
+        self.feedTimeOffset='just now'
+        self.feedtime=datetime.now()
+
+    def __init__(self, description, url):
+        self.description=description
+        self.url=url
+        self.icon=None
+        self.timestamp=DateTime()
+        self.feedTimeOffset='just now'
+        self.feedtime=datetime.now()
+
+    def __init__(self, description, url, icon, timestamp=None):
+        self.description=description
+        self.url=url
+        self.icon=icon
+        if None==timestamp:
+            self.timestamp=DateTime()
+        else:
+            self.timestamp=timestamp
+        self.feedTimeOffset='just now'
+        self.feedtime=datetime.now()
+
+    def getDescription(self):
+        return self.description
+
+    def getUrl(self):
+        return self.url
+
+    def getIcon(self):
+        return self.icon
+
+    def getTimestamp(self):
+        return self.timestamp
+
+# This is a datetime.  This time gets calculated by applying the feedtime offset to today.  
+    def setFeedTime(self,feedtime):
+        self.feedtime=feedtime
+
+    def getFeedTime(self):
+        return self.feedtime
+
+# This is the <time> section in the video feed.  '16 mins ago', 'May 5', 'July 14 2023' etc.,
+    def getFeedTimeOffset(self):
+        return self.feedTimeOffset
+
+    def setFeedTimeOffset(self, feedtimeoffset):
+        self.feedTimeOffset=feedtimeoffset
+
+    def toString(self):
+        return(self.description+"|"+self.url+"|"+self.icon+"|"+self.timestamp.toString())
+
+    @staticmethod
+    def fromString(line):
+        splits=line.split("|")
+        description=splits[0].strip()
+        url=splits[1].strip()
+        icon=splits[2].strip()
+        timestamp=DateTime(splits[3].strip())
+        return(Video(description,url,icon,timestamp))
+
+    @staticmethod
+    def sort(videoList):
+      return sorted(videoList, key=lambda x:x.getTimestamp().toString(),reverse=True)
+
+    def getPureDescription(self):
+      textElement=StringHelper.betweenString(self.description,None,'-')
+      timeElement=StringHelper.betweenString(self.description,'-',None)
+      durationElement=StringHelper.betweenString(timeElement,' ',' ')
+      newDescription=textElement+'-'+' '+ durationElement+' ('+self.getTimestamp().toStringMonthDay()+')'
+      return newDescription
+
+# This method takes a dictionary of video
+    @staticmethod
+    def write(pathOutputFile, videos):
+     unique={}
+
+     if None == videos:
+        return
+     
+     if os.path.exists(pathOutputFile):
+        print('Removing {file}'.format(file=pathOutputFile))
+        os.remove(pathOutputFile)
+     
+     videoList=list(videos.values())
+     for video in videoList:
+       heading = StringHelper.betweenString(video.getDescription(), None, '-')
+       if not heading in unique:
+         unique[heading]=video
+     videoList=Video.sort(list(unique.values()))
+     with open(pathOutputFile, "w", encoding='utf-8') as outputStream: 
+       for video in videoList:
+         outputStream.write(video.toString()+'\r\n')
+     outputStream.close()
+     print('wrote {lines} lines to {file}'.format(lines=len(videoList),file=pathOutputFile))
+     return
+
+# This method returns a dictionary so caller must use videos=list(load("").values()) to create a list
+    @staticmethod
+    def load(pathfilename):
+        unique={}
+        videos={}
+
+        if not os.path.exists(pathfilename):
+           return videos
+        try:  
+          with open(pathfilename, "r", encoding='utf-8') as inputStream:
+            for line in inputStream:
+              lowerLine=line.lower()
+              heading = StringHelper.betweenString(line, None, '-')
+              if not heading in unique:
+                unique[heading]=heading
+                video = Video.fromString(line)
+                video.description=video.getPureDescription()
+                videos[video.description]=video
+          inputStream.close()
+          return videos
+        except Exception as exception:
+          print('Exception reading {file} {exception}'.format(file=pathfilename,exception=exception))
+          return None