From 38f58fa144c9d389bf869347aca15a586fff890a Mon Sep 17 00:00:00 2001 From: Sean Kessler Date: Thu, 29 Jan 2026 18:09:56 -0500 Subject: [PATCH] Initial Commit --- .gitignore | 4 + Test.py | 124 +++++++++++ archive.py | 159 +++++++++++++++ environment.py | 49 +++++ newsfeed.py | 544 +++++++++++++++++++++++++++++++++++++++++++++++++ scraper.py | 183 +++++++++++++++++ simplecache.py | 302 +++++++++++++++++++++++++++ updatefeed.py | 10 + utility.py | 283 +++++++++++++++++++++++++ video.py | 132 ++++++++++++ 10 files changed, 1790 insertions(+) create mode 100644 .gitignore create mode 100755 Test.py create mode 100755 archive.py create mode 100755 environment.py create mode 100755 newsfeed.py create mode 100755 scraper.py create mode 100755 simplecache.py create mode 100755 updatefeed.py create mode 100755 utility.py create mode 100644 video.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1e298ec --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +__pycache__/environment.cpython-39.pyc +__pycache__/utility.cpython-39.pyc +__pycache__/video.cpython-39.pyc +.vscode/settings.json diff --git a/Test.py b/Test.py new file mode 100755 index 0000000..cb722a0 --- /dev/null +++ b/Test.py @@ -0,0 +1,124 @@ +from datetime import datetime +from datetime import timedelta + +class DateTimeHelper: + def __init__(self): + pass + + def __init__(self,capturedatetime,timeChange): + self.datetimestamp=capturedatetime + self.timeChange=timeChange + self.offsetTime=DateTimeHelper.applyRelativeTime(self.datetimestamp, self.timeChange) + + def getDateTimeStamp(self): + return self.datetimestamp + + def getTimeChange(self): + return self.timeChange + + def getOffsetTime(self): + return self.offsetTime + + def getOffsetTimeAsString(self): + return DateTimeHelper.getDateTimeAsString(self.offsetTime) + + def toString(self): + pass + + @staticmethod + def getDateTimeAsString(someDateTime): + if(not isinstance(someDateTime,datetime)): + raise Exception('Invalid type for parameter') + return someDateTime.strftime("%m-%d-%Y %H:%M:%S") + + @staticmethod + def getDateTimeFromString(someDateTimeString): + if(not isinstance(someDateTimeString,str)): + raise Exception('Invalid type for parameter') + return datetime.strptime(someDateTimeString,"%m-%d-%Y %H:%M:%S") + + @staticmethod + def getCurrentDateTime(): + return datetime.now() + + @staticmethod + def applyRelativeTime(sometime,relativetime): + if(not isinstance(sometime,datetime)): + raise Exception('Invalid type for parameter') + if(not isinstance(relativetime,str)): + raise Exception('Invalid type for parameter') + if relativetime=='just now': + return sometime + relativetimesplit=relativetime.split() + if len(relativetimesplit)==2: + year=datetime.now().year + relativetime=relativetime+', '+str(year) + relativeDate = datetime.strptime(relativetime, '%B %d, %Y') + days=sometime-relativeDate + # sometime=sometime-timedelta(days=days) + sometime=sometime-days + elif relativetimesplit[1]=='hour' or relativetimesplit[1]=='hours': + hours=int(relativetimesplit[0]) + sometime=sometime-timedelta(hours=hours) + elif relativetimesplit[1]=='day' or relativetimesplit[1]=='days': + days=int(relativetimesplit[0]) + sometime=sometime-timedelta(days=days) + elif relativetimesplit[1]=='minute' or relativetimesplit[1]=='minutes': + minutes=int(relativetimesplit[0]) + sometime=sometime-timedelta(minutes=minutes) + return sometime + +list=[] + + +# date_time_str='03-03-2023 12:00:00' +# capturedatetime = datetime.strptime(date_time_str, '%m-%d-%Y %H:%M:%S') +# Item.applyRelativeTime(capturedatetime, '2 days ago') + +deltaTime='March 7' +now = datetime.now() +relativeTime=DateTimeHelper.applyRelativeTime(now, deltaTime) +relativeTimeStr=DateTimeHelper.getDateTimeAsString(relativeTime) + +print('time:{p1}'.format(p1=DateTimeHelper.getDateTimeAsString(now))) +print('delta:{p1}'.format(p1=deltaTime)) +print('result:{p1}'.format(p1=relativeTimeStr)) + + +#now = Item.getCurrentDateTime() +#capturedatetime = Item.getDateTimeAsString(now) + +#Item.applyRelativeTime(capturedatetime, 'February 24') + + +list.append(Item(capturedatetime,'February 24')) +list.append(Item(capturedatetime,'2 days ago')) +list.append(Item(capturedatetime,'21 days ago')) +list.append(Item(capturedatetime,'1 day ago')) +list.append(Item(capturedatetime,'1 minute ago')) +list.append(Item(capturedatetime,'5 minutes ago')) + +for item in list: + print (item) + + + +# date_time_str='01-01-2023 12:00:00' +# b = datetime.strptime(date_time_str, '%m-%d-%Y %H:%M:%S') + +# date_time_str='01-01-2023 18:00:00' +# c = datetime.strptime(date_time_str, '%m-%d-%Y %H:%M:%S') + +# list.append(a) +# list.append(b) +# list.append(c) + +# print(list) + +# another=sorted(list, key=lambda x:x) + +# print(another) +# for item in another: +# print('{:02d}-{:02d}-{:d} {:02d}:{:02d}:{:02d}'.format(item.month,item.day,item.year,item.hour,item.minute,item.second)) + + diff --git a/archive.py b/archive.py new file mode 100755 index 0000000..f6df653 --- /dev/null +++ b/archive.py @@ -0,0 +1,159 @@ +import os +import glob +import functools +from environment import * +from utility import * +from video import * + +# This file is executed in a cron job. +# To view the cron schedule type sudo crontab -r in a shell. Use Ctrl-S to save after editing +# This cron job should run evrry 30 minutes. Shorter intervals burden the system +# The ouptut from the print statements is generated in the syslog /var/log/syslog sudo nano /var/log/syslog +# Overall system perfromance can be monitored using htop + +def comparator(item1, item2): + try: + list1=item1.split('.') + list2=item2.split('.') + index1=int(list1[len(list1)-1]) + index2=int(list2[len(list2)-1]) + if index1index2: + return 1 + return 0 + except: + return 0 + +def createArchive(pathOutputFile,tokens,files): + lines=0 + unique={} + videos={} + + for token in tokens: + print('Filtering for "{token}"'.format(token=token)) + + videos = Video.load(pathOutputFile) + + for video in list(videos.values()): + description = description=createDescription(video.description,video.getTimestamp()) + if not description in unique: + unique[description]=createDescription(video.description, video.getTimestamp()) + + try: + print('found {count} archive files.'.format(count=len(files))) + print('processing {pathOutputFile}'.format(pathOutputFile=pathOutputFile)) + for file in files: + try: + with open(file, "r", encoding='utf-8') as inputStream: + for line in inputStream: + lowerLine=line.lower() + for token in tokens: + token=token.lower() + result = lowerLine.find(token) + if -1 != result: + video = Video.fromString(line) + heading = video.getDescription() + if not heading in unique: + unique[heading]=heading + video = Video.fromString(line) + video.description=createDescription(video.description,video.getTimestamp()) + videos[video.description]=video + lines = lines + 1 + inputStream.close() + except Exception as exception: + print('Exception reading {file} {exception}'.format(file=file,exception=exception)) + continue + print('writing {pathOutputFile}'.format(pathOutputFile=pathOutputFile)) + Video.write(pathOutputFile, videos) + except Exception as exception: + print('Exception creating output file {file} {exception}'.format(file=pathOutputFile,exception=exception)) + return + +# clean the archive files by removing files older than 'expiryDays' +def cleanArchive(files, expiryDays): + expiredList = [] + for pathFileName in files: + modification_date = os.path.getmtime(pathFileName) + modification_date = datetime.fromtimestamp(modification_date, timezone.utc) + now = DateTime.now() + days, hours, minutes, seconds = DateTime.deltaTime(modification_date, now) + if(days > expiryDays): + expiredList.append(pathFileName) + print('Expiring {count} files.'.format(count=len(expiredList))) + for file in expiredList: + os.remove(file) + return + +def createDescription(strDescription, timeStamp): + textElement=StringHelper.betweenString(strDescription,None,'-') + timeElement=StringHelper.betweenString(strDescription,'-',None) + durationElement=StringHelper.betweenString(timeElement,' ',' ') + newDescription=textElement+'-'+' '+ durationElement+' ('+timeStamp.toStringMonthDay()+')' + return newDescription + +def getFiles(archiveFileLike): + files = glob.glob(archiveFileLike) + files=files+glob.glob(archiveFileLike+'.*') + return files + +# This program runs through all of the videodb*.txt files looking for keywords with which to +# build each of the individually named mini-archives. +# 1) Search for all videodb.txt.* files +# 2) Expire files older than specified number of days +# 3) Load the archive (for each fo the types enumerated below) +# 4) Run through file collection for the given archive archive and append to the archive as tags are found +# 5) Sort the archive +# 6) Truncate existing archive if it exists +# 7) Write the new archive + +path=PATH_VIDEO_DATABASE +archiveFile=path+'/videodb' +archiveFileLike=archiveFile+'.txt' + +#For debugging +# path='/home/pi/Projects/Python/NewsFeed/Archive' +# archiveFile=path+'/videodb' +# archiveFileLike=archiveFile+'.txt' + +files = getFiles(archiveFileLike) +print('There are {count} archive files to process before cleaning'.format(count=len(files))) +cleanArchive(files, 30) +files = getFiles(archiveFileLike) +print('There are {count} archive files to process after cleaning'.format(count=len(files))) + +print('archive.py running...') + +archiveFileName=ARCHIVEDB_FILENAME +pathOutputFile=PathHelper.makePathFileName(archiveFileName,path) +print('pathOutputFile={pathOutputFile}'.format(pathOutputFile=pathOutputFile)) +tokens=["Keane","Jesse","Israel","Hamas"," War ","Iran","Hezzbollah","Gaza","Ukraine"] +createArchive(pathOutputFile,tokens,files) + +hannityFileName=HANNITYARCHIVEDB_FILENAME +pathOutputFile=PathHelper.makePathFileName(hannityFileName,path) +print('pathOutputFile={pathOutputFile}'.format(pathOutputFile=pathOutputFile)) +tokens=["Hannity"] +createArchive(pathOutputFile,tokens,files) + +levinFileName=LEVINARCHIVEDB_FILENAME +pathOutputFile=PathHelper.makePathFileName(levinFileName,path) +print('pathOutputFile={pathOutputFile}'.format(pathOutputFile=pathOutputFile)) +tokens=["Levin"] +createArchive(pathOutputFile,tokens,files) + +hawleyFileName=HAWLEYARCHIVEDB_FILENAME +pathOutputFile=PathHelper.makePathFileName(hawleyFileName,path) +print('pathOutputFile={pathOutputFile}'.format(pathOutputFile=pathOutputFile)) +tokens=["Hawley"] +createArchive(pathOutputFile,tokens,files) + +militaryFileName=MILITARYARCHIVEDB_FILENAME +pathOutputFile=PathHelper.makePathFileName(militaryFileName,path) +print('pathOutputFile={pathOutputFile}'.format(pathOutputFile=pathOutputFile)) +tokens=["Keane","Kellogg","Russia","Ukraine","Israel","Korea","Iran","Venezuela","Cuba","China"] +createArchive(pathOutputFile,tokens,files) + +print('archive.py done.') + + diff --git a/environment.py b/environment.py new file mode 100755 index 0000000..0f39213 --- /dev/null +++ b/environment.py @@ -0,0 +1,49 @@ +APPEND_SYS_PATH='/home/pi/.kodi/addons/plugin.video.fox.news/resources/lib' +PATH_VIDEO_DATABASE='/home/pi/.kodi/addons/plugin.video.fox.news/resources/lib/archive' +USE_ICON_URL=True +PATH_LOG_FILE="/home/pi/.kodi/temp/MyLog.log" +FOX_NEWS_URL="https://www.foxnews.com/video" +FOX_NEWS_US_URL="hhttps://www.foxnews.com/video/topics/us" +FOX_NEWS_EXCLUSIVE_URL="https://foxnews.com" +FOX_NEWS_AMERICAS_NEWSROOM_URL="https://www.foxnews.com/video/shows/americas-newsroom" +FOX_NEWS_OUTNUMBERED_URL="https://www.foxnews.com/video/shows/outnumbered" +SELECTED_ARCHIVE_URL="https://www.foxnews.com/video/dummy" +LEVIN_ARCHIVE_URL="https://www.foxnews.com/video/levindummyurl" +HANNITY_ARCHIVE_URL="https://www.foxnews.com/video/hannitydummyurl" +HAWLEY_ARCHIVE_URL="https://www.foxnews.com/video/hawleydummyurl" +MILITARY_ARCHIVE_URL="https://www.foxnews.com/video/militarydummyurl" + +#FOX_NEWS_ICON_OF_LAST_RESORT="https://static.foxnews.com/static/orion/styles/img/fox-news/favicons/apple-touch-icon-180x180.png" +#ENABLE_USE_ICON_OF_LAST_RESORT=False + + +VIDEODB_AMERICAS_NEWSROOM_FILENAME="videodb_americasnewsroom.txt" +VIDEODB_OUTNUMBERED_FILENAME="videodb_outnumbered.txt" +VIDEODB_FILENAME="videodb.txt" +VIDEODB_EXCLUSIVE_FILENAME="videodb_exc.txt" +VIDEODB_US_FILENAME="videodb_us.txt" +ARCHIVEDB_FILENAME="archivedb.txt" +LEVINARCHIVEDB_FILENAME="levindb.txt" +HANNITYARCHIVEDB_FILENAME="hannitydb.txt" +HAWLEYARCHIVEDB_FILENAME="hawleydb.txt" +MILITARYARCHIVEDB_FILENAME="militarydb.txt" + +CACHE_EXPIRY_MINS=10 + +LOG_HTTP_RESPONSES = False + +FEED_REJECT_IF_OLDER_THAN_DAYS = 7 + +class PathHelper: + pathChar="/" + + def __init__(self): + pass + + @staticmethod + def makePathFileName(file,path): + while file.endswith(PathHelper.pathChar): + file=file[:-1] + while path.endswith(PathHelper.pathChar): + path=path[:-1] + return path+PathHelper.pathChar+file diff --git a/newsfeed.py b/newsfeed.py new file mode 100755 index 0000000..5516dee --- /dev/null +++ b/newsfeed.py @@ -0,0 +1,544 @@ +import json +import os +import webbrowser +import requests +import traceback +import time +import re +import glob +import shutil +from datetime import timedelta +from datetime import datetime +from datetime import timezone +from environment import * +from utility import * +from video import * + +class NewsFeed: + def __init__(self, pathDb, logger=None): + self.pathDb=pathDb + self.logger=logger + + @staticmethod + def isResourceAvailable(url): + try: + response=requests.head(url, timeout=2.5) + if not response.ok: + return False + return True + except: + return False + + def getItemsInAmericasNewsRoomFeed(self,url): + now=datetime.now() + cachePathFileName=PathHelper.makePathFileName(VIDEODB_AMERICAS_NEWSROOM_FILENAME,self.pathDb) + if self.isFeedCacheAvailable(cachePathFileName,CACHE_EXPIRY_MINS): + videos=self.readFeedCache(cachePathFileName) + if videos is not None: + return(videos) + sections=Sections() + videos = {} + httpNetRequest=HttpNetRequest() + response=httpNetRequest=httpNetRequest.getHttpNetRequest(url) + status=response.status_code + searchIndex=0 + response.close() + if status!=200: + return None + if LOG_HTTP_RESPONSES: + self.writeLog(url) + self.writeLog(response.text) + while -1!= searchIndex: + video, searchIndex = sections.getItemsInSection(response.text,"article",searchIndex) + if video is not None and not (video.description in videos): + videos[video.description]=video + video.setFeedTime(DateTimeHelper.applyRelativeTime(now,video.feedTimeOffset)) + videoList=list(videos.values()) + videoList=sorted(videoList, key=lambda x:x.getFeedTime(),reverse=False) + self.writeFeedCache(cachePathFileName,videoList) + return (videoList) + + def getItemsInOutnumberedFeed(self,url): + now=datetime.now() + cachePathFileName=PathHelper.makePathFileName(VIDEODB_OUTNUMBERED_FILENAME,self.pathDb) + if self.isFeedCacheAvailable(cachePathFileName,CACHE_EXPIRY_MINS): + videos=self.readFeedCache(cachePathFileName) + if videos is not None: + return(videos) + sections=Sections() + videos = {} + httpNetRequest=HttpNetRequest() + response=httpNetRequest=httpNetRequest.getHttpNetRequest(url) + status=response.status_code + searchIndex=0 + response.close() + if status!=200: + return None + if LOG_HTTP_RESPONSES: + self.writeLog(url) + self.writeLog(response.text) + while -1!= searchIndex: + video, searchIndex = sections.getItemsInSection(response.text,"article",searchIndex) + if video is not None and not (video.description in videos): + videos[video.description]=video + video.setFeedTime(DateTimeHelper.applyRelativeTime(now,video.feedTimeOffset)) + videoList=list(videos.values()) + videoList=sorted(videoList, key=lambda x:x.getFeedTime(),reverse=True) + self.writeFeedCache(cachePathFileName,videoList) + return (videoList) + + def getItemsInFeed(self,url): + now=datetime.now() + cachePathFileName=PathHelper.makePathFileName(VIDEODB_FILENAME,self.pathDb) + if self.isFeedCacheAvailable(cachePathFileName,CACHE_EXPIRY_MINS): + self.writeLog(f"Loading videos from cache {cachePathFileName}") + videos=self.readFeedCache(cachePathFileName) + if videos is not None: + return(videos) + sections=Sections() + videos = {} + httpNetRequest=HttpNetRequest() + self.writeLog(f"Loading videos from {url}") + response=httpNetRequest=httpNetRequest.getHttpNetRequest(url) + status=response.status_code + searchIndex=0 + response.close() + if status!=200: + return None + if LOG_HTTP_RESPONSES: + self.writeLog(url) + self.writeLog(response.text) + while -1!= searchIndex: + video, searchIndex= sections.getItemsInSection(response.text,"article",searchIndex) + if video is not None and not (video.description in videos): + videos[video.description]=video + video.setFeedTime(DateTimeHelper.applyRelativeTime(now,video.feedTimeOffset)) +# videoList=list(videos.values()) + videoList=self.filterFeedMaxDays(list(videos.values()),FEED_REJECT_IF_OLDER_THAN_DAYS) + videoList=sorted(videoList, key=lambda x:x.getFeedTime(),reverse=True) + self.writeFeedCache(cachePathFileName,videoList) + return (videoList) + + def filterFeedMaxDays(self, videos, days): + now = datetime.now() + filteredList=[] + for video in videos: + delta = now - video.getFeedTime() + if delta.days <= days: + message = f"INCL. days={delta.days},feed time={video.getFeedTime()} feed time offset (strPublication)=:'{video.feedTimeOffset}', description={video.description}" + self.writeLog(message) + filteredList.insert(0,video) + else: + message = f"EXCL. days={delta.days},feed time={video.getFeedTime()} feed time offset (strPublication)=:'{video.feedTimeOffset}', description={video.description}" + self.writeLog(message) + return filteredList + + def getUSItemsInFeed(self,url): + now=datetime.now() + cachePathFileName=PathHelper.makePathFileName(VIDEODB_US_FILENAME,self.pathDb) + if self.isFeedCacheAvailable(cachePathFileName,CACHE_EXPIRY_MINS): + videos=self.readFeedCache(cachePathFileName) + if videos is not None: + return(videos) + sections=Sections() + videos = {} + httpNetRequest=HttpNetRequest() + response=httpNetRequest.getHttpNetRequest(url) + status=response.status_code + searchIndex=0 + response.close() + if status!=200: + return None + if LOG_HTTP_RESPONSES: + self.writeLog(url) + self.writeLog(response.text) + while -1!= searchIndex: + videoId, searchIndex = sections.getVideoIdInSection(response.text,"article",searchIndex) + if videoId is None: + continue + url='https://video.foxnews.com/v/'+videoId + httpNetRequest=HttpNetRequest() + innerResponse=httpNetRequest.getHttpNetRequest(url) + status=innerResponse.status_code + innerResponse.close() + if status!=200: + continue + video=sections.getVideoContentInSection(innerResponse.text) + if video is not None and not (video.description in videos): + videos[video.description]=video + video.setFeedTime(DateTimeHelper.applyRelativeTime(now,video.feedTimeOffset)) + videoList=list(videos.values()) + videoList=sorted(videoList, key=lambda x:x.getFeedTime(),reverse=True) + self.writeFeedCache(cachePathFileName,videoList) + return (videoList) + + def getExclusiveItemsInFeed(self,url): + now=datetime.now() + cachePathFileName=PathHelper.makePathFileName(VIDEODB_EXCLUSIVE_FILENAME,self.pathDb) + if self.isFeedCacheAvailable(cachePathFileName,CACHE_EXPIRY_MINS): + videos=self.readFeedCache(cachePathFileName) + if videos is not None: + return(videos) + sections=Sections() + videos = {} + httpNetRequest=HttpNetRequest() + response=httpNetRequest.getHttpNetRequest(url) + status=response.status_code + searchIndex=0 + response.close() + if status!=200: + return None + if LOG_HTTP_RESPONSES: + self.writeLog(url) + self.writeLog(response.Text) + while -1!= searchIndex: + videoId, searchIndex = sections.getVideoIdInSection(response.text,"article",searchIndex) + if videoId is None: + continue + url='https://video.foxnews.com/v/'+videoId + httpNetRequest=HttpNetRequest() + innerResponse=httpNetRequest.getHttpNetRequest(url) + status=innerResponse.status_code + innerResponse.close() + if status!=200: + continue + video=sections.getVideoContentInSection(innerResponse.text) + if video is not None and not (video.description in videos): + videos[video.description]=video + video.setFeedTime(DateTimeHelper.applyRelativeTime(now,video.feedTimeOffset)) + videoList=list(videos.values()) + videoList=sorted(videoList, key=lambda x:x.getFeedTime(),reverse=True) + self.writeFeedCache(cachePathFileName,videoList) + return (videoList) + + def getItemsInArchiveFeed(self,url,archiveDbFileName): + cachePathFileName=PathHelper.makePathFileName(archiveDbFileName,self.pathDb) + videos=self.readFeedCache(cachePathFileName) + if videos is not None: + return(videos) + return(None) + + def readFeedCache(self,pathFileName): + try: + videos=[] + with open(pathFileName,"r",encoding='utf-8') as inputStream: + for line in inputStream: + video=Video.fromString(line) + videos.append(video) + inputStream.close() + return(videos) + except: + self.writeLog(traceback.format_exc()) + return(None) + + def writeFeedCache(self,pathFileName,videos): + try: + with open(pathFileName,"w",encoding='utf-8') as outputStream: + for video in videos: + outputStream.write(video.toString()+"\n") + outputStream.close() + return(videos) + except: + self.writeLog(traceback.format_exc()) + return(videos) + + def isFeedCacheAvailable(self,pathFileName,expireMinutes): + try: + self.writeLog('Inspecting cache file {pathFileName}'.format(pathFileName=pathFileName)) + if not os.path.isfile(pathFileName): + return(False) + modifiedTime=os.path.getmtime(pathFileName) + convertTime=time.localtime(modifiedTime) + formatTime=time.strftime('%d%m%Y %H:%M:%S',convertTime) + fileDateTime=DateTimeHelper.strptime(formatTime,'%d%m%Y %H:%M:%S') + currentTime=datetime.now() + timedelta=currentTime-fileDateTime + hours, hremainder = divmod(timedelta.seconds,3600) + minutes, mremainder = divmod(timedelta.seconds,60) + self.writeLog('file is = "{age}" hours old'.format(age=hours)) + self.writeLog('file is = "{age}" minutes old'.format(age=minutes)) + if hours > 1 or minutes > expireMinutes: + self.archiveFile(pathFileName) + return(False) + return (True) + except: + self.writeLog(traceback.format_exc()); + return(False) + + def archiveFile(self, pathFileName): + if not os.path.isfile(pathFileName): + return(False) + archiveFile=StringHelper.betweenString(pathFileName, None, '.txt') + archiveFileLike=archiveFile+'.txt.*' + files = glob.glob(archiveFileLike) + index=len(files)+1 + archiveFileName=archiveFile+'.txt.'+str(index) + print('archiveFile: Copying "{pathFileName}" to "{archiveFileName}".'.format(pathFileName=pathFileName,archiveFileName=archiveFileName)) + shutil.copy(pathFileName,archiveFileName) + os.remove(pathFileName) + return(True) + + def writeLog(self,message): + if self.logger is not None: + self.logger.write(message) + else: + print(message) + +class Sections: + def __init__(self): + self.dummy=None + + def getItemsInSection(self, strInput, sectionName, searchIndex): + video=None + startSection='<'+sectionName + endSection='") + if -1 != indexDuration: + strDuration=strContainingString[indexDuration:] + strDuration=self.betweenString(strDuration,">","<") + description=description+" - "+strDuration + indexPublication=strContainingString.index("
") + if -1 != indexPublication: + strPublication=strContainingString[indexPublication:] + strPublication=self.betweenString(strPublication,"") + description=description+" ("+strPublication+")" + icon=None + indexIcon=strContainingString.index("srcset=") + if -1 != indexIcon: + icon=strContainingString[indexIcon:] + icon=self.betweenString(icon,"\"","\"") + splits=icon.split(',') + icon=self.betweenString(splits[len(splits)-1],None,'?') + icon=icon.strip() + description = description.strip() + video=Video(description,previewUrl,icon) + video.feedTimeOffset=strPublication + return video, searchIndex + + def getVideoIdInSection(self, strInput, sectionName, searchIndex): + video=None + startSection='<'+sectionName + endSection='=length: + return str + while stringLength < length: + sb=sb+filler + stringLength=stringLength+1 + return sb+str + +def parseDuration(strDuration): + expression=re.compile(r"\d+") + result=expression.findall(strDuration) + if 2!=len(result): + return None, None + return pad(result[0],'0',2), pad(result[1],'0',2) + + +# DON'T LEAVE ANYTHING OPEN BELOW THIS LINE BECAUSE THIS FILE IS IMPORTED BY OTHER MODULES AND ANY CODE NOT IN A CLASS WILL BE RUN + +#print(FOX_NEWS_URL) +# pathFileName='/home/pi/.kodi/addons/plugin.video.fox.news/resources/lib/videodb.txt' +# newsFeed=NewsFeed('/home/pi/.kodi/addons/plugin.video.fox.news/resources/lib/') +# newsFeed.ArchiveFile(pathFileName) + + +# pathFileName='/home/pi/.kodi/addons/plugin.video.fox.news/resources/lib/videodb.txt' +# modifiedTime=os.path.getmtime(pathFileName) +# convertTime=time.localtime(modifiedTime) +# formatTime=time.strftime('%d%m%Y %H:%M:%S',convertTime) +# fileDateTime=DateTimeHelper.strptime(formatTime) + +#fileDateTime=datetime.strptime(formatTime,'%d%m%Y %H:%M:%S') +#fileDateTime2=datetime(*(time.strptime(formatTime,'%d%m%Y %H:%M:%S')[0:6])) +#currentTime=datetime.now() + +#Test the main feed +# newsFeed=NewsFeed('/home/pi/Projects/Python/NewsFeed/') +# newsFeed=NewsFeed('/home/pi/.kodi/addons/plugin.video.fox.news/resources/lib/videodb.txt') +# newsFeed=NewsFeed(PATH_VIDEO_DATABASE, myLog()) +# newsFeed=NewsFeed('/home/pi/Projects/Python/NewsFeed/', myLog()) +# videos=newsFeed.getItemsInFeed(FOX_NEWS_URL) +# for video in videos: +# if(video.description.startswith("Martha")): +# print(f"Description={video.description}") +# print(f"Url={video.url}") +# print(f"getTimestamp={video.getTimestamp().toStringMonthDay()}") +# print(f"getFeedTimeOffset={video.getFeedTimeOffset()}") +# print(f"getFeedTime={video.getFeedTime()}") +# print(f"daysOld={(datetime.now()-video.getFeedTime()).days}") +# print(' ') + +# pull the time out of the description and subtract it from the time we scanned the feed. +# the result will be the time of the article..use this to sort on. +# (i.e.) FeedTime:02/03/2023 12:00:00 Article Time:2 hours ago Real time:10:00:00 + + +#Test the exclusive items feed +#newsFeed=NewsFeed('/home/pi/.kodi/addons/plugin.video.fox.news/resources/lib/') +#videos=newsFeed.getExclusiveItemsInFeed("https://www.foxnews.com") +# for video in videos: +# print(video.description) + + +# Test the U.S. Feed +# newsFeed=NewsFeed('/home/pi/.kodi/addons/plugin.video.fox.news/resources/lib/') +# videos=newsFeed.getUSItemsInFeed("https://www.foxnews.com/video/topics/us") +# for video in videos: +# print(video.description) + +# Test the America's NewsRoom Feed +# newsFeed=NewsFeed('/home/pi/.kodi/addons/plugin.video.fox.news/resources/lib/') +# videos=newsFeed.getItemsInAmericasNewsRoomFeed("https://www.foxnews.com/video/shows/americas-newsroom") +# print('got {count} videos for America''s Newsroom'.format(count=len(videos))) +# for video in videos: +# print(video.description) +# print(video.url) + +# Test the Outnumbered Feed +# newsFeed=NewsFeed('/home/pi/.kodi/addons/plugin.video.fox.news/resources/lib/') +# videos=newsFeed.getItemsInOutnumbereFeed("https://www.foxnews.com/video/shows/outnumbered") +# print('got {count} videos for Outnumbered'.format(count=len(videos))) +# for video in videos: +# print(video.description) +# print(video.url) + +#minutes, seconds = parseDuration('PT24M5S') +#print('Duration is {minutes}:{seconds}'.format(minutes=minutes,seconds=seconds)) + +# isoDate="2022-10-27T10:24:11Z".replace("Z","+00:00") +# articleTime=datetime.datetime.fromisoformat(isoDate) +# print('time:{time}'.format(time=articleTime)) +# currentTime=Date.getCurrentTime() +# print('time:{time}'.format(time=currentTime)) +# days, hours, minutes, seconds=Date.deltaTime(articleTime,currentTime) +# print('elapsed time {days} days, {hours} hours, {minutes} minutes, {seconds} seconds'.format(days=days,hours=hours,minutes=minutes,seconds=seconds)) + +# currentTime2=Date.getCurrentTime() +# strCurrentTime2=str(currentTime2) +# currentTime2=datetime.datetime.fromisoformat(strCurrentTime2) +# days, hours, minutes, seconds=Date.deltaTime(currentTime2,currentTime) +# print('elapsed time {days} days, {hours} hours, {minutes} minutes, {seconds} seconds'.format(days=days,hours=hours,minutes=minutes,seconds=seconds)) + +# dateList=[] + +# currentDate=Date() +# dateList.append(currentDate) +# currentDate2=Date() +# dateList.append(currentDate2) + +# dateList.sort(key=lambda x:x.toString()) +# for date in dateList: +# print(date.toString()) +# #print(dateList) diff --git a/scraper.py b/scraper.py new file mode 100755 index 0000000..702d91a --- /dev/null +++ b/scraper.py @@ -0,0 +1,183 @@ +# -*- coding: utf-8 -*- +# Fox News Kodi Video Addon +# + +APPEND_SYS_PATH='/home/pi/.kodi/addons/plugin.video.fox.news/resources/lib' + +import os +import sys +import traceback +sys.path.append(APPEND_SYS_PATH) +from t1mlib import t1mAddon +import json +import re +import xbmc +import xbmcplugin +import xbmcgui +import xbmcvfs +import xbmc +import xbmcaddon +import html.parser +import sys +import datetime +import time +import random +import requests +import sqlite3 +from newsfeed import NewsFeed +from functools import reduce +from simplecache import SimpleCache +from environment import * +from utility import * + +#class myLog(): +# def __init__(self): +# self._file = open(PATH_LOG_FILE,"w",encoding='utf-8') +# +# def write(self,item): +# self._file.write(item) +# self._file.write("\n") +# self._file.flush() + +class myAddon(t1mAddon): + + def __init__(self, aname): + t1mAddon.__init__(self, aname) + self._logfile = myLog() + self._cache = SimpleCache() + self._pDialog = xbmcgui.DialogProgressBG() + + def getAddonMenu(self,url,ilist): + try: + self._logfile.write('getAddonMenu') + ilist = self.addMenuItem("Latest Fox News Featured Clips",'GE', ilist, FOX_NEWS_URL, self.addonIcon, self.addonFanart, {}, isFolder=True) + ilist = self.addMenuItem("Fox News Outnumbered Clips",'GE', ilist, FOX_NEWS_OUTNUMBERED_URL, self.addonIcon, self.addonFanart, {}, isFolder=True) + ilist = self.addMenuItem("Selected Archive Clips",'GE', ilist, SELECTED_ARCHIVE_URL, self.addonIcon, self.addonFanart, {}, isFolder=True) + ilist = self.addMenuItem("Mark Levin Archive Clips",'GE', ilist, LEVIN_ARCHIVE_URL, self.addonIcon, self.addonFanart, {}, isFolder=True) + ilist = self.addMenuItem("Sean Hannity Archive Clips",'GE', ilist, HANNITY_ARCHIVE_URL, self.addonIcon, self.addonFanart, {}, isFolder=True) + ilist = self.addMenuItem("Josh Hawley Archive Clips",'GE', ilist, HAWLEY_ARCHIVE_URL, self.addonIcon, self.addonFanart, {}, isFolder=True) + ilist = self.addMenuItem("Military Archive Clips",'GE', ilist, MILITARY_ARCHIVE_URL, self.addonIcon, self.addonFanart, {}, isFolder=True) + return(ilist) + except: + self._logfile.write(traceback.format_exc()) + raise + finally: + return(ilist) + + def getAddonEpisodes(self,url,ilist): + try: + if url == FOX_NEWS_URL: + self._logfile.write('getAddonEpisodes url={url}'.format(url=url)) + self._pDialog.create('Retrieving News Articles...') + newsFeed=NewsFeed(PATH_VIDEO_DATABASE, self._logfile) + videos=newsFeed.getItemsInFeed(url) + for video in videos: + if USE_ICON_URL: + ilist = self.addMenuItem(video.description,'GV', ilist, video.url, video.icon, self.addonFanart, {}, isFolder=False) + else: + ilist = self.addMenuItem(video.description,'GV', ilist, video.url, self.addonIcon, self.addonFanart, {}, isFolder=False) + self._logfile.write('Processed {articles} articles.'.format(articles=len(videos))) + elif url == SELECTED_ARCHIVE_URL: + self._logfile.write('getAddonEpisodes url={url}'.format(url=url)) + self._pDialog.create('Retrieving News Articles...') + newsFeed=NewsFeed(PATH_VIDEO_DATABASE, self._logfile) + videos=newsFeed.getItemsInArchiveFeed(url,ARCHIVEDB_FILENAME) + for video in videos: + if USE_ICON_URL: + ilist = self.addMenuItem(video.description,'GV', ilist, video.url, video.icon, self.addonFanart, {}, isFolder=False) + else: + ilist = self.addMenuItem(video.description,'GV', ilist, video.url, self.addonIcon, self.addonFanart, {}, isFolder=False) + self._logfile.write('Processed {articles} articles.'.format(articles=len(videos))) + elif url == LEVIN_ARCHIVE_URL: + self._logfile.write('getAddonEpisodes url={url}'.format(url=url)) + self._pDialog.create('Retrieving News Articles...') + newsFeed=NewsFeed(PATH_VIDEO_DATABASE, self._logfile) + videos=newsFeed.getItemsInArchiveFeed(url,LEVINARCHIVEDB_FILENAME) + for video in videos: + if USE_ICON_URL: + ilist = self.addMenuItem(video.description,'GV', ilist, video.url, video.icon, self.addonFanart, {}, isFolder=False) + else: + ilist = self.addMenuItem(video.description,'GV', ilist, video.url, self.addonIcon, self.addonFanart, {}, isFolder=False) + self._logfile.write('Processed {articles} articles.'.format(articles=len(videos))) + elif url == HANNITY_ARCHIVE_URL: + self._logfile.write('getAddonEpisodes url={url}'.format(url=url)) + self._pDialog.create('Retrieving News Articles...') + newsFeed=NewsFeed(PATH_VIDEO_DATABASE, self._logfile) + videos=newsFeed.getItemsInArchiveFeed(url,HANNITYARCHIVEDB_FILENAME) + for video in videos: + if USE_ICON_URL: + ilist = self.addMenuItem(video.description,'GV', ilist, video.url, video.icon, self.addonFanart, {}, isFolder=False) + else: + ilist = self.addMenuItem(video.description,'GV', ilist, video.url, self.addonIcon, self.addonFanart, {}, isFolder=False) + self._logfile.write('Processed {articles} articles.'.format(articles=len(videos))) + elif url == HAWLEY_ARCHIVE_URL: + self._logfile.write('getAddonEpisodes url={url}'.format(url=url)) + self._pDialog.create('Retrieving News Articles...') + newsFeed=NewsFeed(PATH_VIDEO_DATABASE, self._logfile) + videos=newsFeed.getItemsInArchiveFeed(url,HAWLEYARCHIVEDB_FILENAME) + for video in videos: + if USE_ICON_URL: + ilist = self.addMenuItem(video.description,'GV', ilist, video.url, video.icon, self.addonFanart, {}, isFolder=False) + else: + ilist = self.addMenuItem(video.description,'GV', ilist, video.url, self.addonIcon, self.addonFanart, {}, isFolder=False) + self._logfile.write('Processed {articles} articles.'.format(articles=len(videos))) + elif url == MILITARY_ARCHIVE_URL: + self._logfile.write('getAddonEpisodes url={url}'.format(url=url)) + self._pDialog.create('Retrieving News Articles...') + newsFeed=NewsFeed(PATH_VIDEO_DATABASE, self._logfile) + videos=newsFeed.getItemsInArchiveFeed(url,MILITARYARCHIVEDB_FILENAME) + for video in videos: + if USE_ICON_URL: + ilist = self.addMenuItem(video.description,'GV', ilist, video.url, video.icon, self.addonFanart, {}, isFolder=False) + else: + ilist = self.addMenuItem(video.description,'GV', ilist, video.url, self.addonIcon, self.addonFanart, {}, isFolder=False) + self._logfile.write('Processed {articles} articles.'.format(articles=len(videos))) + elif url== FOX_NEWS_OUTNUMBERED_URL: + self._logfile.write('getAddonEpisodes url={url}'.format(url=url)) + self._pDialog.create('Retrieving Outnumbered Articles...') + newsFeed=NewsFeed(PATH_VIDEO_DATABASE, self._logfile) + videos=newsFeed.getItemsInOutnumberedFeed(url) + for video in videos: + if USE_ICON_URL: + ilist = self.addMenuItem(video.description,'GV', ilist, video.url, video.icon, self.addonFanart, {}, isFolder=False) + else: + ilist = self.addMenuItem(video.description,'GV', ilist, video.url, self.addonIcon, self.addonFanart, {}, isFolder=False) + self._logfile.write('Processed {articles} articles.'.format(articles=len(videos))) + except Exception as exception: + self._logfile.write('Exception:{exception}'.format(exception=exception)) + self._logfile.write(traceback.format_exc()) + raise + finally: + self._pDialog.close() + return(ilist) + + def getAddonMovies(self,url,ilist): + try: + self._logfile.write('getAddonMovies url={url}'.format(url=url)) + self._pDialog.create('getAddonMovies...') + self._pDialog.update(0,message='getAddonMovies...') + self._pDialog.close() + return(ilist) + except Exception as exception: + self._logfile.write('Exception:{exception}'.format(exception=exception)) + self._logfile.write(traceback.format_exc()) + raise + finally: + self._pDialog.close() + return(ilist) + + def getAddonShows(self,url,ilist): + try: + self._logfile.write('getAddonShows') + self._pDialog.create('getAddonShows...') + self._pDialog.update(0,message='getAddonShows...') + self._pDialog.close() + return ilist + except Exception as exception: + self._logfile.write('Exception:{exception}'.format(exception=exception)) + self._logfile.write(traceback.format_exc()) + raise + finally: + self._pDialog.close() + return(ilist) + diff --git a/simplecache.py b/simplecache.py new file mode 100755 index 0000000..bddbff4 --- /dev/null +++ b/simplecache.py @@ -0,0 +1,302 @@ +# -*- coding: utf-8 -*- +# Fox News Kodi Video Addon +# +import os +import sys +import traceback +from t1mlib import t1mAddon +import json +import re +import xbmc +import xbmcplugin +import xbmcgui +import xbmcvfs +import xbmc +import xbmcaddon +import html.parser +import sys +import datetime +import time +import random +import requests +import sqlite3 + +ADDON_ID = "plugin.video.fox.news" + +class SimpleCache(object): + '''simple stateless caching system for Kodi''' + enable_mem_cache = True + global_checksum = None + _exit = False + _auto_clean_interval = datetime.timedelta(hours=4) + _win = None + _busy_tasks = [] + _database = None + + def __init__(self): + '''Initialize our caching class''' + self._win = xbmcgui.Window(10000) + self._monitor = xbmc.Monitor() + self.check_cleanup() + self._log_msg("Initialized") + + def close(self): + '''tell any tasks to stop immediately (as we can be called multithreaded) and cleanup objects''' + self._exit = True + # wait for all tasks to complete + while self._busy_tasks and not self._monitor.abortRequested(): + xbmc.sleep(25) + del self._win + del self._monitor + self._log_msg("Closed") + + def __del__(self): + '''make sure close is called''' + if not self._exit: + self.close() + + def get(self, endpoint, checksum=""): + ''' + get object from cache and return the results + endpoint: the (unique) name of the cache object as reference + checkum: optional argument to check if the checksum in the cacheobject matches the checkum provided + ''' + checksum = self._get_checksum(checksum) + cur_time = self._get_timestamp(datetime.datetime.now()) + result = None + # 1: try memory cache first + if self.enable_mem_cache: + result = self._get_mem_cache(endpoint, checksum, cur_time) + + # 2: fallback to _database cache + if result is None: + result = self._get_db_cache(endpoint, checksum, cur_time) + + return result + + def set(self, endpoint, data, checksum="", expiration=datetime.timedelta(days=30)): + ''' + set data in cache + ''' + task_name = "set.%s" % endpoint + self._busy_tasks.append(task_name) + checksum = self._get_checksum(checksum) + expires = self._get_timestamp(datetime.datetime.now() + expiration) + + # memory cache: write to window property + if self.enable_mem_cache and not self._exit: + self._set_mem_cache(endpoint, checksum, expires, data) + + # db cache + if not self._exit: + self._set_db_cache(endpoint, checksum, expires, data) + + # remove this task from list + self._busy_tasks.remove(task_name) + + def check_cleanup(self): + '''check if cleanup is needed - public method, may be called by calling addon''' + cur_time = datetime.datetime.now() + lastexecuted = self._win.getProperty("simplecache.clean.lastexecuted") + if not lastexecuted: + self._win.setProperty("simplecache.clean.lastexecuted", repr(cur_time)) + elif (eval(lastexecuted) + self._auto_clean_interval) < cur_time: + # cleanup needed... + self._do_cleanup() + + def _get_mem_cache(self, endpoint, checksum, cur_time): + ''' + get cache data from memory cache + we use window properties because we need to be stateless + ''' + result = None + cachedata = self._win.getProperty(endpoint) + + if cachedata: + cachedata = eval(cachedata) + if cachedata[0] > cur_time: + if not checksum or checksum == cachedata[2]: + result = cachedata[1] + return result + + def _set_mem_cache(self, endpoint, checksum, expires, data): + ''' + window property cache as alternative for memory cache + usefull for (stateless) plugins + ''' + cachedata = (expires, data, checksum) + cachedata_str = repr(cachedata) + self._win.setProperty(endpoint, cachedata_str) + + + def _get_db_cache(self, endpoint, checksum, cur_time): + '''get cache data from sqllite _database''' + result = None + query = "SELECT expires, data, checksum FROM simplecache WHERE id = ?" + cache_data = self._execute_sql(query, (endpoint,)) + if cache_data: + cache_data = cache_data.fetchone() + if cache_data and cache_data[0] > cur_time: + if not checksum or cache_data[2] == checksum: + result = eval(cache_data[1]) + # also set result in memory cache for further access + if self.enable_mem_cache: + self._set_mem_cache(endpoint, checksum, cache_data[0], result) + return result + + def _set_db_cache(self, endpoint, checksum, expires, data): + ''' store cache data in _database ''' + query = "INSERT OR REPLACE INTO simplecache( id, expires, data, checksum) VALUES (?, ?, ?, ?)" + data = repr(data) + self._execute_sql(query, (endpoint, expires, data, checksum)) + + def _do_cleanup(self): + '''perform cleanup task''' + if self._exit or self._monitor.abortRequested(): + return + self._busy_tasks.append(__name__) + cur_time = datetime.datetime.now() + cur_timestamp = self._get_timestamp(cur_time) + self._log_msg("Running cleanup...") + if self._win.getProperty("simplecachecleanbusy"): + return + self._win.setProperty("simplecachecleanbusy", "busy") + + query = "SELECT id, expires FROM simplecache" + for cache_data in self._execute_sql(query).fetchall(): + cache_id = cache_data[0] + cache_expires = cache_data[1] + + if self._exit or self._monitor.abortRequested(): + return + + # always cleanup all memory objects on each interval + self._win.clearProperty(cache_id) + + # clean up db cache object only if expired + if cache_expires < cur_timestamp: + query = 'DELETE FROM simplecache WHERE id = ?' + self._execute_sql(query, (cache_id,)) + self._log_msg("delete from db %s" % cache_id) + + # compact db + self._execute_sql("VACUUM") + + # remove task from list + self._busy_tasks.remove(__name__) + self._win.setProperty("simplecache.clean.lastexecuted", repr(cur_time)) + self._win.clearProperty("simplecachecleanbusy") + self._log_msg("Auto cleanup done") + + def _get_database(self): + '''get reference to our sqllite _database - performs basic integrity check''' + addon = xbmcaddon.Addon(ADDON_ID) + dbpath = addon.getAddonInfo('profile') + dbfile = xbmcvfs.translatePath("%s/simplecache.db" % dbpath) + + if not xbmcvfs.exists(dbpath): + xbmcvfs.mkdirs(dbpath) + del addon + try: + connection = sqlite3.connect(dbfile, timeout=30, isolation_level=None) + connection.execute('SELECT * FROM simplecache LIMIT 1') + return connection + except Exception as error: + # our _database is corrupt or doesn't exist yet, we simply try to recreate it + if xbmcvfs.exists(dbfile): + xbmcvfs.delete(dbfile) + try: + connection = sqlite3.connect(dbfile, timeout=30, isolation_level=None) + connection.execute( + """CREATE TABLE IF NOT EXISTS simplecache( + id TEXT UNIQUE, expires INTEGER, data TEXT, checksum INTEGER)""") + return connection + except Exception as error: + self._log_msg("Exception while initializing _database: %s" % str(error), xbmc.LOGWARNING) + self.close() + return None + + def _execute_sql(self, query, data=None): + '''little wrapper around execute and executemany to just retry a db command if db is locked''' + retries = 0 + result = None + error = None + # always use new db object because we need to be sure that data is available for other simplecache instances + with self._get_database() as _database: + while not retries == 10 and not self._monitor.abortRequested(): + if self._exit: + return None + try: + if isinstance(data, list): + result = _database.executemany(query, data) + elif data: + result = _database.execute(query, data) + else: + result = _database.execute(query) + return result + except sqlite3.OperationalError as error: + if "_database is locked" in error: + self._log_msg("retrying DB commit...") + retries += 1 + self._monitor.waitForAbort(0.5) + else: + break + except Exception as error: + break + self._log_msg("_database ERROR ! -- %s" % str(error), xbmc.LOGWARNING) + return None + + @staticmethod + def _log_msg(msg, loglevel=xbmc.LOGDEBUG): + '''helper to send a message to the kodi log''' + xbmc.log("Skin Helper Simplecache --> %s" % msg, level=loglevel) + + @staticmethod + def _get_timestamp(date_time): + '''Converts a datetime object to unix timestamp''' + return int(time.mktime(date_time.timetuple())) + + def _get_checksum(self, stringinput): + '''get int checksum from string''' + if not stringinput and not self.global_checksum: + return 0 + if self.global_checksum: + stringinput = "%s-%s" %(self.global_checksum, stringinput) + else: + stringinput = str(stringinput) + return reduce(lambda x, y: x + y, map(ord, stringinput)) + + +def use_cache(cache_days=14): + ''' + wrapper around our simple cache to use as decorator + Usage: define an instance of SimpleCache with name "cache" (self.cache) in your class + Any method that needs caching just add @use_cache as decorator + NOTE: use unnamed arguments for calling the method and named arguments for optional settings + ''' + def decorator(func): + '''our decorator''' + def decorated(*args, **kwargs): + '''process the original method and apply caching of the results''' + method_class = args[0] + method_class_name = method_class.__class__.__name__ + cache_str = "%s.%s" % (method_class_name, func.__name__) + # cache identifier is based on positional args only + # named args are considered optional and ignored + for item in args[1:]: + cache_str += u".%s" % item + cache_str = cache_str.lower() + cachedata = method_class.cache.get(cache_str) + global_cache_ignore = False + try: + global_cache_ignore = method_class.ignore_cache + except Exception: + pass + if cachedata is not None and not kwargs.get("ignore_cache", False) and not global_cache_ignore: + return cachedata + else: + result = func(*args, **kwargs) + method_class.cache.set(cache_str, result, expiration=datetime.timedelta(days=cache_days)) + return result + return decorated + return decorator diff --git a/updatefeed.py b/updatefeed.py new file mode 100755 index 0000000..be5ceb0 --- /dev/null +++ b/updatefeed.py @@ -0,0 +1,10 @@ +from newsfeed import * +from environment import * + +logger = myLog() +logger.write("updatefeed.py running") +newsFeed=NewsFeed(PATH_VIDEO_DATABASE, logger) +logger.write("getItemsInFeed") +videos=newsFeed.getItemsInFeed(FOX_NEWS_URL) +logger.write(f"updatefeed got {len(videos)}") +logger.write("updatefeed.py Done.") diff --git a/utility.py b/utility.py new file mode 100755 index 0000000..08bb68e --- /dev/null +++ b/utility.py @@ -0,0 +1,283 @@ +import time +import webbrowser +import requests +from datetime import timedelta +from datetime import datetime +from datetime import timezone +from environment import * + +class myLog(): + def __init__(self): + self._file = open(PATH_LOG_FILE,"a",encoding='utf-8') + + def write(self,item): + currentDateTime = DateTimeHelper.getCurrentDateTime() + strCurrentDateTime = DateTimeHelper.getDateTimeAsString(currentDateTime) + strOutput = '[' + strCurrentDateTime +'] '+item + self._file.write(strOutput) + self._file.write("\n") + self._file.flush() + +class Utility: + def __init__(self): + pass + + @staticmethod + def pad(strItem, strPad, length): + while len(strItem) maxretries: + raise + + +class DateTimeHelper: + def __init__(self): + pass + + def __init__(self,capturedatetime,timeChange): + self.datetimestamp=capturedatetime + self.timeChange=timeChange + self.offsetTime=DateTimeHelper.applyRelativeTime(self.datetimestamp, self.timeChange) + + def getDateTimeStamp(self): + return self.datetimestamp + + def getTimeChange(self): + return self.timeChange + + def getOffsetTime(self): + return self.offsetTime + + def getOffsetTimeAsString(self): + return DateTimeHelper.getDateTimeAsString(self.offsetTime) + + def toString(self): + pass + + @staticmethod + def getDateTimeAsString(someDateTime): + if(not isinstance(someDateTime,datetime)): + raise Exception('Invalid type for parameter') + return someDateTime.strftime("%m-%d-%Y %H:%M:%S") + + @staticmethod + def getDateTimeFromString(someDateTimeString): + if(not isinstance(someDateTimeString,str)): + raise Exception('Invalid type for parameter') + return DateTimeHelper.strptime(someDateTimeString,"%m-%d-%Y %H:%M:%S") + + @staticmethod + def getCurrentDateTime(): + return datetime.now() + + @staticmethod + def strptime(theTime,theFormat): + try: + return datetime.strptime(theTime,theFormat) + except: + return datetime(*(time.strptime(theTime,theFormat)[0:6])) + + @staticmethod + def canstrptime(theTime,theFormat): + try: + datetime.strptime(theTime,theFormat) + return True + except: + return False + +# returns a datetime + @staticmethod + def applyRelativeTime(sometime,relativetime): + if(not isinstance(sometime,datetime)): + raise Exception('Invalid type for parameter') + if(not isinstance(relativetime,str)): + raise Exception('Invalid type for parameter') + if DateTimeHelper.canstrptime(relativetime,'%B %d, %Y'): + sometime = DateTimeHelper.strptime(relativetime,'%B %d, %Y') + return sometime + if relativetime=='just now': + return sometime + if relativetime=='just in': + return sometime + relativetimesplit=relativetime.split() + if len(relativetimesplit)==2: + year=datetime.now().year + relativetimex=relativetime+', '+str(year) + relativeDate = DateTimeHelper.strptime(relativetimex, '%B %d, %Y') + if(relativeDate>datetime.now()): + year=datetime.now().year-1 + relativetimex=relativetime+', '+str(year) + relativeDate=DateTimeHelper.strptime(relativetimex,'%B %d, %Y') + days=sometime-relativeDate + sometime=sometime-days + elif relativetimesplit[1]=='hour' or relativetimesplit[1]=='hours': + hours=int(relativetimesplit[0]) + sometime=sometime-timedelta(hours=hours) + elif relativetimesplit[1]=='day' or relativetimesplit[1]=='days': + days=int(relativetimesplit[0]) + sometime=sometime-timedelta(days=days) + elif relativetimesplit[1]=='minute' or relativetimesplit[1]=='minutes': + minutes=int(relativetimesplit[0]) + sometime=sometime-timedelta(minutes=minutes) + elif len(relativetimesplit)==3: # '16 mins ago' '2 hours ago' + if relativetimesplit[1]=='mins': + minutes=int(relativetimesplit[0]) + sometime=sometime-timedelta(minutes=minutes) + elif relativetimesplit[1]=='hours': + hours=int(relativetimesplit[0]) + sometime=sometime-timedelta(hours=hours) + elif relativetimesplit[1]=='day' or relativetimesplit[1]=='days': + days=int(relativetimesplit[0]) + sometime=sometime-timedelta(days=days) + return sometime + +class DateTime: + def __init__(self): + self.date=DateTime.getCurrentTime() + + def __init__(self,strDate=None): + if None!=strDate: + self.date=DateTime.dateFromString(strDate) + else: + self.date=DateTime.getCurrentTime() + + def toString(self): + return DateTime.dateToString(self.date) + + def toStringMonthDay(self): + return self.getMonthAsString() + ' ' + Utility.pad(str(self.date.day),'0',2) + ', '+ str(self.date.year) + + def getMonthAsString(self): + strMonth=None + if(self.date.month==1): + strMonth='January' + elif(self.date.month==2): + strMonth='February' + elif(self.date.month==3): + strMonth='March' + elif(self.date.month==4): + strMonth='April' + elif(self.date.month==5): + strMonth='May' + elif(self.date.month==6): + strMonth='June' + elif(self.date.month==7): + strMonth='July' + elif(self.date.month==8): + strMonth='August' + elif(self.date.month==9): + strMonth='September' + elif(self.date.month==10): + strMonth='October' + elif(self.date.month==11): + strMonth='November' + elif(self.date.month==12): + strMonth='December' + else: + strMonth='???' + return strMonth + + def deltaTime(self,someDate): + return DateTime.deltaTime(self.date,someDate) + + def getDate(self): + return self.date + + @staticmethod + def fromdatetime(somedatetime): + if(not isinstance(somedatetime,datetime)): + raise Exception('Invalid type for parameter') + theDate=DateTime() + theDate.date=somedatetime + return theDate + + @staticmethod + def dateToString(someDate): + return str(someDate) + + @staticmethod + def dateFromString(strDate): + return datetime.fromisoformat(strDate) + + @staticmethod + def now(): + return DateTime.getCurrentTime() + + @staticmethod + def getCurrentTime(): + return datetime.now(timezone.utc) + + @staticmethod + def sortList(dateList): + dateList.sort(key=lambda x:x.toString()) + + @staticmethod + def deltaTime(startTime,endTime): + if startTime > endTime: + timedelta=startTime-endTime + else: + timedelta=endTime-startTime + days, seconds=timedelta.days, timedelta.seconds + hours=timedelta.total_seconds()//3600 + minutes=(seconds %3600)//60 + seconds=seconds%60 + return days, hours, minutes, seconds + + + +#currentDate=DateTimeHelper.getCurrentDateTime() +#strDateTime=DateTimeHelper.getDateTimeAsString(currentDate) +#print(strDateTime) +#relativeTime=DateTimeHelper.applyRelativeTime(currentDate,'October 9') +#strDateTime=DateTimeHelper.getDateTimeAsString(relativeTime) +#print(relativeTime) +#if(relativeTime>currentDate): +# print('It is greater') + diff --git a/video.py b/video.py new file mode 100644 index 0000000..b5f5f15 --- /dev/null +++ b/video.py @@ -0,0 +1,132 @@ +from datetime import timedelta +from datetime import datetime +from datetime import timezone +from environment import * +from utility import * +import os + +class Video: + def __init__(self): + self.description=None + self.url=None + self.icon=None + self.timestamp=DateTime() + self.feedTimeOffset='just now' + self.feedtime=datetime.now() + + def __init__(self, description, url): + self.description=description + self.url=url + self.icon=None + self.timestamp=DateTime() + self.feedTimeOffset='just now' + self.feedtime=datetime.now() + + def __init__(self, description, url, icon, timestamp=None): + self.description=description + self.url=url + self.icon=icon + if None==timestamp: + self.timestamp=DateTime() + else: + self.timestamp=timestamp + self.feedTimeOffset='just now' + self.feedtime=datetime.now() + + def getDescription(self): + return self.description + + def getUrl(self): + return self.url + + def getIcon(self): + return self.icon + + def getTimestamp(self): + return self.timestamp + +# This is a datetime. This time gets calculated by applying the feedtime offset to today. + def setFeedTime(self,feedtime): + self.feedtime=feedtime + + def getFeedTime(self): + return self.feedtime + +# This is the