Sean/Newsfeed
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Reviewed with Anthropic Claude and made bug fixes.

Browse Source
This commit is contained in:
Sean Kessler (Europa)
2026-02-22 11:45:01 -05:00
parent 13b18b01dd
commit e660e385e5
1 changed files with 212 additions and 181 deletions
Download Patch File Download Diff File Expand all files Collapse all files

99
newsfeed.py
View File

@@ -30,6 +30,8 @@ class NewsFeed:
return False return False
def getItemsInAmericasNewsRoomFeed(self,url): def getItemsInAmericasNewsRoomFeed(self,url):
response = None
try:
now=datetime.now() now=datetime.now()
cachePathFileName=PathHelper.makePathFileName(VIDEODB_AMERICAS_NEWSROOM_FILENAME,self.pathDb) cachePathFileName=PathHelper.makePathFileName(VIDEODB_AMERICAS_NEWSROOM_FILENAME,self.pathDb)
if self.isFeedCacheAvailable(cachePathFileName,CACHE_EXPIRY_MINS): if self.isFeedCacheAvailable(cachePathFileName,CACHE_EXPIRY_MINS):
@@ -39,10 +41,9 @@ class NewsFeed:
sections=Sections() sections=Sections()
videos = {} videos = {}
httpNetRequest=HttpNetRequest() httpNetRequest=HttpNetRequest()
response=httpNetRequest=httpNetRequest.getHttpNetRequest(url) response=httpNetRequest.getHttpNetRequest(url)
status=response.status_code status=response.status_code
searchIndex=0 searchIndex=0
response.close()
if status!=200: if status!=200:
return None return None
if LOG_HTTP_RESPONSES: if LOG_HTTP_RESPONSES:
@@ -57,8 +58,13 @@ class NewsFeed:
videoList=sorted(videoList, key=lambda x:x.getFeedTime(),reverse=False) videoList=sorted(videoList, key=lambda x:x.getFeedTime(),reverse=False)
self.writeFeedCache(cachePathFileName,videoList) self.writeFeedCache(cachePathFileName,videoList)
return (videoList) return (videoList)
finally:
if None!= response:
response.close()
def getItemsInOutnumberedFeed(self,url): def getItemsInOutnumberedFeed(self,url):
response = None
try:
now=datetime.now() now=datetime.now()
cachePathFileName=PathHelper.makePathFileName(VIDEODB_OUTNUMBERED_FILENAME,self.pathDb) cachePathFileName=PathHelper.makePathFileName(VIDEODB_OUTNUMBERED_FILENAME,self.pathDb)
if self.isFeedCacheAvailable(cachePathFileName,CACHE_EXPIRY_MINS): if self.isFeedCacheAvailable(cachePathFileName,CACHE_EXPIRY_MINS):
@@ -68,10 +74,9 @@ class NewsFeed:
sections=Sections() sections=Sections()
videos = {} videos = {}
httpNetRequest=HttpNetRequest() httpNetRequest=HttpNetRequest()
response=httpNetRequest=httpNetRequest.getHttpNetRequest(url) response=httpNetRequest.getHttpNetRequest(url)
status=response.status_code status=response.status_code
searchIndex=0 searchIndex=0
response.close()
if status!=200: if status!=200:
return None return None
if LOG_HTTP_RESPONSES: if LOG_HTTP_RESPONSES:
@@ -86,8 +91,13 @@ class NewsFeed:
videoList=sorted(videoList, key=lambda x:x.getFeedTime(),reverse=True) videoList=sorted(videoList, key=lambda x:x.getFeedTime(),reverse=True)
self.writeFeedCache(cachePathFileName,videoList) self.writeFeedCache(cachePathFileName,videoList)
return (videoList) return (videoList)
finally:
if None!=response:
response.close()
def getItemsInFeed(self,url): def getItemsInFeed(self,url):
response = None
try:
now=datetime.now() now=datetime.now()
cachePathFileName=PathHelper.makePathFileName(VIDEODB_FILENAME,self.pathDb) cachePathFileName=PathHelper.makePathFileName(VIDEODB_FILENAME,self.pathDb)
if self.isFeedCacheAvailable(cachePathFileName,CACHE_EXPIRY_MINS): if self.isFeedCacheAvailable(cachePathFileName,CACHE_EXPIRY_MINS):
@@ -99,10 +109,9 @@ class NewsFeed:
videos = {} videos = {}
httpNetRequest=HttpNetRequest() httpNetRequest=HttpNetRequest()
self.writeLog(f"Loading videos from {url}") self.writeLog(f"Loading videos from {url}")
response=httpNetRequest=httpNetRequest.getHttpNetRequest(url) response=httpNetRequest.getHttpNetRequest(url)
status=response.status_code status=response.status_code
searchIndex=0 searchIndex=0
response.close()
if status!=200: if status!=200:
return None return None
if LOG_HTTP_RESPONSES: if LOG_HTTP_RESPONSES:
@@ -117,6 +126,9 @@ class NewsFeed:
videoList=sorted(videoList, key=lambda x:x.getFeedTime(),reverse=True) videoList=sorted(videoList, key=lambda x:x.getFeedTime(),reverse=True)
self.writeFeedCache(cachePathFileName,videoList) self.writeFeedCache(cachePathFileName,videoList)
return (videoList) return (videoList)
finally:
if None!=response:
response.close()
def filterFeedMaxDays(self, videos, days): def filterFeedMaxDays(self, videos, days):
now = datetime.now() now = datetime.now()
@@ -126,13 +138,15 @@ class NewsFeed:
if delta.days <= days: if delta.days <= days:
message = f"INCL. days={delta.days},feed time={video.getFeedTime()} feed time offset (strPublication)=:'{video.feedTimeOffset}', description={video.description}" message = f"INCL. days={delta.days},feed time={video.getFeedTime()} feed time offset (strPublication)=:'{video.feedTimeOffset}', description={video.description}"
self.writeLog(message) self.writeLog(message)
filteredList.insert(0,video) filteredList.append(video)
else: else:
message = f"EXCL. days={delta.days},feed time={video.getFeedTime()} feed time offset (strPublication)=:'{video.feedTimeOffset}', description={video.description}" message = f"EXCL. days={delta.days},feed time={video.getFeedTime()} feed time offset (strPublication)=:'{video.feedTimeOffset}', description={video.description}"
self.writeLog(message) self.writeLog(message)
return filteredList return filteredList
def getUSItemsInFeed(self,url): def getUSItemsInFeed(self,url):
response = None
try:
now=datetime.now() now=datetime.now()
cachePathFileName=PathHelper.makePathFileName(VIDEODB_US_FILENAME,self.pathDb) cachePathFileName=PathHelper.makePathFileName(VIDEODB_US_FILENAME,self.pathDb)
if self.isFeedCacheAvailable(cachePathFileName,CACHE_EXPIRY_MINS): if self.isFeedCacheAvailable(cachePathFileName,CACHE_EXPIRY_MINS):
@@ -145,7 +159,6 @@ class NewsFeed:
response=httpNetRequest.getHttpNetRequest(url) response=httpNetRequest.getHttpNetRequest(url)
status=response.status_code status=response.status_code
searchIndex=0 searchIndex=0
response.close()
if status!=200: if status!=200:
return None return None
if LOG_HTTP_RESPONSES: if LOG_HTTP_RESPONSES:
@@ -155,9 +168,9 @@ class NewsFeed:
videoId, searchIndex = sections.getVideoIdInSection(response.text,"article",searchIndex) videoId, searchIndex = sections.getVideoIdInSection(response.text,"article",searchIndex)
if videoId is None: if videoId is None:
continue continue
url='https://video.foxnews.com/v/'+videoId videoUrl='https://video.foxnews.com/v/'+videoId
httpNetRequest=HttpNetRequest() httpNetRequest=HttpNetRequest()
innerResponse=httpNetRequest.getHttpNetRequest(url) innerResponse=httpNetRequest.getHttpNetRequest(videoUrl)
status=innerResponse.status_code status=innerResponse.status_code
innerResponse.close() innerResponse.close()
if status!=200: if status!=200:
@@ -170,8 +183,13 @@ class NewsFeed:
videoList=sorted(videoList, key=lambda x:x.getFeedTime(),reverse=True) videoList=sorted(videoList, key=lambda x:x.getFeedTime(),reverse=True)
self.writeFeedCache(cachePathFileName,videoList) self.writeFeedCache(cachePathFileName,videoList)
return (videoList) return (videoList)
finally:
if None!=response:
response.close()
def getExclusiveItemsInFeed(self,url): def getExclusiveItemsInFeed(self,url):
response = None
try:
now=datetime.now() now=datetime.now()
cachePathFileName=PathHelper.makePathFileName(VIDEODB_EXCLUSIVE_FILENAME,self.pathDb) cachePathFileName=PathHelper.makePathFileName(VIDEODB_EXCLUSIVE_FILENAME,self.pathDb)
if self.isFeedCacheAvailable(cachePathFileName,CACHE_EXPIRY_MINS): if self.isFeedCacheAvailable(cachePathFileName,CACHE_EXPIRY_MINS):
@@ -184,19 +202,18 @@ class NewsFeed:
response=httpNetRequest.getHttpNetRequest(url) response=httpNetRequest.getHttpNetRequest(url)
status=response.status_code status=response.status_code
searchIndex=0 searchIndex=0
response.close()
if status!=200: if status!=200:
return None return None
if LOG_HTTP_RESPONSES: if LOG_HTTP_RESPONSES:
self.writeLog(url) self.writeLog(url)
self.writeLog(response.Text) self.writeLog(response.text)
while -1!= searchIndex: while -1!= searchIndex:
videoId, searchIndex = sections.getVideoIdInSection(response.text,"article",searchIndex) videoId, searchIndex = sections.getVideoIdInSection(response.text,"article",searchIndex)
if videoId is None: if videoId is None:
continue continue
url='https://video.foxnews.com/v/'+videoId videoUrl='https://video.foxnews.com/v/'+videoId
httpNetRequest=HttpNetRequest() httpNetRequest=HttpNetRequest()
innerResponse=httpNetRequest.getHttpNetRequest(url) innerResponse=httpNetRequest.getHttpNetRequest(videoUrl)
status=innerResponse.status_code status=innerResponse.status_code
innerResponse.close() innerResponse.close()
if status!=200: if status!=200:
@@ -209,6 +226,9 @@ class NewsFeed:
videoList=sorted(videoList, key=lambda x:x.getFeedTime(),reverse=True) videoList=sorted(videoList, key=lambda x:x.getFeedTime(),reverse=True)
self.writeFeedCache(cachePathFileName,videoList) self.writeFeedCache(cachePathFileName,videoList)
return (videoList) return (videoList)
finally:
if None!=response:
response.close()
def getItemsInArchiveFeed(self,url,archiveDbFileName): def getItemsInArchiveFeed(self,url,archiveDbFileName):
cachePathFileName=PathHelper.makePathFileName(archiveDbFileName,self.pathDb) cachePathFileName=PathHelper.makePathFileName(archiveDbFileName,self.pathDb)
@@ -220,11 +240,11 @@ class NewsFeed:
def readFeedCache(self,pathFileName): def readFeedCache(self,pathFileName):
try: try:
videos=[] videos=[]
# 'with' will automatically close the stream
with open(pathFileName,"r",encoding='utf-8') as inputStream: with open(pathFileName,"r",encoding='utf-8') as inputStream:
for line in inputStream: for line in inputStream:
video=Video.fromString(line) video=Video.fromString(line)
videos.append(video) videos.append(video)
inputStream.close()
return(videos) return(videos)
except: except:
self.writeLog(traceback.format_exc()) self.writeLog(traceback.format_exc())
@@ -235,34 +255,35 @@ class NewsFeed:
with open(pathFileName,"w",encoding='utf-8') as outputStream: with open(pathFileName,"w",encoding='utf-8') as outputStream:
for video in videos: for video in videos:
outputStream.write(video.toString()+"\n") outputStream.write(video.toString()+"\n")
outputStream.close() # 'with' will automatically close the stream
return(videos) return(videos)
except: except:
self.writeLog(traceback.format_exc()) self.writeLog(traceback.format_exc())
return(videos) return(videos)
def isFeedCacheAvailable(self,pathFileName,expireMinutes): def isFeedCacheAvailable(self, pathFileName, expireMinutes):
try: try:
self.writeLog('Inspecting cache file {pathFileName}'.format(pathFileName=pathFileName)) self.writeLog('Inspecting cache file {pathFileName}'.format(pathFileName=pathFileName))
if not os.path.isfile(pathFileName): if not os.path.isfile(pathFileName):
return(False) return False
modifiedTime=os.path.getmtime(pathFileName) modifiedTime = os.path.getmtime(pathFileName)
convertTime=time.localtime(modifiedTime) convertTime = time.localtime(modifiedTime)
formatTime=time.strftime('%d%m%Y %H:%M:%S',convertTime) formatTime = time.strftime('%d%m%Y %H:%M:%S', convertTime)
fileDateTime=time.strptime(formatTime,'%d%m%Y %H:%M:%S') fileDateTime = time.strptime(formatTime, '%d%m%Y %H:%M:%S')
currentTime=datetime.now() currentTime = datetime.now()
timedelta=currentTime-datetime(*(fileDateTime[0:6])) elapsed = currentTime - datetime(*(fileDateTime[0:6]))
hours, hremainder = divmod(timedelta.seconds,3600) totalSeconds = int(elapsed.total_seconds())
minutes, mremainder = divmod(timedelta.seconds,60) hours, remainder = divmod(totalSeconds, 3600)
minutes, _ = divmod(remainder, 60)
self.writeLog('file is = "{age}" hours old'.format(age=hours)) self.writeLog('file is = "{age}" hours old'.format(age=hours))
self.writeLog('file is = "{age}" minutes old'.format(age=minutes)) self.writeLog('file is = "{age}" minutes old'.format(age=minutes))
if hours > 1 or minutes > expireMinutes: if hours > 1 or minutes > expireMinutes:
self.archiveFile(pathFileName) self.archiveFile(pathFileName)
return(False) return False
return (True) return True
except: except:
self.writeLog(traceback.format_exc()); self.writeLog(traceback.format_exc())
return(False) return False
def archiveFile(self, pathFileName): def archiveFile(self, pathFileName):
if not os.path.isfile(pathFileName): if not os.path.isfile(pathFileName):
@@ -316,25 +337,35 @@ class Sections:
if "tokenvod" in previewUrl: if "tokenvod" in previewUrl:
return video, searchIndex return video, searchIndex
indexDescription=strContainingString.index("alt=\"") # Handle video description
indexDescription=strContainingString.find("alt=\"")
if -1 == indexDescription:
return video, searchIndex
description=strContainingString[indexDescription:] description=strContainingString[indexDescription:]
description=self.betweenString(description,'"','"') description=self.betweenString(description,'"','"')
description=self.removeHtml(description) description=self.removeHtml(description)
description=description.replace("- Fox News","") description=description.replace("- Fox News","")
if "vod.foxbusiness" in description: if "vod.foxbusiness" in description:
return video, searchIndex return video, searchIndex
indexDuration=strContainingString.index("<div class=\"duration\">")
# Handle video duration
indexDuration=strContainingString.find("<div class=\"duration\">")
if -1 != indexDuration: if -1 != indexDuration:
strDuration=strContainingString[indexDuration:] strDuration=strContainingString[indexDuration:]
strDuration=self.betweenString(strDuration,">","<") strDuration=self.betweenString(strDuration,">","<")
description=description+" - "+strDuration description=description+" - "+strDuration
indexPublication=strContainingString.index("<div class=\"pub-date\">")
# Handle video publication
strPublication = ""
indexPublication=strContainingString.find("<div class=\"pub-date\">")
if -1 != indexPublication: if -1 != indexPublication:
strPublication=strContainingString[indexPublication:] strPublication=strContainingString[indexPublication:]
strPublication=self.betweenString(strPublication,"<time>","</time>") strPublication=self.betweenString(strPublication,"<time>","</time>")
description=description+" ("+strPublication+")" description=description+" ("+strPublication+")"
# Handle the icon
icon=None icon=None
indexIcon=strContainingString.index("srcset=") indexIcon=strContainingString.find("srcset=")
if -1 != indexIcon: if -1 != indexIcon:
icon=strContainingString[indexIcon:] icon=strContainingString[indexIcon:]
icon=self.betweenString(icon,"\"","\"") icon=self.betweenString(icon,"\"","\"")