diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1e298ec --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +__pycache__/environment.cpython-39.pyc +__pycache__/utility.cpython-39.pyc +__pycache__/video.cpython-39.pyc +.vscode/settings.json diff --git a/Test.py b/Test.py new file mode 100755 index 0000000..cb722a0 --- /dev/null +++ b/Test.py @@ -0,0 +1,124 @@ +from datetime import datetime +from datetime import timedelta + +class DateTimeHelper: + def __init__(self): + pass + + def __init__(self,capturedatetime,timeChange): + self.datetimestamp=capturedatetime + self.timeChange=timeChange + self.offsetTime=DateTimeHelper.applyRelativeTime(self.datetimestamp, self.timeChange) + + def getDateTimeStamp(self): + return self.datetimestamp + + def getTimeChange(self): + return self.timeChange + + def getOffsetTime(self): + return self.offsetTime + + def getOffsetTimeAsString(self): + return DateTimeHelper.getDateTimeAsString(self.offsetTime) + + def toString(self): + pass + + @staticmethod + def getDateTimeAsString(someDateTime): + if(not isinstance(someDateTime,datetime)): + raise Exception('Invalid type for parameter') + return someDateTime.strftime("%m-%d-%Y %H:%M:%S") + + @staticmethod + def getDateTimeFromString(someDateTimeString): + if(not isinstance(someDateTimeString,str)): + raise Exception('Invalid type for parameter') + return datetime.strptime(someDateTimeString,"%m-%d-%Y %H:%M:%S") + + @staticmethod + def getCurrentDateTime(): + return datetime.now() + + @staticmethod + def applyRelativeTime(sometime,relativetime): + if(not isinstance(sometime,datetime)): + raise Exception('Invalid type for parameter') + if(not isinstance(relativetime,str)): + raise Exception('Invalid type for parameter') + if relativetime=='just now': + return sometime + relativetimesplit=relativetime.split() + if len(relativetimesplit)==2: + year=datetime.now().year + relativetime=relativetime+', '+str(year) + relativeDate = datetime.strptime(relativetime, '%B %d, %Y') + days=sometime-relativeDate + # sometime=sometime-timedelta(days=days) + sometime=sometime-days + elif relativetimesplit[1]=='hour' or relativetimesplit[1]=='hours': + hours=int(relativetimesplit[0]) + sometime=sometime-timedelta(hours=hours) + elif relativetimesplit[1]=='day' or relativetimesplit[1]=='days': + days=int(relativetimesplit[0]) + sometime=sometime-timedelta(days=days) + elif relativetimesplit[1]=='minute' or relativetimesplit[1]=='minutes': + minutes=int(relativetimesplit[0]) + sometime=sometime-timedelta(minutes=minutes) + return sometime + +list=[] + + +# date_time_str='03-03-2023 12:00:00' +# capturedatetime = datetime.strptime(date_time_str, '%m-%d-%Y %H:%M:%S') +# Item.applyRelativeTime(capturedatetime, '2 days ago') + +deltaTime='March 7' +now = datetime.now() +relativeTime=DateTimeHelper.applyRelativeTime(now, deltaTime) +relativeTimeStr=DateTimeHelper.getDateTimeAsString(relativeTime) + +print('time:{p1}'.format(p1=DateTimeHelper.getDateTimeAsString(now))) +print('delta:{p1}'.format(p1=deltaTime)) +print('result:{p1}'.format(p1=relativeTimeStr)) + + +#now = Item.getCurrentDateTime() +#capturedatetime = Item.getDateTimeAsString(now) + +#Item.applyRelativeTime(capturedatetime, 'February 24') + + +list.append(Item(capturedatetime,'February 24')) +list.append(Item(capturedatetime,'2 days ago')) +list.append(Item(capturedatetime,'21 days ago')) +list.append(Item(capturedatetime,'1 day ago')) +list.append(Item(capturedatetime,'1 minute ago')) +list.append(Item(capturedatetime,'5 minutes ago')) + +for item in list: + print (item) + + + +# date_time_str='01-01-2023 12:00:00' +# b = datetime.strptime(date_time_str, '%m-%d-%Y %H:%M:%S') + +# date_time_str='01-01-2023 18:00:00' +# c = datetime.strptime(date_time_str, '%m-%d-%Y %H:%M:%S') + +# list.append(a) +# list.append(b) +# list.append(c) + +# print(list) + +# another=sorted(list, key=lambda x:x) + +# print(another) +# for item in another: +# print('{:02d}-{:02d}-{:d} {:02d}:{:02d}:{:02d}'.format(item.month,item.day,item.year,item.hour,item.minute,item.second)) + + diff --git a/archive.py b/archive.py new file mode 100755 index 0000000..f6df653 --- /dev/null +++ b/archive.py @@ -0,0 +1,159 @@ +import os +import glob +import functools +from environment import * +from utility import * +from video import * + +# This file is executed in a cron job. +# To view the cron schedule type sudo crontab -r in a shell. Use Ctrl-S to save after editing +# This cron job should run evrry 30 minutes. Shorter intervals burden the system +# The ouptut from the print statements is generated in the syslog /var/log/syslog sudo nano /var/log/syslog +# Overall system perfromance can be monitored using htop + +def comparator(item1, item2): + try: + list1=item1.split('.') + list2=item2.split('.') + index1=int(list1[len(list1)-1]) + index2=int(list2[len(list2)-1]) + if index1index2: + return 1 + return 0 + except: + return 0 + +def createArchive(pathOutputFile,tokens,files): + lines=0 + unique={} + videos={} + + for token in tokens: + print('Filtering for "{token}"'.format(token=token)) + + videos = Video.load(pathOutputFile) + + for video in list(videos.values()): + description = description=createDescription(video.description,video.getTimestamp()) + if not description in unique: + unique[description]=createDescription(video.description, video.getTimestamp()) + + try: + print('found {count} archive files.'.format(count=len(files))) + print('processing {pathOutputFile}'.format(pathOutputFile=pathOutputFile)) + for file in files: + try: + with open(file, "r", encoding='utf-8') as inputStream: + for line in inputStream: + lowerLine=line.lower() + for token in tokens: + token=token.lower() + result = lowerLine.find(token) + if -1 != result: + video = Video.fromString(line) + heading = video.getDescription() + if not heading in unique: + unique[heading]=heading + video = Video.fromString(line) + video.description=createDescription(video.description,video.getTimestamp()) + videos[video.description]=video + lines = lines + 1 + inputStream.close() + except Exception as exception: + print('Exception reading {file} {exception}'.format(file=file,exception=exception)) + continue + print('writing {pathOutputFile}'.format(pathOutputFile=pathOutputFile)) + Video.write(pathOutputFile, videos) + except Exception as exception: + print('Exception creating output file {file} {exception}'.format(file=pathOutputFile,exception=exception)) + return + +# clean the archive files by removing files older than 'expiryDays' +def cleanArchive(files, expiryDays): + expiredList = [] + for pathFileName in files: + modification_date = os.path.getmtime(pathFileName) + modification_date = datetime.fromtimestamp(modification_date, timezone.utc) + now = DateTime.now() + days, hours, minutes, seconds = DateTime.deltaTime(modification_date, now) + if(days > expiryDays): + expiredList.append(pathFileName) + print('Expiring {count} files.'.format(count=len(expiredList))) + for file in expiredList: + os.remove(file) + return + +def createDescription(strDescription, timeStamp): + textElement=StringHelper.betweenString(strDescription,None,'-') + timeElement=StringHelper.betweenString(strDescription,'-',None) + durationElement=StringHelper.betweenString(timeElement,' ',' ') + newDescription=textElement+'-'+' '+ durationElement+' ('+timeStamp.toStringMonthDay()+')' + return newDescription + +def getFiles(archiveFileLike): + files = glob.glob(archiveFileLike) + files=files+glob.glob(archiveFileLike+'.*') + return files + +# This program runs through all of the videodb*.txt files looking for keywords with which to +# build each of the individually named mini-archives. +# 1) Search for all videodb.txt.* files +# 2) Expire files older than specified number of days +# 3) Load the archive (for each fo the types enumerated below) +# 4) Run through file collection for the given archive archive and append to the archive as tags are found +# 5) Sort the archive +# 6) Truncate existing archive if it exists +# 7) Write the new archive + +path=PATH_VIDEO_DATABASE +archiveFile=path+'/videodb' +archiveFileLike=archiveFile+'.txt' + +#For debugging +# path='/home/pi/Projects/Python/NewsFeed/Archive' +# archiveFile=path+'/videodb' +# archiveFileLike=archiveFile+'.txt' + +files = getFiles(archiveFileLike) +print('There are {count} archive files to process before cleaning'.format(count=len(files))) +cleanArchive(files, 30) +files = getFiles(archiveFileLike) +print('There are {count} archive files to process after cleaning'.format(count=len(files))) + +print('archive.py running...') + +archiveFileName=ARCHIVEDB_FILENAME +pathOutputFile=PathHelper.makePathFileName(archiveFileName,path) +print('pathOutputFile={pathOutputFile}'.format(pathOutputFile=pathOutputFile)) +tokens=["Keane","Jesse","Israel","Hamas"," War ","Iran","Hezzbollah","Gaza","Ukraine"] +createArchive(pathOutputFile,tokens,files) + +hannityFileName=HANNITYARCHIVEDB_FILENAME +pathOutputFile=PathHelper.makePathFileName(hannityFileName,path) +print('pathOutputFile={pathOutputFile}'.format(pathOutputFile=pathOutputFile)) +tokens=["Hannity"] +createArchive(pathOutputFile,tokens,files) + +levinFileName=LEVINARCHIVEDB_FILENAME +pathOutputFile=PathHelper.makePathFileName(levinFileName,path) +print('pathOutputFile={pathOutputFile}'.format(pathOutputFile=pathOutputFile)) +tokens=["Levin"] +createArchive(pathOutputFile,tokens,files) + +hawleyFileName=HAWLEYARCHIVEDB_FILENAME +pathOutputFile=PathHelper.makePathFileName(hawleyFileName,path) +print('pathOutputFile={pathOutputFile}'.format(pathOutputFile=pathOutputFile)) +tokens=["Hawley"] +createArchive(pathOutputFile,tokens,files) + +militaryFileName=MILITARYARCHIVEDB_FILENAME +pathOutputFile=PathHelper.makePathFileName(militaryFileName,path) +print('pathOutputFile={pathOutputFile}'.format(pathOutputFile=pathOutputFile)) +tokens=["Keane","Kellogg","Russia","Ukraine","Israel","Korea","Iran","Venezuela","Cuba","China"] +createArchive(pathOutputFile,tokens,files) + +print('archive.py done.') + + diff --git a/environment.py b/environment.py new file mode 100755 index 0000000..0f39213 --- /dev/null +++ b/environment.py @@ -0,0 +1,49 @@ +APPEND_SYS_PATH='/home/pi/.kodi/addons/plugin.video.fox.news/resources/lib' +PATH_VIDEO_DATABASE='/home/pi/.kodi/addons/plugin.video.fox.news/resources/lib/archive' +USE_ICON_URL=True +PATH_LOG_FILE="/home/pi/.kodi/temp/MyLog.log" +FOX_NEWS_URL="https://www.foxnews.com/video" +FOX_NEWS_US_URL="hhttps://www.foxnews.com/video/topics/us" +FOX_NEWS_EXCLUSIVE_URL="https://foxnews.com" +FOX_NEWS_AMERICAS_NEWSROOM_URL="https://www.foxnews.com/video/shows/americas-newsroom" +FOX_NEWS_OUTNUMBERED_URL="https://www.foxnews.com/video/shows/outnumbered" +SELECTED_ARCHIVE_URL="https://www.foxnews.com/video/dummy" +LEVIN_ARCHIVE_URL="https://www.foxnews.com/video/levindummyurl" +HANNITY_ARCHIVE_URL="https://www.foxnews.com/video/hannitydummyurl" +HAWLEY_ARCHIVE_URL="https://www.foxnews.com/video/hawleydummyurl" +MILITARY_ARCHIVE_URL="https://www.foxnews.com/video/militarydummyurl" + +#FOX_NEWS_ICON_OF_LAST_RESORT="https://static.foxnews.com/static/orion/styles/img/fox-news/favicons/apple-touch-icon-180x180.png" +#ENABLE_USE_ICON_OF_LAST_RESORT=False + + +VIDEODB_AMERICAS_NEWSROOM_FILENAME="videodb_americasnewsroom.txt" +VIDEODB_OUTNUMBERED_FILENAME="videodb_outnumbered.txt" +VIDEODB_FILENAME="videodb.txt" +VIDEODB_EXCLUSIVE_FILENAME="videodb_exc.txt" +VIDEODB_US_FILENAME="videodb_us.txt" +ARCHIVEDB_FILENAME="archivedb.txt" +LEVINARCHIVEDB_FILENAME="levindb.txt" +HANNITYARCHIVEDB_FILENAME="hannitydb.txt" +HAWLEYARCHIVEDB_FILENAME="hawleydb.txt" +MILITARYARCHIVEDB_FILENAME="militarydb.txt" + +CACHE_EXPIRY_MINS=10 + +LOG_HTTP_RESPONSES = False + +FEED_REJECT_IF_OLDER_THAN_DAYS = 7 + +class PathHelper: + pathChar="/" + + def __init__(self): + pass + + @staticmethod + def makePathFileName(file,path): + while file.endswith(PathHelper.pathChar): + file=file[:-1] + while path.endswith(PathHelper.pathChar): + path=path[:-1] + return path+PathHelper.pathChar+file diff --git a/newsfeed.py b/newsfeed.py new file mode 100755 index 0000000..5516dee --- /dev/null +++ b/newsfeed.py @@ -0,0 +1,544 @@ +import json +import os +import webbrowser +import requests +import traceback +import time +import re +import glob +import shutil +from datetime import timedelta +from datetime import datetime +from datetime import timezone +from environment import * +from utility import * +from video import * + +class NewsFeed: + def __init__(self, pathDb, logger=None): + self.pathDb=pathDb + self.logger=logger + + @staticmethod + def isResourceAvailable(url): + try: + response=requests.head(url, timeout=2.5) + if not response.ok: + return False + return True + except: + return False + + def getItemsInAmericasNewsRoomFeed(self,url): + now=datetime.now() + cachePathFileName=PathHelper.makePathFileName(VIDEODB_AMERICAS_NEWSROOM_FILENAME,self.pathDb) + if self.isFeedCacheAvailable(cachePathFileName,CACHE_EXPIRY_MINS): + videos=self.readFeedCache(cachePathFileName) + if videos is not None: + return(videos) + sections=Sections() + videos = {} + httpNetRequest=HttpNetRequest() + response=httpNetRequest=httpNetRequest.getHttpNetRequest(url) + status=response.status_code + searchIndex=0 + response.close() + if status!=200: + return None + if LOG_HTTP_RESPONSES: + self.writeLog(url) + self.writeLog(response.text) + while -1!= searchIndex: + video, searchIndex = sections.getItemsInSection(response.text,"article",searchIndex) + if video is not None and not (video.description in videos): + videos[video.description]=video + video.setFeedTime(DateTimeHelper.applyRelativeTime(now,video.feedTimeOffset)) + videoList=list(videos.values()) + videoList=sorted(videoList, key=lambda x:x.getFeedTime(),reverse=False) + self.writeFeedCache(cachePathFileName,videoList) + return (videoList) + + def getItemsInOutnumberedFeed(self,url): + now=datetime.now() + cachePathFileName=PathHelper.makePathFileName(VIDEODB_OUTNUMBERED_FILENAME,self.pathDb) + if self.isFeedCacheAvailable(cachePathFileName,CACHE_EXPIRY_MINS): + videos=self.readFeedCache(cachePathFileName) + if videos is not None: + return(videos) + sections=Sections() + videos = {} + httpNetRequest=HttpNetRequest() + response=httpNetRequest=httpNetRequest.getHttpNetRequest(url) + status=response.status_code + searchIndex=0 + response.close() + if status!=200: + return None + if LOG_HTTP_RESPONSES: + self.writeLog(url) + self.writeLog(response.text) + while -1!= searchIndex: + video, searchIndex = sections.getItemsInSection(response.text,"article",searchIndex) + if video is not None and not (video.description in videos): + videos[video.description]=video + video.setFeedTime(DateTimeHelper.applyRelativeTime(now,video.feedTimeOffset)) + videoList=list(videos.values()) + videoList=sorted(videoList, key=lambda x:x.getFeedTime(),reverse=True) + self.writeFeedCache(cachePathFileName,videoList) + return (videoList) + + def getItemsInFeed(self,url): + now=datetime.now() + cachePathFileName=PathHelper.makePathFileName(VIDEODB_FILENAME,self.pathDb) + if self.isFeedCacheAvailable(cachePathFileName,CACHE_EXPIRY_MINS): + self.writeLog(f"Loading videos from cache {cachePathFileName}") + videos=self.readFeedCache(cachePathFileName) + if videos is not None: + return(videos) + sections=Sections() + videos = {} + httpNetRequest=HttpNetRequest() + self.writeLog(f"Loading videos from {url}") + response=httpNetRequest=httpNetRequest.getHttpNetRequest(url) + status=response.status_code + searchIndex=0 + response.close() + if status!=200: + return None + if LOG_HTTP_RESPONSES: + self.writeLog(url) + self.writeLog(response.text) + while -1!= searchIndex: + video, searchIndex= sections.getItemsInSection(response.text,"article",searchIndex) + if video is not None and not (video.description in videos): + videos[video.description]=video + video.setFeedTime(DateTimeHelper.applyRelativeTime(now,video.feedTimeOffset)) +# videoList=list(videos.values()) + videoList=self.filterFeedMaxDays(list(videos.values()),FEED_REJECT_IF_OLDER_THAN_DAYS) + videoList=sorted(videoList, key=lambda x:x.getFeedTime(),reverse=True) + self.writeFeedCache(cachePathFileName,videoList) + return (videoList) + + def filterFeedMaxDays(self, videos, days): + now = datetime.now() + filteredList=[] + for video in videos: + delta = now - video.getFeedTime() + if delta.days <= days: + message = f"INCL. days={delta.days},feed time={video.getFeedTime()} feed time offset (strPublication)=:'{video.feedTimeOffset}', description={video.description}" + self.writeLog(message) + filteredList.insert(0,video) + else: + message = f"EXCL. days={delta.days},feed time={video.getFeedTime()} feed time offset (strPublication)=:'{video.feedTimeOffset}', description={video.description}" + self.writeLog(message) + return filteredList + + def getUSItemsInFeed(self,url): + now=datetime.now() + cachePathFileName=PathHelper.makePathFileName(VIDEODB_US_FILENAME,self.pathDb) + if self.isFeedCacheAvailable(cachePathFileName,CACHE_EXPIRY_MINS): + videos=self.readFeedCache(cachePathFileName) + if videos is not None: + return(videos) + sections=Sections() + videos = {} + httpNetRequest=HttpNetRequest() + response=httpNetRequest.getHttpNetRequest(url) + status=response.status_code + searchIndex=0 + response.close() + if status!=200: + return None + if LOG_HTTP_RESPONSES: + self.writeLog(url) + self.writeLog(response.text) + while -1!= searchIndex: + videoId, searchIndex = sections.getVideoIdInSection(response.text,"article",searchIndex) + if videoId is None: + continue + url='https://video.foxnews.com/v/'+videoId + httpNetRequest=HttpNetRequest() + innerResponse=httpNetRequest.getHttpNetRequest(url) + status=innerResponse.status_code + innerResponse.close() + if status!=200: + continue + video=sections.getVideoContentInSection(innerResponse.text) + if video is not None and not (video.description in videos): + videos[video.description]=video + video.setFeedTime(DateTimeHelper.applyRelativeTime(now,video.feedTimeOffset)) + videoList=list(videos.values()) + videoList=sorted(videoList, key=lambda x:x.getFeedTime(),reverse=True) + self.writeFeedCache(cachePathFileName,videoList) + return (videoList) + + def getExclusiveItemsInFeed(self,url): + now=datetime.now() + cachePathFileName=PathHelper.makePathFileName(VIDEODB_EXCLUSIVE_FILENAME,self.pathDb) + if self.isFeedCacheAvailable(cachePathFileName,CACHE_EXPIRY_MINS): + videos=self.readFeedCache(cachePathFileName) + if videos is not None: + return(videos) + sections=Sections() + videos = {} + httpNetRequest=HttpNetRequest() + response=httpNetRequest.getHttpNetRequest(url) + status=response.status_code + searchIndex=0 + response.close() + if status!=200: + return None + if LOG_HTTP_RESPONSES: + self.writeLog(url) + self.writeLog(response.Text) + while -1!= searchIndex: + videoId, searchIndex = sections.getVideoIdInSection(response.text,"article",searchIndex) + if videoId is None: + continue + url='https://video.foxnews.com/v/'+videoId + httpNetRequest=HttpNetRequest() + innerResponse=httpNetRequest.getHttpNetRequest(url) + status=innerResponse.status_code + innerResponse.close() + if status!=200: + continue + video=sections.getVideoContentInSection(innerResponse.text) + if video is not None and not (video.description in videos): + videos[video.description]=video + video.setFeedTime(DateTimeHelper.applyRelativeTime(now,video.feedTimeOffset)) + videoList=list(videos.values()) + videoList=sorted(videoList, key=lambda x:x.getFeedTime(),reverse=True) + self.writeFeedCache(cachePathFileName,videoList) + return (videoList) + + def getItemsInArchiveFeed(self,url,archiveDbFileName): + cachePathFileName=PathHelper.makePathFileName(archiveDbFileName,self.pathDb) + videos=self.readFeedCache(cachePathFileName) + if videos is not None: + return(videos) + return(None) + + def readFeedCache(self,pathFileName): + try: + videos=[] + with open(pathFileName,"r",encoding='utf-8') as inputStream: + for line in inputStream: + video=Video.fromString(line) + videos.append(video) + inputStream.close() + return(videos) + except: + self.writeLog(traceback.format_exc()) + return(None) + + def writeFeedCache(self,pathFileName,videos): + try: + with open(pathFileName,"w",encoding='utf-8') as outputStream: + for video in videos: + outputStream.write(video.toString()+"\n") + outputStream.close() + return(videos) + except: + self.writeLog(traceback.format_exc()) + return(videos) + + def isFeedCacheAvailable(self,pathFileName,expireMinutes): + try: + self.writeLog('Inspecting cache file {pathFileName}'.format(pathFileName=pathFileName)) + if not os.path.isfile(pathFileName): + return(False) + modifiedTime=os.path.getmtime(pathFileName) + convertTime=time.localtime(modifiedTime) + formatTime=time.strftime('%d%m%Y %H:%M:%S',convertTime) + fileDateTime=DateTimeHelper.strptime(formatTime,'%d%m%Y %H:%M:%S') + currentTime=datetime.now() + timedelta=currentTime-fileDateTime + hours, hremainder = divmod(timedelta.seconds,3600) + minutes, mremainder = divmod(timedelta.seconds,60) + self.writeLog('file is = "{age}" hours old'.format(age=hours)) + self.writeLog('file is = "{age}" minutes old'.format(age=minutes)) + if hours > 1 or minutes > expireMinutes: + self.archiveFile(pathFileName) + return(False) + return (True) + except: + self.writeLog(traceback.format_exc()); + return(False) + + def archiveFile(self, pathFileName): + if not os.path.isfile(pathFileName): + return(False) + archiveFile=StringHelper.betweenString(pathFileName, None, '.txt') + archiveFileLike=archiveFile+'.txt.*' + files = glob.glob(archiveFileLike) + index=len(files)+1 + archiveFileName=archiveFile+'.txt.'+str(index) + print('archiveFile: Copying "{pathFileName}" to "{archiveFileName}".'.format(pathFileName=pathFileName,archiveFileName=archiveFileName)) + shutil.copy(pathFileName,archiveFileName) + os.remove(pathFileName) + return(True) + + def writeLog(self,message): + if self.logger is not None: + self.logger.write(message) + else: + print(message) + +class Sections: + def __init__(self): + self.dummy=None + + def getItemsInSection(self, strInput, sectionName, searchIndex): + video=None + startSection='<'+sectionName + endSection='") + if -1 != indexDuration: + strDuration=strContainingString[indexDuration:] + strDuration=self.betweenString(strDuration,">","<") + description=description+" - "+strDuration + indexPublication=strContainingString.index("
") + if -1 != indexPublication: + strPublication=strContainingString[indexPublication:] + strPublication=self.betweenString(strPublication,"") + description=description+" ("+strPublication+")" + icon=None + indexIcon=strContainingString.index("srcset=") + if -1 != indexIcon: + icon=strContainingString[indexIcon:] + icon=self.betweenString(icon,"\"","\"") + splits=icon.split(',') + icon=self.betweenString(splits[len(splits)-1],None,'?') + icon=icon.strip() + description = description.strip() + video=Video(description,previewUrl,icon) + video.feedTimeOffset=strPublication + return video, searchIndex + + def getVideoIdInSection(self, strInput, sectionName, searchIndex): + video=None + startSection='<'+sectionName + endSection='=length: + return str + while stringLength < length: + sb=sb+filler + stringLength=stringLength+1 + return sb+str + +def parseDuration(strDuration): + expression=re.compile(r"\d+") + result=expression.findall(strDuration) + if 2!=len(result): + return None, None + return pad(result[0],'0',2), pad(result[1],'0',2) + + +# DON'T LEAVE ANYTHING OPEN BELOW THIS LINE BECAUSE THIS FILE IS IMPORTED BY OTHER MODULES AND ANY CODE NOT IN A CLASS WILL BE RUN + +#print(FOX_NEWS_URL) +# pathFileName='/home/pi/.kodi/addons/plugin.video.fox.news/resources/lib/videodb.txt' +# newsFeed=NewsFeed('/home/pi/.kodi/addons/plugin.video.fox.news/resources/lib/') +# newsFeed.ArchiveFile(pathFileName) + + +# pathFileName='/home/pi/.kodi/addons/plugin.video.fox.news/resources/lib/videodb.txt' +# modifiedTime=os.path.getmtime(pathFileName) +# convertTime=time.localtime(modifiedTime) +# formatTime=time.strftime('%d%m%Y %H:%M:%S',convertTime) +# fileDateTime=DateTimeHelper.strptime(formatTime) + +#fileDateTime=datetime.strptime(formatTime,'%d%m%Y %H:%M:%S') +#fileDateTime2=datetime(*(time.strptime(formatTime,'%d%m%Y %H:%M:%S')[0:6])) +#currentTime=datetime.now() + +#Test the main feed +# newsFeed=NewsFeed('/home/pi/Projects/Python/NewsFeed/') +# newsFeed=NewsFeed('/home/pi/.kodi/addons/plugin.video.fox.news/resources/lib/videodb.txt') +# newsFeed=NewsFeed(PATH_VIDEO_DATABASE, myLog()) +# newsFeed=NewsFeed('/home/pi/Projects/Python/NewsFeed/', myLog()) +# videos=newsFeed.getItemsInFeed(FOX_NEWS_URL) +# for video in videos: +# if(video.description.startswith("Martha")): +# print(f"Description={video.description}") +# print(f"Url={video.url}") +# print(f"getTimestamp={video.getTimestamp().toStringMonthDay()}") +# print(f"getFeedTimeOffset={video.getFeedTimeOffset()}") +# print(f"getFeedTime={video.getFeedTime()}") +# print(f"daysOld={(datetime.now()-video.getFeedTime()).days}") +# print(' ') + +# pull the time out of the description and subtract it from the time we scanned the feed. +# the result will be the time of the article..use this to sort on. +# (i.e.) FeedTime:02/03/2023 12:00:00 Article Time:2 hours ago Real time:10:00:00 + + +#Test the exclusive items feed +#newsFeed=NewsFeed('/home/pi/.kodi/addons/plugin.video.fox.news/resources/lib/') +#videos=newsFeed.getExclusiveItemsInFeed("https://www.foxnews.com") +# for video in videos: +# print(video.description) + + +# Test the U.S. Feed +# newsFeed=NewsFeed('/home/pi/.kodi/addons/plugin.video.fox.news/resources/lib/') +# videos=newsFeed.getUSItemsInFeed("https://www.foxnews.com/video/topics/us") +# for video in videos: +# print(video.description) + +# Test the America's NewsRoom Feed +# newsFeed=NewsFeed('/home/pi/.kodi/addons/plugin.video.fox.news/resources/lib/') +# videos=newsFeed.getItemsInAmericasNewsRoomFeed("https://www.foxnews.com/video/shows/americas-newsroom") +# print('got {count} videos for America''s Newsroom'.format(count=len(videos))) +# for video in videos: +# print(video.description) +# print(video.url) + +# Test the Outnumbered Feed +# newsFeed=NewsFeed('/home/pi/.kodi/addons/plugin.video.fox.news/resources/lib/') +# videos=newsFeed.getItemsInOutnumbereFeed("https://www.foxnews.com/video/shows/outnumbered") +# print('got {count} videos for Outnumbered'.format(count=len(videos))) +# for video in videos: +# print(video.description) +# print(video.url) + +#minutes, seconds = parseDuration('PT24M5S') +#print('Duration is {minutes}:{seconds}'.format(minutes=minutes,seconds=seconds)) + +# isoDate="2022-10-27T10:24:11Z".replace("Z","+00:00") +# articleTime=datetime.datetime.fromisoformat(isoDate) +# print('time:{time}'.format(time=articleTime)) +# currentTime=Date.getCurrentTime() +# print('time:{time}'.format(time=currentTime)) +# days, hours, minutes, seconds=Date.deltaTime(articleTime,currentTime) +# print('elapsed time {days} days, {hours} hours, {minutes} minutes, {seconds} seconds'.format(days=days,hours=hours,minutes=minutes,seconds=seconds)) + +# currentTime2=Date.getCurrentTime() +# strCurrentTime2=str(currentTime2) +# currentTime2=datetime.datetime.fromisoformat(strCurrentTime2) +# days, hours, minutes, seconds=Date.deltaTime(currentTime2,currentTime) +# print('elapsed time {days} days, {hours} hours, {minutes} minutes, {seconds} seconds'.format(days=days,hours=hours,minutes=minutes,seconds=seconds)) + +# dateList=[] + +# currentDate=Date() +# dateList.append(currentDate) +# currentDate2=Date() +# dateList.append(currentDate2) + +# dateList.sort(key=lambda x:x.toString()) +# for date in dateList: +# print(date.toString()) +# #print(dateList) diff --git a/scraper.py b/scraper.py new file mode 100755 index 0000000..702d91a --- /dev/null +++ b/scraper.py @@ -0,0 +1,183 @@ +# -*- coding: utf-8 -*- +# Fox News Kodi Video Addon +# + +APPEND_SYS_PATH='/home/pi/.kodi/addons/plugin.video.fox.news/resources/lib' + +import os +import sys +import traceback +sys.path.append(APPEND_SYS_PATH) +from t1mlib import t1mAddon +import json +import re +import xbmc +import xbmcplugin +import xbmcgui +import xbmcvfs +import xbmc +import xbmcaddon +import html.parser +import sys +import datetime +import time +import random +import requests +import sqlite3 +from newsfeed import NewsFeed +from functools import reduce +from simplecache import SimpleCache +from environment import * +from utility import * + +#class myLog(): +# def __init__(self): +# self._file = open(PATH_LOG_FILE,"w",encoding='utf-8') +# +# def write(self,item): +# self._file.write(item) +# self._file.write("\n") +# self._file.flush() + +class myAddon(t1mAddon): + + def __init__(self, aname): + t1mAddon.__init__(self, aname) + self._logfile = myLog() + self._cache = SimpleCache() + self._pDialog = xbmcgui.DialogProgressBG() + + def getAddonMenu(self,url,ilist): + try: + self._logfile.write('getAddonMenu') + ilist = self.addMenuItem("Latest Fox News Featured Clips",'GE', ilist, FOX_NEWS_URL, self.addonIcon, self.addonFanart, {}, isFolder=True) + ilist = self.addMenuItem("Fox News Outnumbered Clips",'GE', ilist, FOX_NEWS_OUTNUMBERED_URL, self.addonIcon, self.addonFanart, {}, isFolder=True) + ilist = self.addMenuItem("Selected Archive Clips",'GE', ilist, SELECTED_ARCHIVE_URL, self.addonIcon, self.addonFanart, {}, isFolder=True) + ilist = self.addMenuItem("Mark Levin Archive Clips",'GE', ilist, LEVIN_ARCHIVE_URL, self.addonIcon, self.addonFanart, {}, isFolder=True) + ilist = self.addMenuItem("Sean Hannity Archive Clips",'GE', ilist, HANNITY_ARCHIVE_URL, self.addonIcon, self.addonFanart, {}, isFolder=True) + ilist = self.addMenuItem("Josh Hawley Archive Clips",'GE', ilist, HAWLEY_ARCHIVE_URL, self.addonIcon, self.addonFanart, {}, isFolder=True) + ilist = self.addMenuItem("Military Archive Clips",'GE', ilist, MILITARY_ARCHIVE_URL, self.addonIcon, self.addonFanart, {}, isFolder=True) + return(ilist) + except: + self._logfile.write(traceback.format_exc()) + raise + finally: + return(ilist) + + def getAddonEpisodes(self,url,ilist): + try: + if url == FOX_NEWS_URL: + self._logfile.write('getAddonEpisodes url={url}'.format(url=url)) + self._pDialog.create('Retrieving News Articles...') + newsFeed=NewsFeed(PATH_VIDEO_DATABASE, self._logfile) + videos=newsFeed.getItemsInFeed(url) + for video in videos: + if USE_ICON_URL: + ilist = self.addMenuItem(video.description,'GV', ilist, video.url, video.icon, self.addonFanart, {}, isFolder=False) + else: + ilist = self.addMenuItem(video.description,'GV', ilist, video.url, self.addonIcon, self.addonFanart, {}, isFolder=False) + self._logfile.write('Processed {articles} articles.'.format(articles=len(videos))) + elif url == SELECTED_ARCHIVE_URL: + self._logfile.write('getAddonEpisodes url={url}'.format(url=url)) + self._pDialog.create('Retrieving News Articles...') + newsFeed=NewsFeed(PATH_VIDEO_DATABASE, self._logfile) + videos=newsFeed.getItemsInArchiveFeed(url,ARCHIVEDB_FILENAME) + for video in videos: + if USE_ICON_URL: + ilist = self.addMenuItem(video.description,'GV', ilist, video.url, video.icon, self.addonFanart, {}, isFolder=False) + else: + ilist = self.addMenuItem(video.description,'GV', ilist, video.url, self.addonIcon, self.addonFanart, {}, isFolder=False) + self._logfile.write('Processed {articles} articles.'.format(articles=len(videos))) + elif url == LEVIN_ARCHIVE_URL: + self._logfile.write('getAddonEpisodes url={url}'.format(url=url)) + self._pDialog.create('Retrieving News Articles...') + newsFeed=NewsFeed(PATH_VIDEO_DATABASE, self._logfile) + videos=newsFeed.getItemsInArchiveFeed(url,LEVINARCHIVEDB_FILENAME) + for video in videos: + if USE_ICON_URL: + ilist = self.addMenuItem(video.description,'GV', ilist, video.url, video.icon, self.addonFanart, {}, isFolder=False) + else: + ilist = self.addMenuItem(video.description,'GV', ilist, video.url, self.addonIcon, self.addonFanart, {}, isFolder=False) + self._logfile.write('Processed {articles} articles.'.format(articles=len(videos))) + elif url == HANNITY_ARCHIVE_URL: + self._logfile.write('getAddonEpisodes url={url}'.format(url=url)) + self._pDialog.create('Retrieving News Articles...') + newsFeed=NewsFeed(PATH_VIDEO_DATABASE, self._logfile) + videos=newsFeed.getItemsInArchiveFeed(url,HANNITYARCHIVEDB_FILENAME) + for video in videos: + if USE_ICON_URL: + ilist = self.addMenuItem(video.description,'GV', ilist, video.url, video.icon, self.addonFanart, {}, isFolder=False) + else: + ilist = self.addMenuItem(video.description,'GV', ilist, video.url, self.addonIcon, self.addonFanart, {}, isFolder=False) + self._logfile.write('Processed {articles} articles.'.format(articles=len(videos))) + elif url == HAWLEY_ARCHIVE_URL: + self._logfile.write('getAddonEpisodes url={url}'.format(url=url)) + self._pDialog.create('Retrieving News Articles...') + newsFeed=NewsFeed(PATH_VIDEO_DATABASE, self._logfile) + videos=newsFeed.getItemsInArchiveFeed(url,HAWLEYARCHIVEDB_FILENAME) + for video in videos: + if USE_ICON_URL: + ilist = self.addMenuItem(video.description,'GV', ilist, video.url, video.icon, self.addonFanart, {}, isFolder=False) + else: + ilist = self.addMenuItem(video.description,'GV', ilist, video.url, self.addonIcon, self.addonFanart, {}, isFolder=False) + self._logfile.write('Processed {articles} articles.'.format(articles=len(videos))) + elif url == MILITARY_ARCHIVE_URL: + self._logfile.write('getAddonEpisodes url={url}'.format(url=url)) + self._pDialog.create('Retrieving News Articles...') + newsFeed=NewsFeed(PATH_VIDEO_DATABASE, self._logfile) + videos=newsFeed.getItemsInArchiveFeed(url,MILITARYARCHIVEDB_FILENAME) + for video in videos: + if USE_ICON_URL: + ilist = self.addMenuItem(video.description,'GV', ilist, video.url, video.icon, self.addonFanart, {}, isFolder=False) + else: + ilist = self.addMenuItem(video.description,'GV', ilist, video.url, self.addonIcon, self.addonFanart, {}, isFolder=False) + self._logfile.write('Processed {articles} articles.'.format(articles=len(videos))) + elif url== FOX_NEWS_OUTNUMBERED_URL: + self._logfile.write('getAddonEpisodes url={url}'.format(url=url)) + self._pDialog.create('Retrieving Outnumbered Articles...') + newsFeed=NewsFeed(PATH_VIDEO_DATABASE, self._logfile) + videos=newsFeed.getItemsInOutnumberedFeed(url) + for video in videos: + if USE_ICON_URL: + ilist = self.addMenuItem(video.description,'GV', ilist, video.url, video.icon, self.addonFanart, {}, isFolder=False) + else: + ilist = self.addMenuItem(video.description,'GV', ilist, video.url, self.addonIcon, self.addonFanart, {}, isFolder=False) + self._logfile.write('Processed {articles} articles.'.format(articles=len(videos))) + except Exception as exception: + self._logfile.write('Exception:{exception}'.format(exception=exception)) + self._logfile.write(traceback.format_exc()) + raise + finally: + self._pDialog.close() + return(ilist) + + def getAddonMovies(self,url,ilist): + try: + self._logfile.write('getAddonMovies url={url}'.format(url=url)) + self._pDialog.create('getAddonMovies...') + self._pDialog.update(0,message='getAddonMovies...') + self._pDialog.close() + return(ilist) + except Exception as exception: + self._logfile.write('Exception:{exception}'.format(exception=exception)) + self._logfile.write(traceback.format_exc()) + raise + finally: + self._pDialog.close() + return(ilist) + + def getAddonShows(self,url,ilist): + try: + self._logfile.write('getAddonShows') + self._pDialog.create('getAddonShows...') + self._pDialog.update(0,message='getAddonShows...') + self._pDialog.close() + return ilist + except Exception as exception: + self._logfile.write('Exception:{exception}'.format(exception=exception)) + self._logfile.write(traceback.format_exc()) + raise + finally: + self._pDialog.close() + return(ilist) + diff --git a/simplecache.py b/simplecache.py new file mode 100755 index 0000000..bddbff4 --- /dev/null +++ b/simplecache.py @@ -0,0 +1,302 @@ +# -*- coding: utf-8 -*- +# Fox News Kodi Video Addon +# +import os +import sys +import traceback +from t1mlib import t1mAddon +import json +import re +import xbmc +import xbmcplugin +import xbmcgui +import xbmcvfs +import xbmc +import xbmcaddon +import html.parser +import sys +import datetime +import time +import random +import requests +import sqlite3 + +ADDON_ID = "plugin.video.fox.news" + +class SimpleCache(object): + '''simple stateless caching system for Kodi''' + enable_mem_cache = True + global_checksum = None + _exit = False + _auto_clean_interval = datetime.timedelta(hours=4) + _win = None + _busy_tasks = [] + _database = None + + def __init__(self): + '''Initialize our caching class''' + self._win = xbmcgui.Window(10000) + self._monitor = xbmc.Monitor() + self.check_cleanup() + self._log_msg("Initialized") + + def close(self): + '''tell any tasks to stop immediately (as we can be called multithreaded) and cleanup objects''' + self._exit = True + # wait for all tasks to complete + while self._busy_tasks and not self._monitor.abortRequested(): + xbmc.sleep(25) + del self._win + del self._monitor + self._log_msg("Closed") + + def __del__(self): + '''make sure close is called''' + if not self._exit: + self.close() + + def get(self, endpoint, checksum=""): + ''' + get object from cache and return the results + endpoint: the (unique) name of the cache object as reference + checkum: optional argument to check if the checksum in the cacheobject matches the checkum provided + ''' + checksum = self._get_checksum(checksum) + cur_time = self._get_timestamp(datetime.datetime.now()) + result = None + # 1: try memory cache first + if self.enable_mem_cache: + result = self._get_mem_cache(endpoint, checksum, cur_time) + + # 2: fallback to _database cache + if result is None: + result = self._get_db_cache(endpoint, checksum, cur_time) + + return result + + def set(self, endpoint, data, checksum="", expiration=datetime.timedelta(days=30)): + ''' + set data in cache + ''' + task_name = "set.%s" % endpoint + self._busy_tasks.append(task_name) + checksum = self._get_checksum(checksum) + expires = self._get_timestamp(datetime.datetime.now() + expiration) + + # memory cache: write to window property + if self.enable_mem_cache and not self._exit: + self._set_mem_cache(endpoint, checksum, expires, data) + + # db cache + if not self._exit: + self._set_db_cache(endpoint, checksum, expires, data) + + # remove this task from list + self._busy_tasks.remove(task_name) + + def check_cleanup(self): + '''check if cleanup is needed - public method, may be called by calling addon''' + cur_time = datetime.datetime.now() + lastexecuted = self._win.getProperty("simplecache.clean.lastexecuted") + if not lastexecuted: + self._win.setProperty("simplecache.clean.lastexecuted", repr(cur_time)) + elif (eval(lastexecuted) + self._auto_clean_interval) < cur_time: + # cleanup needed... + self._do_cleanup() + + def _get_mem_cache(self, endpoint, checksum, cur_time): + ''' + get cache data from memory cache + we use window properties because we need to be stateless + ''' + result = None + cachedata = self._win.getProperty(endpoint) + + if cachedata: + cachedata = eval(cachedata) + if cachedata[0] > cur_time: + if not checksum or checksum == cachedata[2]: + result = cachedata[1] + return result + + def _set_mem_cache(self, endpoint, checksum, expires, data): + ''' + window property cache as alternative for memory cache + usefull for (stateless) plugins + ''' + cachedata = (expires, data, checksum) + cachedata_str = repr(cachedata) + self._win.setProperty(endpoint, cachedata_str) + + + def _get_db_cache(self, endpoint, checksum, cur_time): + '''get cache data from sqllite _database''' + result = None + query = "SELECT expires, data, checksum FROM simplecache WHERE id = ?" + cache_data = self._execute_sql(query, (endpoint,)) + if cache_data: + cache_data = cache_data.fetchone() + if cache_data and cache_data[0] > cur_time: + if not checksum or cache_data[2] == checksum: + result = eval(cache_data[1]) + # also set result in memory cache for further access + if self.enable_mem_cache: + self._set_mem_cache(endpoint, checksum, cache_data[0], result) + return result + + def _set_db_cache(self, endpoint, checksum, expires, data): + ''' store cache data in _database ''' + query = "INSERT OR REPLACE INTO simplecache( id, expires, data, checksum) VALUES (?, ?, ?, ?)" + data = repr(data) + self._execute_sql(query, (endpoint, expires, data, checksum)) + + def _do_cleanup(self): + '''perform cleanup task''' + if self._exit or self._monitor.abortRequested(): + return + self._busy_tasks.append(__name__) + cur_time = datetime.datetime.now() + cur_timestamp = self._get_timestamp(cur_time) + self._log_msg("Running cleanup...") + if self._win.getProperty("simplecachecleanbusy"): + return + self._win.setProperty("simplecachecleanbusy", "busy") + + query = "SELECT id, expires FROM simplecache" + for cache_data in self._execute_sql(query).fetchall(): + cache_id = cache_data[0] + cache_expires = cache_data[1] + + if self._exit or self._monitor.abortRequested(): + return + + # always cleanup all memory objects on each interval + self._win.clearProperty(cache_id) + + # clean up db cache object only if expired + if cache_expires < cur_timestamp: + query = 'DELETE FROM simplecache WHERE id = ?' + self._execute_sql(query, (cache_id,)) + self._log_msg("delete from db %s" % cache_id) + + # compact db + self._execute_sql("VACUUM") + + # remove task from list + self._busy_tasks.remove(__name__) + self._win.setProperty("simplecache.clean.lastexecuted", repr(cur_time)) + self._win.clearProperty("simplecachecleanbusy") + self._log_msg("Auto cleanup done") + + def _get_database(self): + '''get reference to our sqllite _database - performs basic integrity check''' + addon = xbmcaddon.Addon(ADDON_ID) + dbpath = addon.getAddonInfo('profile') + dbfile = xbmcvfs.translatePath("%s/simplecache.db" % dbpath) + + if not xbmcvfs.exists(dbpath): + xbmcvfs.mkdirs(dbpath) + del addon + try: + connection = sqlite3.connect(dbfile, timeout=30, isolation_level=None) + connection.execute('SELECT * FROM simplecache LIMIT 1') + return connection + except Exception as error: + # our _database is corrupt or doesn't exist yet, we simply try to recreate it + if xbmcvfs.exists(dbfile): + xbmcvfs.delete(dbfile) + try: + connection = sqlite3.connect(dbfile, timeout=30, isolation_level=None) + connection.execute( + """CREATE TABLE IF NOT EXISTS simplecache( + id TEXT UNIQUE, expires INTEGER, data TEXT, checksum INTEGER)""") + return connection + except Exception as error: + self._log_msg("Exception while initializing _database: %s" % str(error), xbmc.LOGWARNING) + self.close() + return None + + def _execute_sql(self, query, data=None): + '''little wrapper around execute and executemany to just retry a db command if db is locked''' + retries = 0 + result = None + error = None + # always use new db object because we need to be sure that data is available for other simplecache instances + with self._get_database() as _database: + while not retries == 10 and not self._monitor.abortRequested(): + if self._exit: + return None + try: + if isinstance(data, list): + result = _database.executemany(query, data) + elif data: + result = _database.execute(query, data) + else: + result = _database.execute(query) + return result + except sqlite3.OperationalError as error: + if "_database is locked" in error: + self._log_msg("retrying DB commit...") + retries += 1 + self._monitor.waitForAbort(0.5) + else: + break + except Exception as error: + break + self._log_msg("_database ERROR ! -- %s" % str(error), xbmc.LOGWARNING) + return None + + @staticmethod + def _log_msg(msg, loglevel=xbmc.LOGDEBUG): + '''helper to send a message to the kodi log''' + xbmc.log("Skin Helper Simplecache --> %s" % msg, level=loglevel) + + @staticmethod + def _get_timestamp(date_time): + '''Converts a datetime object to unix timestamp''' + return int(time.mktime(date_time.timetuple())) + + def _get_checksum(self, stringinput): + '''get int checksum from string''' + if not stringinput and not self.global_checksum: + return 0 + if self.global_checksum: + stringinput = "%s-%s" %(self.global_checksum, stringinput) + else: + stringinput = str(stringinput) + return reduce(lambda x, y: x + y, map(ord, stringinput)) + + +def use_cache(cache_days=14): + ''' + wrapper around our simple cache to use as decorator + Usage: define an instance of SimpleCache with name "cache" (self.cache) in your class + Any method that needs caching just add @use_cache as decorator + NOTE: use unnamed arguments for calling the method and named arguments for optional settings + ''' + def decorator(func): + '''our decorator''' + def decorated(*args, **kwargs): + '''process the original method and apply caching of the results''' + method_class = args[0] + method_class_name = method_class.__class__.__name__ + cache_str = "%s.%s" % (method_class_name, func.__name__) + # cache identifier is based on positional args only + # named args are considered optional and ignored + for item in args[1:]: + cache_str += u".%s" % item + cache_str = cache_str.lower() + cachedata = method_class.cache.get(cache_str) + global_cache_ignore = False + try: + global_cache_ignore = method_class.ignore_cache + except Exception: + pass + if cachedata is not None and not kwargs.get("ignore_cache", False) and not global_cache_ignore: + return cachedata + else: + result = func(*args, **kwargs) + method_class.cache.set(cache_str, result, expiration=datetime.timedelta(days=cache_days)) + return result + return decorated + return decorator diff --git a/updatefeed.py b/updatefeed.py new file mode 100755 index 0000000..be5ceb0 --- /dev/null +++ b/updatefeed.py @@ -0,0 +1,10 @@ +from newsfeed import * +from environment import * + +logger = myLog() +logger.write("updatefeed.py running") +newsFeed=NewsFeed(PATH_VIDEO_DATABASE, logger) +logger.write("getItemsInFeed") +videos=newsFeed.getItemsInFeed(FOX_NEWS_URL) +logger.write(f"updatefeed got {len(videos)}") +logger.write("updatefeed.py Done.") diff --git a/utility.py b/utility.py new file mode 100755 index 0000000..08bb68e --- /dev/null +++ b/utility.py @@ -0,0 +1,283 @@ +import time +import webbrowser +import requests +from datetime import timedelta +from datetime import datetime +from datetime import timezone +from environment import * + +class myLog(): + def __init__(self): + self._file = open(PATH_LOG_FILE,"a",encoding='utf-8') + + def write(self,item): + currentDateTime = DateTimeHelper.getCurrentDateTime() + strCurrentDateTime = DateTimeHelper.getDateTimeAsString(currentDateTime) + strOutput = '[' + strCurrentDateTime +'] '+item + self._file.write(strOutput) + self._file.write("\n") + self._file.flush() + +class Utility: + def __init__(self): + pass + + @staticmethod + def pad(strItem, strPad, length): + while len(strItem) maxretries: + raise + + +class DateTimeHelper: + def __init__(self): + pass + + def __init__(self,capturedatetime,timeChange): + self.datetimestamp=capturedatetime + self.timeChange=timeChange + self.offsetTime=DateTimeHelper.applyRelativeTime(self.datetimestamp, self.timeChange) + + def getDateTimeStamp(self): + return self.datetimestamp + + def getTimeChange(self): + return self.timeChange + + def getOffsetTime(self): + return self.offsetTime + + def getOffsetTimeAsString(self): + return DateTimeHelper.getDateTimeAsString(self.offsetTime) + + def toString(self): + pass + + @staticmethod + def getDateTimeAsString(someDateTime): + if(not isinstance(someDateTime,datetime)): + raise Exception('Invalid type for parameter') + return someDateTime.strftime("%m-%d-%Y %H:%M:%S") + + @staticmethod + def getDateTimeFromString(someDateTimeString): + if(not isinstance(someDateTimeString,str)): + raise Exception('Invalid type for parameter') + return DateTimeHelper.strptime(someDateTimeString,"%m-%d-%Y %H:%M:%S") + + @staticmethod + def getCurrentDateTime(): + return datetime.now() + + @staticmethod + def strptime(theTime,theFormat): + try: + return datetime.strptime(theTime,theFormat) + except: + return datetime(*(time.strptime(theTime,theFormat)[0:6])) + + @staticmethod + def canstrptime(theTime,theFormat): + try: + datetime.strptime(theTime,theFormat) + return True + except: + return False + +# returns a datetime + @staticmethod + def applyRelativeTime(sometime,relativetime): + if(not isinstance(sometime,datetime)): + raise Exception('Invalid type for parameter') + if(not isinstance(relativetime,str)): + raise Exception('Invalid type for parameter') + if DateTimeHelper.canstrptime(relativetime,'%B %d, %Y'): + sometime = DateTimeHelper.strptime(relativetime,'%B %d, %Y') + return sometime + if relativetime=='just now': + return sometime + if relativetime=='just in': + return sometime + relativetimesplit=relativetime.split() + if len(relativetimesplit)==2: + year=datetime.now().year + relativetimex=relativetime+', '+str(year) + relativeDate = DateTimeHelper.strptime(relativetimex, '%B %d, %Y') + if(relativeDate>datetime.now()): + year=datetime.now().year-1 + relativetimex=relativetime+', '+str(year) + relativeDate=DateTimeHelper.strptime(relativetimex,'%B %d, %Y') + days=sometime-relativeDate + sometime=sometime-days + elif relativetimesplit[1]=='hour' or relativetimesplit[1]=='hours': + hours=int(relativetimesplit[0]) + sometime=sometime-timedelta(hours=hours) + elif relativetimesplit[1]=='day' or relativetimesplit[1]=='days': + days=int(relativetimesplit[0]) + sometime=sometime-timedelta(days=days) + elif relativetimesplit[1]=='minute' or relativetimesplit[1]=='minutes': + minutes=int(relativetimesplit[0]) + sometime=sometime-timedelta(minutes=minutes) + elif len(relativetimesplit)==3: # '16 mins ago' '2 hours ago' + if relativetimesplit[1]=='mins': + minutes=int(relativetimesplit[0]) + sometime=sometime-timedelta(minutes=minutes) + elif relativetimesplit[1]=='hours': + hours=int(relativetimesplit[0]) + sometime=sometime-timedelta(hours=hours) + elif relativetimesplit[1]=='day' or relativetimesplit[1]=='days': + days=int(relativetimesplit[0]) + sometime=sometime-timedelta(days=days) + return sometime + +class DateTime: + def __init__(self): + self.date=DateTime.getCurrentTime() + + def __init__(self,strDate=None): + if None!=strDate: + self.date=DateTime.dateFromString(strDate) + else: + self.date=DateTime.getCurrentTime() + + def toString(self): + return DateTime.dateToString(self.date) + + def toStringMonthDay(self): + return self.getMonthAsString() + ' ' + Utility.pad(str(self.date.day),'0',2) + ', '+ str(self.date.year) + + def getMonthAsString(self): + strMonth=None + if(self.date.month==1): + strMonth='January' + elif(self.date.month==2): + strMonth='February' + elif(self.date.month==3): + strMonth='March' + elif(self.date.month==4): + strMonth='April' + elif(self.date.month==5): + strMonth='May' + elif(self.date.month==6): + strMonth='June' + elif(self.date.month==7): + strMonth='July' + elif(self.date.month==8): + strMonth='August' + elif(self.date.month==9): + strMonth='September' + elif(self.date.month==10): + strMonth='October' + elif(self.date.month==11): + strMonth='November' + elif(self.date.month==12): + strMonth='December' + else: + strMonth='???' + return strMonth + + def deltaTime(self,someDate): + return DateTime.deltaTime(self.date,someDate) + + def getDate(self): + return self.date + + @staticmethod + def fromdatetime(somedatetime): + if(not isinstance(somedatetime,datetime)): + raise Exception('Invalid type for parameter') + theDate=DateTime() + theDate.date=somedatetime + return theDate + + @staticmethod + def dateToString(someDate): + return str(someDate) + + @staticmethod + def dateFromString(strDate): + return datetime.fromisoformat(strDate) + + @staticmethod + def now(): + return DateTime.getCurrentTime() + + @staticmethod + def getCurrentTime(): + return datetime.now(timezone.utc) + + @staticmethod + def sortList(dateList): + dateList.sort(key=lambda x:x.toString()) + + @staticmethod + def deltaTime(startTime,endTime): + if startTime > endTime: + timedelta=startTime-endTime + else: + timedelta=endTime-startTime + days, seconds=timedelta.days, timedelta.seconds + hours=timedelta.total_seconds()//3600 + minutes=(seconds %3600)//60 + seconds=seconds%60 + return days, hours, minutes, seconds + + + +#currentDate=DateTimeHelper.getCurrentDateTime() +#strDateTime=DateTimeHelper.getDateTimeAsString(currentDate) +#print(strDateTime) +#relativeTime=DateTimeHelper.applyRelativeTime(currentDate,'October 9') +#strDateTime=DateTimeHelper.getDateTimeAsString(relativeTime) +#print(relativeTime) +#if(relativeTime>currentDate): +# print('It is greater') + diff --git a/video.py b/video.py new file mode 100644 index 0000000..b5f5f15 --- /dev/null +++ b/video.py @@ -0,0 +1,132 @@ +from datetime import timedelta +from datetime import datetime +from datetime import timezone +from environment import * +from utility import * +import os + +class Video: + def __init__(self): + self.description=None + self.url=None + self.icon=None + self.timestamp=DateTime() + self.feedTimeOffset='just now' + self.feedtime=datetime.now() + + def __init__(self, description, url): + self.description=description + self.url=url + self.icon=None + self.timestamp=DateTime() + self.feedTimeOffset='just now' + self.feedtime=datetime.now() + + def __init__(self, description, url, icon, timestamp=None): + self.description=description + self.url=url + self.icon=icon + if None==timestamp: + self.timestamp=DateTime() + else: + self.timestamp=timestamp + self.feedTimeOffset='just now' + self.feedtime=datetime.now() + + def getDescription(self): + return self.description + + def getUrl(self): + return self.url + + def getIcon(self): + return self.icon + + def getTimestamp(self): + return self.timestamp + +# This is a datetime. This time gets calculated by applying the feedtime offset to today. + def setFeedTime(self,feedtime): + self.feedtime=feedtime + + def getFeedTime(self): + return self.feedtime + +# This is the