## object to retrieve feed and extract news Items from it

import requests
from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry
from classes.feed import Feed
from datetime import datetime,timezone
from classes.item import Item
from email.utils import parsedate_to_datetime

class FeedProcessor:
    def __init__(self, feed):
        self.feed = feed;
        
    def setFeed(self, feed):
        self.feed = feed;
        
    def retrieveFeed(self):
        
        #set up conditions for retrying failed feed retrievals
        retry_policy = Retry(
            total=3,
            connect=3,
            read=3,
            status=3,
            allowed_methods={"GET","HEAD"},
            status_forcelist={429,500,502,503,504},
            backoff_factor=1,
            backoff_max=30,

            respect_retry_after_header=True)
        session = requests.Session()
        adapter = HTTPAdapter(max_retries=retry_policy)
        
        session.mount("https://", adapter)
        session.mount("http://",adapter)
        
        try:
            response = session.get(self.feed.feedURL(),
            
        headers = {"User-Agent": "SocialistNewsApp Crawler 0.1 (https://socialistnewsapp.com/contact.html)"},
        
        timeout=(10,30))
     
            response.raise_for_status()
            self.feed.setFeedContent(response.text)
        
        except requests.RequestException as error:
            print("Retrieval failed: ", error)
            
        finally:
            session.close()
            
    def extractItems(self):
        
        content = self.feed.feedContent()
        
        position = content.find("<item>")
        
        #print("opening tag:", position)
        
        endPosition = content.find("</item>")
        
        #print("closing tag:", endPosition)
        
        #remove everything in  content before the first item, so grabbing the first link etc is easier
        content = content[position:]
        
        #loop through feed data until all items are extracted
        while "<item>" in content:
            
            startPos = content.find("<item>")
            
            endPos = content.find("</item>")
            
            #check that a complete item exists
            itemString = None
            
            if startPos < endPos:
                itemString = content[startPos:endPos + 7]
                
            #extract data
            if itemString != None:
                startPos = content.find("<title>")
                
                endPos = content.find("</title>")
                
                title = content[startPos + 7:endPos]
                
                startPos = content.find("<link>")
                
                endPos = content.find("</link>")
                
                link = content[startPos + 6:endPos]
                
                startPos = content.find("<pubDate>")
                
                endPos = content.find("</pubDate>")
                
                pubDate = content[startPos + 9:endPos]
                
                #convert pubDate to timestamp
                timestamp = self.date2timestamp(pubDate)
                
                #remove this item from the feed string copy
                endPos = content.find("</item>")
                content = content[endPos + 7:]
               
               
               #validate data and insert item into feed
                if len(title) > 5 and len(link) > 10 and link[0:4] == "http" and timestamp > 0:
                    temp = Item()
                    temp.setTitle(title)
                    temp.setURL(link)
                    temp.setTimestamp(timestamp)
                    temp.setDisplayURL(self.feed.getDisplayURL())
                    
                    self.feed.addItem(temp)
                  
    def date2timestamp(self, string):
        pubDate = string
        
        #strip out troublesome characters
        pubDate = pubDate.replace(",","")
        pubDate = pubDate.replace("-","")
        pubDate = pubDate.replace("  ","")
        
        #convert into a list
        date = pubDate.split()
        
        #convert black agenda report date string into the standard format
        if len(date) == 5:
            date.append("+0000")
            pubDate = date[0] + " " + date[1] + " " + date[2] + " " + date[3] + " " + date[4]
        
        #convert date back to string
        if len(date) == 6:
            date = date[0] + " " + date[1] + " " + date[2] + " " + date[3] + " " + date[4] + " " + date[5].lower()
            #print("date: " + date)
        else:
            temp = " ".join(date)
            print("date length: " + len(date))
            print("date: " + date)
        
        #convert date to timestamp and return it
        timestamp = 0
        
#        try:
#            #try first date format
#            date = datetime.strptime(date, "%a %d %b %Y %H:%M:%S %z")
#            date = date.replace(tzinfo=timezone.utc)  
#        
#            timestamp = date.timestamp()
#            timestamp = int(timestamp)
#            
#            return timestamp
#            
#        except ValueError:
#            print("date:" + date)
            
#        try:
            #try first date format
#            date = datetime.strptime(date, "%a %d %b %Y %H:%M:%S %Z")
#            date = date.replace(tzinfo=timezone.utc)  
#        
#            timestamp = date.timestamp()
#            timestamp = int(timestamp)
#            
#            return timestamp
#            
#        except ValueError:
#            print("date:" + date)
        
        date = parsedate_to_datetime(date.strip())
        
        timestamp = int(date.timestamp())
        
        return timestamp