import pandas as pd
import requests
import datetime as dt
import asyncio
import io

today = dt.datetime.today().replace(hour=8, minute=0, second=0, microsecond=0).timestamp()
today_minus_seven = (dt.datetime.today().replace(hour=8, minute=0, second=0, microsecond=0) - 
                     dt.timedelta(days=7)).timestamp()
today_minus_eight = (dt.datetime.today().replace(hour=8, minute=0, second=0, microsecond=0) - 
                     dt.timedelta(days=8)).timestamp()

reddit_response = requests.get("https://api.pushshift.io/reddit/search/submission",
                              params={'subreddit': 'stocks',
                                      'before': int(today_minus_seven), 
                                      'after': int(today_minus_eight)})

reddit_response.status_code

reddit_response.json()['data'][0]['title'],reddit_response.json()['data'][0]['selftext'],

ticker_response = requests.get("https://www.sec.gov/include/ticker.txt")

tickers = pd.read_csv(
    io.StringIO(ticker_response.text), 
    delimiter='\t', 
    header=None, 
    usecols=[0],
)[0].to_list()

tickers[:5]

import string                                                                         
from typing import List, Union, Dict, Optional, Any 
from collections import Counter
from requests import Response
from dataclasses import dataclass

@dataclass
class RedditSubmission:
    title: str 
    body: str 
    permalink: str 
    author: str 
    score: float
    timestamp: dt.datetime

    def summarize(self, 
        tickers: List[str], 
        weighted: bool = True
    ) -> Dict[str, Union[float, int]]:
        """ Process RedditSubmission for tickers 
        
        Use a Counter to count the number of times a ticker occurs.
        Include some corrections for punctuation
        """
        if self.title is not None:
            title_no_punctuation = self.title.translate(
                str.maketrans('', '', string.punctuation)
            )
            tickers_title = Counter(
                filter(lambda x: x in tickers, title_no_punctuation.split())
            )
        else:
            tickers_title = Counter()
        if self.body is not None:
            body_no_punctuation = self.body.translate(
                str.maketrans('', '', string.punctuation)
            )

            tickers_body = Counter(
                filter(lambda x: x in tickers, body_no_punctuation.split())
            )
        else:
            tickers_body = Counter()
        total_tickers = tickers_title + tickers_body
        
        return total_tickers
    
    def to_dict(self):
        return {
            'title': self.title,
            'body': self.body,
            'permalink': self.permalink,
            'author': self.author,
            'score': self.score,
            'timestamp': self.timestamp
        }
    
    @classmethod
    def from_response(
        cls, 
        resp_object: Response
    ) -> Optional[List[Any]]:
        """ Create a list of RedditSubmission objects from response"""
        if resp_object.status_code == 200:
            processed_response = [
                cls(
                    msg.get("title", None),
                    msg.get("body", None),
                    msg.get("permalink", None),
                    msg.get("author", None),
                    msg.get("score", None),
                    (
                        dt.datetime.fromtimestamp(msg['created_utc']) 
                        if msg['created_utc'] is not None else None
                    )
                ) for msg in resp_object.json()['data']
            ]
            return processed_response
        else:
            return None

async def submission_request_coroutine(**kwargs):
    await asyncio.sleep(5)
    reddit_response = requests.get("https://api.pushshift.io/reddit/search/submission",
                              params=kwargs)
    return reddit_response

snapshots = pd.date_range(
    start=dt.datetime.now(tz=dt.timezone.utc) - dt.timedelta(days=7),
    end=dt.datetime.now(tz=dt.timezone.utc) - dt.timedelta(days=1),
    freq='10min'
)

tasks = [
    submission_request_coroutine(subreddit='stocks', 
                 after=int(snapshot.timestamp()),
                 before=int(snapshots[i+1].timestamp()),
                 size=10
                ) 
    for i, snapshot in enumerate(snapshots[:-1])
]
all_submission_responses = await asyncio.gather(
    *tasks
)

import itertools as it

reddit_submissions = [*it.chain.from_iterable(
    RedditSubmission.from_response(resp) for resp in all_submission_responses
    if resp.status_code == 200
)]

from functools import reduce
from collections import Counter

def aggregate_dictionaries(d1, d2):
    """ Given two dictionaries, aggregate key-value pairs """
    if len(d1) == 0:
        return dict(Counter(**d2).most_common())
    my_counter = Counter(**d1)
    my_counter.update(d2)
    return dict(my_counter.most_common())

submissions_breakdown = reduce(
    aggregate_dictionaries, 
    (submission.summarize(tickers) for submission in reddit_submissions)
)

submissions_breakdown['gme'], submissions_breakdown['amc']

submissions_breakdown

df = pd.DataFrame(a.to_dict() for a in reddit_submissions)

df