from pycoingecko import CoinGeckoAPI
gecko = CoinGeckoAPI()

import pandas as pd

import matplotlib.pyplot as plt
%matplotlib notebook

import datetime as dt

import requests

from time import sleep

from IPython.display import clear_output

def __get_historical_data_url(coin_id, currency_id, start_date, end_date):
    return 'https://www.coingecko.com/en/coins/{}/historical_data/{}?end_date={}&start_date={}'.format(coin_id, currency_id, end_date.isoformat(), start_date.isoformat()) 

def get_historical_data_scraper(coin_id, currency_id, start_date_s, end_date_s):
    start_date = dt.date.fromisoformat(start_date_s)
    end_date = dt.date.fromisoformat(end_date_s)

    url = __get_historical_data_url(coin_id, currency_id, start_date, end_date)
    r = requests.get(url, timeout=5, stream=True)
    
    try:
        dfs = pd.read_html(r.content, parse_dates=['Date'])
    except:
        return None
    if len(dfs) != 1:
        return None

    df = dfs[0][::-1].set_index("Date")
    df = df.replace({'[$,]': ''}, regex=True).apply(pd.to_numeric)
    df.columns = pd.MultiIndex.from_product([[coin_id], df.columns])

    return df

def get_historical_data_gecko(coin_id, currency_id, start_date_s, end_date_s):
    p = gecko.get_coin_market_chart_range_by_id(coin_id, currency_id, dt.datetime.fromisoformat(start_date_s).strftime("%s"), dt.datetime.fromisoformat(end_date_s).strftime("%s"))

    df_mcaps = pd.DataFrame(p['market_caps'], columns=['Date', 'Market Cap'])
    df_mcaps['Date'] = pd.to_datetime(df_mcaps['Date'].apply(lambda x: dt.datetime.utcfromtimestamp(x/1000).date()))
    df_mcaps.set_index("Date", inplace=True)

    df_vol = pd.DataFrame(p['total_volumes'], columns=['Date', 'Volume'])
    df_vol['Date'] = pd.to_datetime(df_vol['Date'].apply(lambda x: dt.datetime.utcfromtimestamp(x/1000).date()))
    df_vol.set_index("Date", inplace=True)

    df_prices = pd.DataFrame(p['prices'], columns=['Date', 'Open'])
    df_prices['Date'] = pd.to_datetime(df_prices['Date'].apply(lambda x: dt.datetime.utcfromtimestamp(x/1000).date()))
    df_prices.set_index("Date", inplace=True)

    df = pd.concat([df_mcaps, df_vol, df_prices], axis=1)
    df['Close'] = df['Open'].shift(-1)

    df.columns = pd.MultiIndex.from_product([[coin_id], df.columns])

    return df

def get_historical_data(coin_id, currency_id, start_date_s, end_date_s):
    df = None
    try:
        df = get_historical_data_gecko(coin_id, currency_id, start_date_s, end_date_s)
    except:
        try:
            df = get_historical_data_scraper(coin_id, currency_id, start_date_s, end_date_s)
        except:
            return None
    return df

df = get_historical_data('bitcoin', 'usd', '2013-01-01', '2021-04-01')
df.head()

coins_list = pd.DataFrame(gecko.get_coins_list())

df_complete = None
coins_list_ids = coins_list['id']
coins_list_ids_len = len(coins_list_ids)

i = 2206
timeout = 1
while i < coins_list_ids_len: 
    coin = coins_list_ids[i]
    
    df = get_historical_data(coin, 'usd', '2013-01-01', '2021-04-01')
    if df is None:
        print ("Failed to download", coin)
        sleep(timeout)
        continue
    
    i = i + 1
    if df_complete is None:
        df_complete = df
    else:
        df_complete = df_complete.join(df, how='outer')
    
    clear_output(wait=True)
    print ("Downloaded", coin, "{}/{}".format(i, coins_list_ids_len - 1))

# df_complete.to_parquet("CoinGecko_2013-04-28_2021-03-28_2205.parquet")

df = get_historical_data('0-5x-long-shitcoin-index-token', 'usd', '2013-01-01', '2021-04-01')

df_complete.head()

df_complete_first

df_complete_second

df_complete_third

df_complete.info()

coins_list.iloc[109]

df_complete = df_complete_first.join(df_complete_second.join(df_complete_third, how='outer'), how='outer')

df_complete.to_parquet("CoinGecko_2013-04-28_2021-03-28.parquet")

df_spendcoin = get_historical_data('spendcoin', 'usd', '2010-01-01', '2021-03-28')

df_spendcoin['spendcoin']['Close'].plot()

coin_id = 'bitcoin'
currency_id = 'usd'
start_date_s = '2013-01-01'
end_date_s = '2021-04-01'

p = gecko.get_coin_market_chart_range_by_id(coin_id, currency_id, dt.datetime.fromisoformat(start_date_s).strftime("%s"), dt.datetime.fromisoformat(end_date_s).strftime("%s"))

df_mcaps = pd.DataFrame(p['market_caps'], columns=['Date', 'Market Cap'])
df_mcaps['Date'] = pd.to_datetime(df_mcaps['Date'].apply(lambda x: dt.datetime.utcfromtimestamp(x/1000).date()))
df_mcaps.set_index("Date", inplace=True)

df_vol = pd.DataFrame(p['total_volumes'], columns=['Date', 'Volume'])
df_vol['Date'] = pd.to_datetime(df_vol['Date'].apply(lambda x: dt.datetime.utcfromtimestamp(x/1000).date()))
df_vol.set_index("Date", inplace=True)

df_prices = pd.DataFrame(p['prices'], columns=['Date', 'Open'])
df_prices['Date'] = pd.to_datetime(df_prices['Date'].apply(lambda x: dt.datetime.utcfromtimestamp(x/1000).date()))
df_prices.set_index("Date", inplace=True)

df = pd.concat([df_mcaps, df_vol, df_prices], axis=1)
df['Close'] = df['Open'].shift(-1)

df.columns = pd.MultiIndex.from_product([[coin_id], df.columns])

df = get_historical_data('bitcoin', 'usd', '2013-01-01', '2021-04-01')
df.head()