import random as rnd
from datetime import datetime

import pandas as pd
from mimesis import Finance, Person
from mimesis.data import STOCK_EXCHANGES, CURRENCY_ISO_CODES

start = datetime.now()
def replace (rnd, mylist: list, perc: float=0.01, value=None):
    sample = int(len(mylist) * perc)
    source_list = mylist[sample:]
    empties_list = [None for _ in range(sample)]
    incomplete_list = (source_list + empties_list)
    rnd.shuffle(incomplete_list)
    return incomplete_list

def generate_ric(rnd):
    _alph= "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
    ticker = "".join(rnd.choices(_alph, k=3))
    exchange = "".join(rnd.choices(_alph, k=2))
    return "{}.{}".format(ticker, exchange)

# Number of rows to generate
nrows = 1000000
data = {}

# set the seed for all generators
seed = "aws"
rnd.seed(seed)
finance = Finance(seed=seed)
person = Person(seed=seed)

data["ts"] = [datetime(2022, 11, 23, 5, 00).timestamp() + rnd.randint(1e6, 3e6) for i in range (nrows)]
data["owner"] = [person.identifier(mask="####-####") for i in range(nrows)]

print (f"{datetime.now()-start} basic IDs ... ")
market_type = ["bond", "commodity", "derivatives", "forex", "otc", "pe", "spot", "stock", None]
data["market_type"] = [rnd.randrange(0, 9, 1) for i in range(nrows)]
data["ticker"] = [finance.stock_ticker() for i in range(nrows)]
data["ric"] = [generate_ric(rnd) for i in range(nrows)]
data["accountId"] = [person.identifier(mask="######") for i in range(nrows)]
data["geo"] = [rnd.choice(CURRENCY_ISO_CODES) for i in range(nrows)]
data["src_market"] = [rnd.choice(STOCK_EXCHANGES) for i in range(nrows)]

print (f"{datetime.now()-start} metrics 1 ... ")
metrics = [f"m_{col}" for col in range (20)]  # normal(100, 33)
for m in metrics:
    data[m] = [rnd.gauss(100, 33) for i in range(nrows)]
metrics = [f"i_{col}" for col in range (15)]  # [1,17]
for m in metrics:
    data[m] = [rnd.randrange(1, 17, 2) for i in range(nrows)]

print (f"{datetime.now()-start} metrics 2 ... ")
metrics = [f"b_{col}" for col in range (5)]  # [Y,N]
for m in metrics:
    data[m] = [rnd.choice(["Y", "N"]) for i in range(nrows)]

print (f"{datetime.now()-start} sod and eod ... ")
data["m_sod"] = [rnd.lognormvariate(100, 0.2) for i in range(nrows)]
data["m_eod"] = [rnd.lognormvariate(50, 0.2) for i in range(nrows)]

volumes = ["vol_10", "vol_30", "vol_60", "vol_90", "vol_180", "vol_360"] # 6
for v in volumes:
    data[v] = [rnd.randrange(12543, 58902, 1247) for i in range(nrows)]

print (f"{datetime.now()-start} risks  ... ")
risks = [f"risk_{col}" for col in range (10)]  # 10
for r in risks:
    data[r] = replace(rnd, [rnd.betavariate(0.6, 0.221) for i in range(nrows)], 0.1)

print (f"{datetime.now()-start} forecast days ... ")
forecasts = [f"for_{days}" for days in [1, 5, 10, 15, 30, 60, 90, 120, 240, 365, 720]] # 11
for f in forecasts:
    data[f] = [rnd.gauss(502, 253) for i in range(nrows)]

print (f"{datetime.now()-start} forecast years ... ")
data["for_3y"] = replace(rnd, [rnd.gauss(502, 253) for i in range(nrows)], 0.01)
data["for_5y"] = replace(rnd, [rnd.gauss(502, 253) for i in range(nrows)], 0.05)

print (f"{datetime.now()-start} sentiment ... ")
sentiment = ["pos", "neg", "neutral"]
for f in forecasts:
    data[f"sent_{f}"] = [rnd.choice(sentiment) for i in range(nrows)]

print (f"{datetime.now()-start} exchanges ... ")
for se in STOCK_EXCHANGES: # 7
    data[f"{se}_adj"] = [rnd.uniform(0.0, 1.0) for i in range(nrows)]
for se in STOCK_EXCHANGES: # 7
    data[f"{se}_corr"] = [rnd.uniform(-1.0, 1.0) for i in range(nrows)]

print (f"{datetime.now()-start} creating dataframe ... ")
df = pd.DataFrame(data=data)
print (f"{datetime.now()-start} storing dataset with {len(df.columns)} cols and {nrows} rows ... ")

df.to_parquet(f"dataset_{len(df.columns)}_{nrows}")
print(f"total time: {datetime.now() - start}")